author | Claus Gittinger <cg@exept.de> |
Fri, 28 Jun 2019 08:49:56 +0200 | |
changeset 24385 | dd84e4dfb5cc |
parent 24214 | 7853c5e56c17 |
child 25270 | abd76d94ad4f |
permissions | -rw-r--r-- |
8903 | 1 |
" |
2 |
COPYRIGHT (c) 2005 by eXept Software AG |
|
3 |
All Rights Reserved |
|
4 |
||
5 |
This software is furnished under a license and may be used |
|
6 |
only in accordance with the terms of that license and with the |
|
7 |
inclusion of the above copyright notice. This software may not |
|
8 |
be provided or otherwise made available to, or used by, any |
|
9 |
other person. No title to or ownership of the software is |
|
10 |
hereby transferred. |
|
11 |
" |
|
12 |
"{ Package: 'stx:libbasic' }" |
|
13 |
||
14 |
"{ NameSpace: CharacterEncoderImplementations }" |
|
15 |
||
22472 | 16 |
VariableBytesEncoder subclass:#ISO10646_to_UTF16BE |
8903 | 17 |
instanceVariableNames:'' |
18 |
classVariableNames:'' |
|
19 |
poolDictionaries:'' |
|
20 |
category:'Collections-Text-Encodings' |
|
21 |
! |
|
22 |
||
23 |
!ISO10646_to_UTF16BE class methodsFor:'documentation'! |
|
24 |
||
25 |
copyright |
|
26 |
" |
|
27 |
COPYRIGHT (c) 2005 by eXept Software AG |
|
28 |
All Rights Reserved |
|
29 |
||
30 |
This software is furnished under a license and may be used |
|
31 |
only in accordance with the terms of that license and with the |
|
32 |
inclusion of the above copyright notice. This software may not |
|
33 |
be provided or otherwise made available to, or used by, any |
|
34 |
other person. No title to or ownership of the software is |
|
35 |
hereby transferred. |
|
36 |
" |
|
37 |
! |
|
38 |
||
39 |
documentation |
|
40 |
" |
|
41 |
encodes/decodes UTF16 BigEndian (big-end-first) |
|
21300 | 42 |
|
43 |
Notice the naming (many are confused): |
|
44 |
Unicode is the set of number-to-glyph assignments |
|
45 |
whereas: |
|
46 |
UTF8, UTF16 etc. are a concrete way of xmitting Unicode codePoints (numbers). |
|
47 |
||
48 |
ST/X NEVER uses UTF8 or UTF16 internally - all characters are full 24bit characters. |
|
49 |
Only when exchanging data, are these converted into UTF8 (or other) byte sequences. |
|
8903 | 50 |
" |
51 |
! |
|
52 |
||
53 |
examples |
|
54 |
" |
|
55 |
Encoding (unicode to utf16BE) |
|
56 |
ISO10646_to_UTF16BE encodeString:'hello'. |
|
57 |
||
58 |
||
59 |
Decoding (utf16BE to unicode): |
|
60 |
|t| |
|
61 |
||
24214 | 62 |
t := ISO10646_to_UTF16BE encodeString:'ÄÖÜß'. |
8903 | 63 |
ISO10646_to_UTF16BE decodeString:t. |
9325 | 64 |
|
65 |
Decoding (utf16LE-Bytes to unicode): |
|
22472 | 66 |
ISO10646_to_UTF16LE decodeString:#[ 16r40 0 16r41 0 16r42 0 16r43 0 16r44 0 ]. |
67 |
ISO10646_to_UTF16BE decodeString:#[ 16r40 0 16r41 0 16r42 0 16r43 0 16r44 0 ] copy swapBytes. |
|
8903 | 68 |
" |
69 |
! ! |
|
70 |
||
71 |
!ISO10646_to_UTF16BE methodsFor:'encoding & decoding'! |
|
72 |
||
73 |
decodeString:aStringOrByteCollection |
|
14170
eed0dbcc471c
comment/format in: #decodeString:
Stefan Vogel <sv@exept.de>
parents:
12432
diff
changeset
|
74 |
"given a byteArray (2-bytes per character) or unsignedShortArray in UTF16 encoding, |
8903 | 75 |
return a new string containing the same characters, in 8, 16bit (or more) encoding. |
76 |
Returns either a normal String, a TwoByte- or a FourByte-String instance. |
|
77 |
Only useful, when reading from external sources. |
|
78 |
This only handles up-to 30bit characters." |
|
79 |
||
14170
eed0dbcc471c
comment/format in: #decodeString:
Stefan Vogel <sv@exept.de>
parents:
12432
diff
changeset
|
80 |
|s newString bitsPerElementIn nextIn |
12432 | 81 |
codeIn codeIn1 codeIn2 estimatedSize out| |
8903 | 82 |
|
83 |
aStringOrByteCollection isByteArray ifTrue:[ |
|
84 |
bitsPerElementIn := 8. |
|
85 |
] ifFalse:[ |
|
86 |
aStringOrByteCollection isString ifTrue:[ |
|
87 |
bitsPerElementIn := aStringOrByteCollection bitsPerCharacter. |
|
88 |
] ifFalse:[ |
|
14175
84e1adb65a5d
changed: #decodeString: detect odd number of bytes
Stefan Vogel <sv@exept.de>
parents:
14170
diff
changeset
|
89 |
"can be a ShortArray" |
8903 | 90 |
bitsPerElementIn := 16. |
91 |
]. |
|
92 |
]. |
|
93 |
||
14175
84e1adb65a5d
changed: #decodeString: detect odd number of bytes
Stefan Vogel <sv@exept.de>
parents:
14170
diff
changeset
|
94 |
s := aStringOrByteCollection readStream. |
8903 | 95 |
bitsPerElementIn == 8 ifTrue:[ |
14175
84e1adb65a5d
changed: #decodeString: detect odd number of bytes
Stefan Vogel <sv@exept.de>
parents:
14170
diff
changeset
|
96 |
s size odd ifTrue:[ |
14208 | 97 |
InvalidEncodingError raiseWith:aStringOrByteCollection errorString:' - size is not a multiple of 2 bytes'. |
14175
84e1adb65a5d
changed: #decodeString: detect odd number of bytes
Stefan Vogel <sv@exept.de>
parents:
14170
diff
changeset
|
98 |
]. |
8903 | 99 |
nextIn := [self nextTwoByteValueFrom:s]. |
100 |
] ifFalse:[ |
|
101 |
nextIn := [s next]. |
|
102 |
]. |
|
103 |
||
14175
84e1adb65a5d
changed: #decodeString: detect odd number of bytes
Stefan Vogel <sv@exept.de>
parents:
14170
diff
changeset
|
104 |
estimatedSize := s size * bitsPerElementIn // 16. |
12432 | 105 |
out := CharacterWriteStream on:(String new:estimatedSize). |
8903 | 106 |
[s atEnd] whileFalse:[ |
107 |
codeIn := nextIn value. |
|
24214 | 108 |
codeIn > 16rFF ifTrue:[ |
8903 | 109 |
(codeIn between:16rD800 and:16rDBFF) ifTrue:[ |
110 |
codeIn1 := codeIn. |
|
111 |
codeIn2 := nextIn value. |
|
112 |
codeIn := ((codeIn1 - 16rD800) bitShift:10) |
|
113 |
+ |
|
114 |
(codeIn2 - 16rDC00) |
|
115 |
+ 16r00010000. |
|
116 |
]. |
|
117 |
]. |
|
12432 | 118 |
out nextPut:(Character value:codeIn). |
8903 | 119 |
]. |
12432 | 120 |
newString := out contents. |
121 |
||
122 |
"/ nBitsRequired := 8. |
|
123 |
"/ sz := 0. |
|
124 |
"/ [s atEnd] whileFalse:[ |
|
125 |
"/ codeIn := nextIn value. |
|
126 |
"/ sz := sz + 1. |
|
127 |
"/ |
|
128 |
"/ codeIn <= 16rFF ifTrue:[ |
|
129 |
"/ ] ifFalse:[ |
|
130 |
"/ nBitsRequired := nBitsRequired max:16. |
|
131 |
"/ (codeIn between:16rD800 and:16rDBFF) ifTrue:[ |
|
132 |
"/ nBitsRequired := 32. |
|
133 |
"/ codeIn2 := nextIn value. |
|
134 |
"/ ]. |
|
135 |
"/ ] |
|
136 |
"/ ]. |
|
137 |
"/ |
|
138 |
"/ nBitsRequired == 8 ifTrue:[ |
|
139 |
"/ newString := String uninitializedNew:sz |
|
140 |
"/ ] ifFalse:[ |
|
141 |
"/ nBitsRequired <= 16 ifTrue:[ |
|
142 |
"/ newString := Unicode16String new:sz |
|
143 |
"/ ] ifFalse:[ |
|
144 |
"/ newString := Unicode32String new:sz |
|
145 |
"/ ] |
|
146 |
"/ ]. |
|
147 |
"/ |
|
148 |
"/ s := aStringOrByteCollection readStream. |
|
149 |
"/ idx := 1. |
|
150 |
"/ [s atEnd] whileFalse:[ |
|
151 |
"/ codeIn := nextIn value. |
|
152 |
"/ codeIn <= 16rFF ifTrue:[ |
|
153 |
"/ ] ifFalse:[ |
|
154 |
"/ nBitsRequired := nBitsRequired max:16. |
|
155 |
"/ (codeIn between:16rD800 and:16rDBFF) ifTrue:[ |
|
156 |
"/ nBitsRequired := 32. |
|
157 |
"/ codeIn1 := codeIn. |
|
158 |
"/ codeIn2 := nextIn value. |
|
159 |
"/ codeIn := ((codeIn1 - 16rD800) bitShift:10) |
|
160 |
"/ + |
|
161 |
"/ (codeIn2 - 16rDC00) |
|
162 |
"/ + 16r00010000. |
|
163 |
"/ ]. |
|
164 |
"/ ]. |
|
165 |
"/ newString at:idx put:(Character value:codeIn). |
|
166 |
"/ idx := idx + 1. |
|
167 |
"/ ]. |
|
8903 | 168 |
^ newString |
169 |
||
170 |
" |
|
171 |
self new decodeString:#[ 16r00 16r42 ] |
|
172 |
self new decodeString:#[ 16r01 16r42 ] |
|
173 |
self new decodeString:#[ 16r00 16r48 |
|
174 |
16r00 16r69 |
|
175 |
16rD8 16r00 |
|
176 |
16rDC 16r00 |
|
177 |
16r00 16r21 |
|
178 |
16r00 16r21 |
|
179 |
] |
|
180 |
||
181 |
self new decodeString:#( 16r0048 |
|
182 |
16r0069 |
|
183 |
16rD800 |
|
184 |
16rDC00 |
|
185 |
16r0021 |
|
186 |
16r0021 |
|
187 |
) |
|
188 |
" |
|
14208 | 189 |
|
190 |
"Modified: / 12-07-2012 / 19:56:12 / cg" |
|
24214 | 191 |
"Modified: / 28-05-2019 / 14:08:19 / Stefan Vogel" |
8903 | 192 |
! |
193 |
||
194 |
encode:aCode |
|
12432 | 195 |
^ aCode |
8903 | 196 |
! |
197 |
||
198 |
encodeString:aUnicodeString |
|
199 |
"return the UTF-16 representation of a aUnicodeString. |
|
200 |
The resulting string is only useful to be stored on some external file, |
|
201 |
not for being used inside ST/X." |
|
202 |
||
22472 | 203 |
|stream size "{ Class:SmallInteger }"| |
8903 | 204 |
|
22472 | 205 |
size := aUnicodeString size. |
24214 | 206 |
stream := WriteStream on:(ByteArray uninitializedNew:size * 2). |
22472 | 207 |
|
208 |
1 to:size do:[:idx | |
|
209 |
stream nextPutUtf16Bytes:(aUnicodeString at:idx) MSB:true. |
|
8903 | 210 |
]. |
211 |
||
22472 | 212 |
^ stream contents |
8903 | 213 |
|
214 |
" |
|
215 |
(self encodeString:'hello') #[0 104 0 101 0 108 0 108 0 111] |
|
216 |
(self encodeString:(Character value:16r40) asString) #[0 64] |
|
217 |
(self encodeString:(Character value:16rFF) asString) #[0 255] |
|
218 |
(self encodeString:(Character value:16r100) asString) #[1 0] |
|
219 |
(self encodeString:(Character value:16r1000) asString) #[16 0] |
|
220 |
(self encodeString:(Character value:16r2000) asString) #[32 0] |
|
221 |
(self encodeString:(Character value:16r4000) asString) #[64 0] |
|
222 |
(self encodeString:(Character value:16r8000) asString) #[128 0] |
|
223 |
(self encodeString:(Character value:16rD7FF) asString) #[215 255] |
|
224 |
(self encodeString:(Character value:16rE000) asString) #[224 0] |
|
225 |
(self encodeString:(Character value:16rFFFF) asString) #[255 255] |
|
226 |
(self encodeString:(Character value:16r10000) asString) #[216 64 220 0] |
|
227 |
(self encodeString:(Character value:16r10FFF) asString) #[216 67 223 255] |
|
228 |
(self encodeString:(Character value:16r1FFFF) asString) #[216 127 223 255] |
|
229 |
(self encodeString:(Character value:16r10FFFF) asString) #[219 255 223 255] |
|
230 |
error cases: |
|
231 |
(self encodeString:(Character value:16rD800) asString) |
|
232 |
(self encodeString:(Character value:16rD801) asString) |
|
233 |
(self encodeString:(Character value:16rDFFF) asString) |
|
234 |
(self encodeString:(Character value:16r110000) asString) |
|
235 |
" |
|
22472 | 236 |
|
237 |
"Modified: / 16-01-2018 / 19:38:30 / stefan" |
|
24214 | 238 |
"Modified: / 28-05-2019 / 13:49:53 / Stefan Vogel" |
22472 | 239 |
! ! |
240 |
||
241 |
!ISO10646_to_UTF16BE methodsFor:'private'! |
|
242 |
||
243 |
nextTwoByteValueFrom:aStream |
|
244 |
^ aStream nextUnsignedInt16MSB:true |
|
245 |
! ! |
|
246 |
||
247 |
!ISO10646_to_UTF16BE methodsFor:'queries'! |
|
248 |
||
249 |
characterSize:charOrCodePoint |
|
250 |
"return the number of bytes required to encode codePoint" |
|
251 |
||
252 |
^ charOrCodePoint codePoint <= 16rFFFF ifTrue:[2] ifFalse:[4] |
|
253 |
||
254 |
"Created: / 16-01-2018 / 19:21:09 / stefan" |
|
255 |
! |
|
256 |
||
257 |
nameOfEncoding |
|
258 |
^ #utf16be |
|
259 |
! ! |
|
260 |
||
261 |
!ISO10646_to_UTF16BE methodsFor:'stream support'! |
|
262 |
||
263 |
encodeCharacter:aUnicodeCharacter on:aStream |
|
264 |
"given a string in unicode, encode it onto aStream." |
|
265 |
||
266 |
aStream nextPutUtf16Bytes:aUnicodeCharacter MSB:true. |
|
267 |
||
268 |
"Created: / 16-02-2017 / 16:41:25 / stefan" |
|
21475 | 269 |
! |
270 |
||
271 |
encodeString:aUnicodeString on:aStream |
|
272 |
"given a string in unicode, encode it onto aStream." |
|
273 |
||
274 |
aStream nextPutAllUtf16Bytes:aUnicodeString MSB:true. |
|
275 |
||
276 |
"Created: / 16-02-2017 / 16:40:32 / stefan" |
|
8903 | 277 |
! |
278 |
||
22472 | 279 |
readNextCharacterFrom:aStream |
280 |
|codeIn codeIn2| |
|
8903 | 281 |
|
22472 | 282 |
codeIn := self nextTwoByteValueFrom:aStream. |
283 |
codeIn isNil ifTrue:[ |
|
284 |
^ nil. |
|
285 |
]. |
|
286 |
(codeIn between:16rD800 and:16rDBFF) ifTrue:[ |
|
287 |
codeIn2 := self nextTwoByteValueFrom:aStream. |
|
288 |
codeIn2 isNil ifTrue:[ |
|
289 |
InvalidEncodingError raiseErrorString:' - UTF16 missing followBytes'. |
|
290 |
]. |
|
291 |
codeIn := ((codeIn - 16rD800) bitShift:10) |
|
292 |
+ (codeIn2 - 16rDC00) |
|
293 |
+ 16r00010000. |
|
294 |
]. |
|
8903 | 295 |
|
22472 | 296 |
^ Character codePoint:codeIn. |
297 |
||
298 |
"Created: / 16-01-2018 / 22:31:29 / stefan" |
|
299 |
"Modified: / 17-01-2018 / 14:41:31 / stefan" |
|
8903 | 300 |
! ! |
301 |
||
302 |
!ISO10646_to_UTF16BE class methodsFor:'documentation'! |
|
303 |
||
304 |
version |
|
19217 | 305 |
^ '$Header$' |
12432 | 306 |
! |
307 |
||
308 |
version_CVS |
|
19217 | 309 |
^ '$Header$' |
8903 | 310 |
! ! |
19217 | 311 |