author | Jan Vrany <jan.vrany@labware.com> |
Thu, 25 Mar 2021 20:30:03 +0000 | |
branch | jv |
changeset 25411 | 248600ba8fd9 |
parent 23547 | c69c97cec351 |
permissions | -rw-r--r-- |
23547
c69c97cec351
Issue #239: Fix all Smalltak/X source files to be in unicode (UTF8 without BOM) and prefixed by "{ Encoding: utf8 }" when any unicode character is present
Patrik Svestka <patrik.svestka@gmail.com>
parents:
19863
diff
changeset
|
1 |
"{ Encoding: utf8 }" |
c69c97cec351
Issue #239: Fix all Smalltak/X source files to be in unicode (UTF8 without BOM) and prefixed by "{ Encoding: utf8 }" when any unicode character is present
Patrik Svestka <patrik.svestka@gmail.com>
parents:
19863
diff
changeset
|
2 |
|
1 | 3 |
" |
4 |
COPYRIGHT (c) 1993 by Claus Gittinger |
|
235 | 5 |
All Rights Reserved |
1 | 6 |
|
7 |
This software is furnished under a license and may be used |
|
8 |
only in accordance with the terms of that license and with the |
|
9 |
inclusion of the above copyright notice. This software may not |
|
10 |
be provided or otherwise made available to, or used by, any |
|
11 |
other person. No title to or ownership of the software is |
|
12 |
hereby transferred. |
|
13 |
" |
|
10223
761e2a050b69
twoByteString moved (req'd in VM)
Claus Gittinger <cg@exept.de>
parents:
8094
diff
changeset
|
14 |
"{ Package: 'stx:libbasic' }" |
8094
d05f69bd0097
Use #codePoint instead of deprecated #asciiValue
Stefan Vogel <sv@exept.de>
parents:
5761
diff
changeset
|
15 |
|
17621 | 16 |
"{ NameSpace: Smalltalk }" |
17 |
||
5761 | 18 |
CharacterArray variableWordSubclass:#TwoByteString |
992
f456f8f7d421
JIS decode now in CharacterArray
Claus Gittinger <cg@exept.de>
parents:
631
diff
changeset
|
19 |
instanceVariableNames:'' |
f456f8f7d421
JIS decode now in CharacterArray
Claus Gittinger <cg@exept.de>
parents:
631
diff
changeset
|
20 |
classVariableNames:'' |
f456f8f7d421
JIS decode now in CharacterArray
Claus Gittinger <cg@exept.de>
parents:
631
diff
changeset
|
21 |
poolDictionaries:'' |
f456f8f7d421
JIS decode now in CharacterArray
Claus Gittinger <cg@exept.de>
parents:
631
diff
changeset
|
22 |
category:'Collections-Text' |
1 | 23 |
! |
24 |
||
89 | 25 |
!TwoByteString class methodsFor:'documentation'! |
26 |
||
27 |
copyright |
|
28 |
" |
|
29 |
COPYRIGHT (c) 1993 by Claus Gittinger |
|
235 | 30 |
All Rights Reserved |
1 | 31 |
|
89 | 32 |
This software is furnished under a license and may be used |
33 |
only in accordance with the terms of that license and with the |
|
34 |
inclusion of the above copyright notice. This software may not |
|
35 |
be provided or otherwise made available to, or used by, any |
|
36 |
other person. No title to or ownership of the software is |
|
37 |
hereby transferred. |
|
38 |
" |
|
39 |
! |
|
40 |
||
41 |
documentation |
|
42 |
" |
|
43 |
TwoByteStrings are like strings, but storing 16bits per character. |
|
44 |
The integration of them into the system is not completed .... |
|
1290 | 45 |
|
46 |
[author:] |
|
47 |
Claus Gittinger |
|
1309 | 48 |
|
49 |
[see also:] |
|
1382 | 50 |
Text JISEncodedString |
1309 | 51 |
StringCollection |
89 | 52 |
" |
1214 | 53 |
! ! |
54 |
||
55 |
!TwoByteString class methodsFor:'initialization'! |
|
996 | 56 |
|
57 |
initialize |
|
1253 | 58 |
"initialize the class - private" |
59 |
||
996 | 60 |
self flags:(Behavior flagWords) |
61 |
||
62 |
" |
|
63 |
TwoByteString initialize |
|
64 |
" |
|
1214 | 65 |
|
1253 | 66 |
"Modified: 22.4.1996 / 16:14:14 / cg" |
89 | 67 |
! ! |
1 | 68 |
|
69 |
!TwoByteString class methodsFor:'instance creation'! |
|
70 |
||
71 |
basicNew:anInteger |
|
72 |
"return a new empty string with anInteger characters" |
|
73 |
||
1024 | 74 |
^ (super basicNew:anInteger) atAllPut:(Character space) |
75 |
||
76 |
"Modified: 26.2.1996 / 14:38:47 / cg" |
|
17621 | 77 |
! |
78 |
||
79 |
uninitializedNew:anInteger |
|
80 |
"return a new empty string with anInteger characters" |
|
81 |
||
82 |
^ super basicNew:anInteger |
|
83 |
||
84 |
" |
|
85 |
self uninitializedNew:10 |
|
86 |
" |
|
1 | 87 |
! ! |
88 |
||
89 |
!TwoByteString methodsFor:'accessing'! |
|
90 |
||
91 |
basicAt:index |
|
92 |
"return the character at position index, an Integer |
|
73 | 93 |
- reimplemented here since we return 16-bit characters" |
1 | 94 |
|
1024 | 95 |
|val| |
63 | 96 |
|
1024 | 97 |
val := super basicAt:index. |
63 | 98 |
^ Character value:val |
1024 | 99 |
|
100 |
"Modified: 26.2.1996 / 17:02:16 / cg" |
|
1 | 101 |
! |
102 |
||
103 |
basicAt:index put:aCharacter |
|
1230 | 104 |
"store the argument, aCharacter at position index, an Integer. |
105 |
Returns aCharacter (sigh). |
|
73 | 106 |
- reimplemented here since we store 16-bit characters" |
1 | 107 |
|
8094
d05f69bd0097
Use #codePoint instead of deprecated #asciiValue
Stefan Vogel <sv@exept.de>
parents:
5761
diff
changeset
|
108 |
super basicAt:index put:aCharacter codePoint. |
63 | 109 |
^ aCharacter |
608 | 110 |
|
1230 | 111 |
"Modified: 19.4.1996 / 11:16:22 / cg" |
14123 | 112 |
! |
113 |
||
114 |
unsignedShortAt:index |
|
115 |
"return the short at position index, an Integer" |
|
116 |
||
117 |
^ super basicAt:index. |
|
1014 | 118 |
! ! |
119 |
||
16750 | 120 |
!TwoByteString methodsFor:'filling and replacing'! |
121 |
||
122 |
replaceFrom:start to:stop with:aString startingAt:repStart |
|
123 |
"replace the characters starting at index start, anInteger and ending |
|
124 |
at stop, anInteger with characters from aString starting at repStart. |
|
125 |
Return the receiver. |
|
126 |
||
127 |
- reimplemented here for speed" |
|
128 |
||
129 |
%{ /* NOCONTEXT */ |
|
130 |
||
131 |
#ifndef NO_PRIM_STRING |
|
132 |
if (__bothSmallInteger(start, stop)) { |
|
18576 | 133 |
int len; |
134 |
int index1 = __intVal(start); |
|
135 |
int index2 = __intVal(stop); |
|
136 |
int count = index2 - index1 + 1; |
|
16750 | 137 |
|
138 |
if (count <= 0) { |
|
139 |
RETURN (self); |
|
140 |
} |
|
141 |
len = __twoByteStringSize(self); |
|
142 |
if ((index2 <= len) && (index1 > 0)) { |
|
18576 | 143 |
int repIndex = __intVal(repStart); |
16750 | 144 |
|
145 |
if (__isStringLike(aString)) { |
|
18576 | 146 |
int repLen = __stringSize(aString); |
16750 | 147 |
if ((repIndex > 0) && ((repIndex + count - 1) <= repLen)) { |
18576 | 148 |
REGISTER unsigned char *srcp = __stringVal(aString) + repIndex - 1; |
149 |
REGISTER unsigned short *dstp = __twoByteStringVal(self) + index1 - 1; |
|
16750 | 150 |
|
151 |
while (count-- > 0) { |
|
152 |
*dstp++ = *srcp++; |
|
153 |
} |
|
154 |
RETURN (self); |
|
155 |
} |
|
156 |
} else if (__isTwoByteString(aString) || __isUnicode16String(aString)) { |
|
18576 | 157 |
int repLen = __twoByteStringSize(aString); |
16750 | 158 |
if ((repIndex > 0) && ((repIndex + count - 1) <= repLen)) { |
18576 | 159 |
REGISTER unsigned short *srcp = __twoByteStringVal(aString) + repIndex - 1; |
160 |
REGISTER unsigned short *dstp = __twoByteStringVal(self) + index1 - 1; |
|
16750 | 161 |
|
162 |
if (aString == self) { |
|
163 |
/* take care of overlapping copy */ |
|
18576 | 164 |
memmove(dstp, srcp, count*sizeof(short)); |
165 |
RETURN (self); |
|
16750 | 166 |
} |
167 |
if (count > 5) { |
|
168 |
memcpy(dstp, srcp, count*sizeof(short)); |
|
169 |
} else { |
|
170 |
while (count-- > 0) { |
|
171 |
*dstp++ = *srcp++; |
|
172 |
} |
|
173 |
} |
|
174 |
RETURN (self); |
|
175 |
} |
|
176 |
} |
|
177 |
} |
|
178 |
} |
|
179 |
#endif |
|
180 |
%}. |
|
181 |
"/ arrive here if any index arg is out o range, or the source is neither a string, |
|
182 |
"/ nor a two-byte string. |
|
183 |
^ super replaceFrom:start to:stop with:aString startingAt:repStart |
|
184 |
||
185 |
" |
|
186 |
'hello world' asUnicode16String replaceFrom:1 to:5 with:'123456' startingAt:2 |
|
187 |
'hello world' asUnicode16String replaceFrom:1 to:5 with:'123456' asUnicode16String startingAt:2 |
|
188 |
'hello world' asUnicode16String replaceFrom:1 to:0 with:'123456' startingAt:2 |
|
189 |
'hello' asUnicode16String replaceFrom:1 to:6 with:'123456' startingAt:2 |
|
190 |
'hello world' asUnicode16String replaceFrom:1 to:1 with:'123456' startingAt:2 |
|
191 |
" |
|
192 |
! ! |
|
193 |
||
608 | 194 |
!TwoByteString methodsFor:'queries'! |
195 |
||
1017 | 196 |
bitsPerCharacter |
1239 | 197 |
"return the number of bits each character has. |
198 |
Here, 16 is returned (storing double byte characters)." |
|
199 |
||
1017 | 200 |
^ 16 |
1239 | 201 |
|
202 |
"Modified: 20.4.1996 / 23:08:38 / cg" |
|
14557 | 203 |
! |
204 |
||
19758 | 205 |
bytesPerCharacter |
206 |
"return the number of bytes each character has. |
|
207 |
Here, 2 is returned (storing double byte characters)." |
|
208 |
||
209 |
^ 2 |
|
210 |
! |
|
211 |
||
18593 | 212 |
characterSize |
213 |
"answer the size in bits of my largest character (actually only 7, 8 or 16)" |
|
214 |
||
215 |
%{ /* NOCONTEXT */ |
|
216 |
||
217 |
REGISTER unsigned short *sp = __twoByteStringVal(self); |
|
218 |
REGISTER unsigned short *last = sp + __twoByteStringSize(self); |
|
219 |
OBJ cls = __qClass(self); |
|
220 |
int has8BitChars = 0; |
|
221 |
||
222 |
if (cls != Unicode16String && cls != TwoByteString) { |
|
223 |
sp += __OBJS2BYTES__(__intVal(__ClassInstPtr(cls)->c_ninstvars)) / 2; |
|
224 |
} |
|
18586 | 225 |
|
18593 | 226 |
#if __POINTER_SIZE__ == 8 |
227 |
if (sizeof(unsigned INT) == 8) { |
|
228 |
if (!has8BitChars) { |
|
229 |
for ( ; (sp+4) <= last; sp += 4) { |
|
230 |
if (*(unsigned INT *)sp & 0xFF80FF80FF80FF80) { |
|
231 |
/* there are at least 8-bit chars - check for more */ |
|
232 |
has8BitChars = 1; |
|
233 |
break; |
|
234 |
} |
|
235 |
} |
|
236 |
} |
|
237 |
for ( ; (sp+4) <= last; sp += 4) { |
|
238 |
if (*(unsigned INT *)sp & 0xFF00FF00FF00FF00) { |
|
239 |
RETURN(__mkSmallInteger(16)); |
|
240 |
} |
|
241 |
} |
|
242 |
} |
|
243 |
#endif |
|
244 |
if (sizeof(unsigned int) == 4) { |
|
245 |
if (!has8BitChars) { |
|
246 |
for ( ; (sp+2) <= last; sp += 2) { |
|
247 |
if (*(unsigned int *)sp & 0xFF80FF80) { |
|
248 |
/* there are at least 8-bit chars - check for more */ |
|
249 |
has8BitChars = 1; |
|
250 |
break; |
|
251 |
} |
|
252 |
} |
|
253 |
} |
|
254 |
for ( ; (sp+2) <= last; sp += 2) { |
|
255 |
if (*(unsigned int *)sp & 0xFF00FF00) { |
|
256 |
RETURN(__mkSmallInteger(16)); |
|
257 |
} |
|
258 |
} |
|
259 |
} |
|
260 |
if (!has8BitChars) { |
|
261 |
for ( ; sp < last; sp++) { |
|
262 |
if (*sp & 0xFF80) { |
|
263 |
/* there are at least 8-bit chars - check for more */ |
|
264 |
has8BitChars = 1; |
|
265 |
break; |
|
266 |
} |
|
267 |
} |
|
268 |
} |
|
269 |
for ( ; sp < last; sp++) { |
|
270 |
if (*sp & 0xFF00) { |
|
271 |
RETURN(__mkSmallInteger(16)); |
|
272 |
} |
|
273 |
} |
|
274 |
RETURN (__mkSmallInteger(has8BitChars ? 8 : 7)); |
|
275 |
%}. |
|
276 |
||
277 |
" |
|
278 |
'hello world' asUnicode16String characterSize |
|
23547
c69c97cec351
Issue #239: Fix all Smalltak/X source files to be in unicode (UTF8 without BOM) and prefixed by "{ Encoding: utf8 }" when any unicode character is present
Patrik Svestka <patrik.svestka@gmail.com>
parents:
19863
diff
changeset
|
279 |
'hello worldüäö' asUnicode16String characterSize |
18593 | 280 |
'a' asUnicode16String characterSize |
23547
c69c97cec351
Issue #239: Fix all Smalltak/X source files to be in unicode (UTF8 without BOM) and prefixed by "{ Encoding: utf8 }" when any unicode character is present
Patrik Svestka <patrik.svestka@gmail.com>
parents:
19863
diff
changeset
|
281 |
'ü' asUnicode16String characterSize |
18593 | 282 |
'aa' asUnicode16String characterSize |
23547
c69c97cec351
Issue #239: Fix all Smalltak/X source files to be in unicode (UTF8 without BOM) and prefixed by "{ Encoding: utf8 }" when any unicode character is present
Patrik Svestka <patrik.svestka@gmail.com>
parents:
19863
diff
changeset
|
283 |
'aü' asUnicode16String characterSize |
18593 | 284 |
'aaa' asUnicode16String characterSize |
23547
c69c97cec351
Issue #239: Fix all Smalltak/X source files to be in unicode (UTF8 without BOM) and prefixed by "{ Encoding: utf8 }" when any unicode character is present
Patrik Svestka <patrik.svestka@gmail.com>
parents:
19863
diff
changeset
|
285 |
'aaü' asUnicode16String characterSize |
c69c97cec351
Issue #239: Fix all Smalltak/X source files to be in unicode (UTF8 without BOM) and prefixed by "{ Encoding: utf8 }" when any unicode character is present
Patrik Svestka <patrik.svestka@gmail.com>
parents:
19863
diff
changeset
|
286 |
'aaaü' asUnicode16String characterSize |
18593 | 287 |
'aaaa' asUnicode16String characterSize |
23547
c69c97cec351
Issue #239: Fix all Smalltak/X source files to be in unicode (UTF8 without BOM) and prefixed by "{ Encoding: utf8 }" when any unicode character is present
Patrik Svestka <patrik.svestka@gmail.com>
parents:
19863
diff
changeset
|
288 |
'aaaaü' asUnicode16String characterSize |
18593 | 289 |
" |
18586 | 290 |
! |
291 |
||
18593 | 292 |
containsNon7BitAscii |
17621 | 293 |
"return true, if the underlying string contains 8BitCharacters (or widers) |
294 |
(i.e. if it is non-ascii)" |
|
295 |
||
296 |
%{ /* NOCONTEXT */ |
|
297 |
||
18593 | 298 |
REGISTER unsigned short *sp = __twoByteStringVal(self); |
299 |
REGISTER unsigned short *last = sp + __twoByteStringSize(self); |
|
300 |
OBJ cls = __qClass(self); |
|
17621 | 301 |
|
18593 | 302 |
if ( cls != Unicode16String && cls != TwoByteString) { |
17621 | 303 |
sp += __OBJS2BYTES__(__intVal(__ClassInstPtr(cls)->c_ninstvars)) / 2; |
304 |
} |
|
305 |
#if __POINTER_SIZE__ == 8 |
|
18593 | 306 |
if (sizeof(unsigned INT) == 8) { |
307 |
for ( ; (sp+4) <= last; sp += 4) { |
|
308 |
if (*(unsigned INT *)sp & 0xFF80FF80FF80FF80) { |
|
17621 | 309 |
RETURN ( true ); |
310 |
} |
|
311 |
} |
|
312 |
} |
|
313 |
#endif |
|
18593 | 314 |
if (sizeof(unsigned int) == 4) { |
315 |
for ( ; (sp+2) <= last; sp += 2) { |
|
17621 | 316 |
if (*(unsigned int *)sp & 0xFF80FF80) { |
317 |
RETURN ( true ); |
|
318 |
} |
|
319 |
} |
|
320 |
} |
|
18593 | 321 |
for ( ; sp < last; sp++) { |
17621 | 322 |
if (*sp & 0xFF80) { |
323 |
RETURN ( true ); |
|
324 |
} |
|
325 |
} |
|
326 |
RETURN (false); |
|
327 |
%}. |
|
328 |
||
329 |
" |
|
18593 | 330 |
'hello world' asUnicode16String containsNon7BitAscii |
23547
c69c97cec351
Issue #239: Fix all Smalltak/X source files to be in unicode (UTF8 without BOM) and prefixed by "{ Encoding: utf8 }" when any unicode character is present
Patrik Svestka <patrik.svestka@gmail.com>
parents:
19863
diff
changeset
|
331 |
'hello worldüäö' asUnicode16String containsNon7BitAscii |
c69c97cec351
Issue #239: Fix all Smalltak/X source files to be in unicode (UTF8 without BOM) and prefixed by "{ Encoding: utf8 }" when any unicode character is present
Patrik Svestka <patrik.svestka@gmail.com>
parents:
19863
diff
changeset
|
332 |
'ü' asUnicode16String containsNon7BitAscii |
c69c97cec351
Issue #239: Fix all Smalltak/X source files to be in unicode (UTF8 without BOM) and prefixed by "{ Encoding: utf8 }" when any unicode character is present
Patrik Svestka <patrik.svestka@gmail.com>
parents:
19863
diff
changeset
|
333 |
'aü' asUnicode16String containsNon7BitAscii |
c69c97cec351
Issue #239: Fix all Smalltak/X source files to be in unicode (UTF8 without BOM) and prefixed by "{ Encoding: utf8 }" when any unicode character is present
Patrik Svestka <patrik.svestka@gmail.com>
parents:
19863
diff
changeset
|
334 |
'aaü' asUnicode16String containsNon7BitAscii |
c69c97cec351
Issue #239: Fix all Smalltak/X source files to be in unicode (UTF8 without BOM) and prefixed by "{ Encoding: utf8 }" when any unicode character is present
Patrik Svestka <patrik.svestka@gmail.com>
parents:
19863
diff
changeset
|
335 |
'aaaü' asUnicode16String containsNon7BitAscii |
c69c97cec351
Issue #239: Fix all Smalltak/X source files to be in unicode (UTF8 without BOM) and prefixed by "{ Encoding: utf8 }" when any unicode character is present
Patrik Svestka <patrik.svestka@gmail.com>
parents:
19863
diff
changeset
|
336 |
'aaaaü' asUnicode16String containsNon7BitAscii |
18593 | 337 |
'aaaaa' asUnicode16String containsNon7BitAscii |
17621 | 338 |
" |
339 |
! |
|
340 |
||
14557 | 341 |
isWideString |
19581 | 342 |
"true if I require more than one byte per character" |
343 |
||
14557 | 344 |
^ true |
608 | 345 |
! ! |
346 |
||
631 | 347 |
!TwoByteString class methodsFor:'documentation'! |
348 |
||
349 |
version |
|
18576 | 350 |
^ '$Header$' |
631 | 351 |
! ! |
8094
d05f69bd0097
Use #codePoint instead of deprecated #asciiValue
Stefan Vogel <sv@exept.de>
parents:
5761
diff
changeset
|
352 |
|
16750 | 353 |
|
996 | 354 |
TwoByteString initialize! |