|
0
|
1 |
"
|
|
4
|
2 |
COPYRIGHT (c) 1989 by Claus Gittinger
|
|
45
|
3 |
All Rights Reserved
|
|
0
|
4 |
|
|
|
5 |
This software is furnished under a license and may be used
|
|
|
6 |
only in accordance with the terms of that license and with the
|
|
|
7 |
inclusion of the above copyright notice. This software may not
|
|
|
8 |
be provided or otherwise made available to, or used by, any
|
|
|
9 |
other person. No title to or ownership of the software is
|
|
|
10 |
hereby transferred.
|
|
|
11 |
"
|
|
|
12 |
|
|
|
13 |
Object subclass:#Scanner
|
|
76
|
14 |
instanceVariableNames:'source collectedSource
|
|
45
|
15 |
token tokenType tokenPosition tokenValue
|
|
|
16 |
tokenName tokenLineNr tokenRadix
|
|
64
|
17 |
hereChar peekChar
|
|
45
|
18 |
requestor exitBlock
|
|
|
19 |
errorFlag
|
|
|
20 |
ignoreErrors ignoreWarnings
|
|
|
21 |
saveComments currentComments
|
|
60
|
22 |
warnSTXSpecialComment warnUnderscoreInIdentifier
|
|
87
|
23 |
warnOldStyleAssignment
|
|
45
|
24 |
outStream outCol'
|
|
60
|
25 |
classVariableNames:'TypeArray ActionArray
|
|
|
26 |
AllowUnderscoreInIdentifier
|
|
87
|
27 |
Warnings
|
|
|
28 |
WarnSTXSpecials WarnOldStyleAssignment
|
|
60
|
29 |
WarnUnderscoreInIdentifier'
|
|
45
|
30 |
poolDictionaries:''
|
|
|
31 |
category:'System-Compiler'
|
|
0
|
32 |
!
|
|
|
33 |
|
|
|
34 |
Scanner comment:'
|
|
4
|
35 |
COPYRIGHT (c) 1989 by Claus Gittinger
|
|
45
|
36 |
All Rights Reserved
|
|
30
|
37 |
|
|
97
|
38 |
$Header: /cvs/stx/stx/libcomp/Scanner.st,v 1.30 1995-07-03 02:38:54 claus Exp $
|
|
20
|
39 |
'!
|
|
0
|
40 |
|
|
20
|
41 |
!Scanner class methodsFor:'documentation'!
|
|
|
42 |
|
|
|
43 |
copyright
|
|
|
44 |
"
|
|
|
45 |
COPYRIGHT (c) 1989 by Claus Gittinger
|
|
45
|
46 |
All Rights Reserved
|
|
20
|
47 |
|
|
|
48 |
This software is furnished under a license and may be used
|
|
|
49 |
only in accordance with the terms of that license and with the
|
|
|
50 |
inclusion of the above copyright notice. This software may not
|
|
|
51 |
be provided or otherwise made available to, or used by, any
|
|
|
52 |
other person. No title to or ownership of the software is
|
|
|
53 |
hereby transferred.
|
|
|
54 |
"
|
|
|
55 |
!
|
|
|
56 |
|
|
|
57 |
version
|
|
|
58 |
"
|
|
97
|
59 |
$Header: /cvs/stx/stx/libcomp/Scanner.st,v 1.30 1995-07-03 02:38:54 claus Exp $
|
|
20
|
60 |
"
|
|
|
61 |
!
|
|
|
62 |
|
|
|
63 |
documentation
|
|
|
64 |
"
|
|
|
65 |
Scanner reads from a stream and returns individual smalltalk tokens
|
|
75
|
66 |
|
|
|
67 |
TODO: some testers noticed, that ST-80's scanner methods are called
|
|
|
68 |
xLetter, xDigit etc. For code using these (internals), the nextNumber,
|
|
|
69 |
nextIdentifier etc. methods should be renamed.
|
|
20
|
70 |
"
|
|
|
71 |
! !
|
|
0
|
72 |
|
|
|
73 |
!Scanner class methodsFor:'instance creation'!
|
|
|
74 |
|
|
30
|
75 |
for:aStringOrStream
|
|
|
76 |
"return a new scanner reading from aStringOrStream"
|
|
0
|
77 |
|
|
30
|
78 |
^ (super new) initializeFor:aStringOrStream
|
|
0
|
79 |
! !
|
|
|
80 |
|
|
41
|
81 |
!Scanner class methodsFor:'defaults'!
|
|
|
82 |
|
|
80
|
83 |
warnings
|
|
|
84 |
"return true, if any warnings are to be shown"
|
|
|
85 |
|
|
|
86 |
^ Warnings
|
|
|
87 |
!
|
|
|
88 |
|
|
41
|
89 |
warnings:aBoolean
|
|
|
90 |
"this allows turning on/off warnings; the default is on.
|
|
|
91 |
You can turn off warnings in your 'private.rc' file with
|
|
45
|
92 |
Compiler warnings:false
|
|
41
|
93 |
"
|
|
|
94 |
|
|
|
95 |
Warnings := aBoolean
|
|
|
96 |
!
|
|
|
97 |
|
|
80
|
98 |
warnSTXSpecials
|
|
|
99 |
"return true, if ST/X specials are to be warned about"
|
|
|
100 |
|
|
|
101 |
^ WarnSTXSpecials
|
|
|
102 |
!
|
|
|
103 |
|
|
41
|
104 |
warnSTXSpecials:aBoolean
|
|
|
105 |
"this allows turning on/off warnings about stx specials.
|
|
|
106 |
If you get bored by those warnings, turn them off by adding
|
|
|
107 |
a line as:
|
|
45
|
108 |
Compiler warnSTXSpecials:false
|
|
41
|
109 |
in your 'private.rc' file"
|
|
|
110 |
|
|
|
111 |
WarnSTXSpecials := aBoolean
|
|
60
|
112 |
!
|
|
|
113 |
|
|
80
|
114 |
warnUnderscoreInIdentifier
|
|
|
115 |
"return true, if underscores in identifiers are to be warned about"
|
|
|
116 |
|
|
|
117 |
^ WarnUnderscoreInIdentifier
|
|
|
118 |
!
|
|
|
119 |
|
|
60
|
120 |
warnUnderscoreInIdentifier:aBoolean
|
|
|
121 |
"this allows turning on/off warnings about underscores in identifiers.
|
|
|
122 |
If you get bored by those warnings, turn them off by adding
|
|
|
123 |
a line as:
|
|
|
124 |
Compiler warnUnderscoreInIdentifier:false
|
|
|
125 |
in your 'private.rc' file"
|
|
|
126 |
|
|
|
127 |
WarnUnderscoreInIdentifier := aBoolean
|
|
|
128 |
!
|
|
|
129 |
|
|
87
|
130 |
warnOldStyleAssignment
|
|
|
131 |
"return true, if underscore-assignment (pre ST-80v4 syntax) are to be warned about"
|
|
|
132 |
|
|
|
133 |
^ WarnOldStyleAssignment
|
|
|
134 |
!
|
|
|
135 |
|
|
|
136 |
warnOldStyleAssignment:aBoolean
|
|
|
137 |
"this allows turning on/off warnings about underscore-assignment (pre ST-80v4 syntax).
|
|
|
138 |
If you get bored by those warnings, turn them off by adding
|
|
|
139 |
a line as:
|
|
|
140 |
Compiler warnOldStyleAssignment:false
|
|
|
141 |
in your 'private.rc' file"
|
|
|
142 |
|
|
|
143 |
WarnOldStyleAssignment := aBoolean
|
|
|
144 |
!
|
|
|
145 |
|
|
80
|
146 |
allowUnderscoreInIdentifier
|
|
|
147 |
"return true, if underscores are allowed in identifiers"
|
|
|
148 |
|
|
|
149 |
^ AllowUnderscoreInIdentifier
|
|
|
150 |
!
|
|
|
151 |
|
|
60
|
152 |
allowUnderscoreInIdentifier:aBoolean
|
|
|
153 |
"this allows turning on/off underscores in identifiers.
|
|
|
154 |
If turned off (the default), underscores are not allowed in identifiers,
|
|
|
155 |
but instead scanned as assignment character (old ST/80 syntax).
|
|
|
156 |
If turned on, underscores are in identifiers are allowed, while extra
|
|
|
157 |
underscores are still scanned as assignment.
|
|
|
158 |
If you have to fileIn VV-Vsn2 classes,
|
|
|
159 |
add a line such as:
|
|
|
160 |
Compiler allowUnderscoreInIdentifiers:false
|
|
|
161 |
in your 'private.rc'/'smalltalk.rc' file"
|
|
|
162 |
|
|
71
|
163 |
AllowUnderscoreInIdentifier := aBoolean.
|
|
|
164 |
self setupActions
|
|
41
|
165 |
! !
|
|
|
166 |
|
|
33
|
167 |
!Scanner class methodsFor:'initialization'!
|
|
|
168 |
|
|
60
|
169 |
initialize
|
|
|
170 |
Warnings := true.
|
|
|
171 |
WarnSTXSpecials := true.
|
|
|
172 |
WarnUnderscoreInIdentifier := true.
|
|
87
|
173 |
WarnOldStyleAssignment := true.
|
|
60
|
174 |
AllowUnderscoreInIdentifier := false.
|
|
|
175 |
!
|
|
|
176 |
|
|
33
|
177 |
setupActions
|
|
|
178 |
|block|
|
|
|
179 |
|
|
|
180 |
ActionArray := Array new:256.
|
|
|
181 |
TypeArray := Array new:256.
|
|
|
182 |
|
|
|
183 |
block := [:s :char | s nextNumber].
|
|
|
184 |
($0 asciiValue) to:($9 asciiValue) do:[:index |
|
|
45
|
185 |
ActionArray at:index put:block
|
|
33
|
186 |
].
|
|
|
187 |
|
|
87
|
188 |
block := [:s :char | s nextSpecial].
|
|
89
|
189 |
#( $& $- $+ $= $* $/ $\ $< $> $~ $@ $, $? "new:" $!! ) do:[:binop |
|
|
87
|
190 |
TypeArray at:(binop asciiValue) put:#special.
|
|
|
191 |
ActionArray at:(binop asciiValue) put:block
|
|
|
192 |
].
|
|
|
193 |
|
|
|
194 |
block := [:s :char | s nextToken:char].
|
|
89
|
195 |
#( $; $. $( $) $[ $] "$!!" $^ $| $_ ) do:[:ch |
|
|
87
|
196 |
ActionArray at:(ch asciiValue) put:block
|
|
|
197 |
].
|
|
|
198 |
|
|
33
|
199 |
block := [:s :char | s nextIdentifier].
|
|
|
200 |
($a asciiValue) to:($z asciiValue) do:[:index |
|
|
45
|
201 |
ActionArray at:index put:block
|
|
33
|
202 |
].
|
|
|
203 |
($A asciiValue) to:($Z asciiValue) do:[:index |
|
|
45
|
204 |
ActionArray at:index put:block
|
|
33
|
205 |
].
|
|
71
|
206 |
AllowUnderscoreInIdentifier ifTrue:[
|
|
|
207 |
ActionArray at:$_ asciiValue put:block
|
|
|
208 |
].
|
|
33
|
209 |
|
|
|
210 |
"kludge: action is characterToken, but type is special"
|
|
|
211 |
TypeArray at:($| asciiValue) put:#special.
|
|
|
212 |
|
|
80
|
213 |
"kludge: action is nextColonOrAssign, but type is special"
|
|
|
214 |
TypeArray at:($: asciiValue) put:#special.
|
|
|
215 |
|
|
33
|
216 |
ActionArray at:($' asciiValue) put:[:s :char | s nextString].
|
|
|
217 |
ActionArray at:($$ asciiValue) put:[:s :char | s nextCharacter].
|
|
|
218 |
ActionArray at:($# asciiValue) put:[:s :char | s nextHash].
|
|
|
219 |
ActionArray at:($% asciiValue) put:[:s :char | s nextPrimitive].
|
|
|
220 |
ActionArray at:($: asciiValue) put:[:s :char | s nextColonOrAssign]
|
|
|
221 |
|
|
|
222 |
"
|
|
|
223 |
Scanner setupActions
|
|
|
224 |
"
|
|
|
225 |
! !
|
|
|
226 |
|
|
0
|
227 |
!Scanner methodsFor:'private'!
|
|
|
228 |
|
|
30
|
229 |
initializeFor:aStringOrStream
|
|
0
|
230 |
"initialize -
|
|
|
231 |
if this is the first time, setup character- and action tables"
|
|
|
232 |
|
|
49
|
233 |
self initialize.
|
|
|
234 |
|
|
30
|
235 |
aStringOrStream isStream ifFalse:[
|
|
45
|
236 |
source := ReadStream on:aStringOrStream
|
|
30
|
237 |
] ifTrue:[
|
|
45
|
238 |
source := aStringOrStream.
|
|
30
|
239 |
].
|
|
0
|
240 |
!
|
|
|
241 |
|
|
3
|
242 |
initialize
|
|
|
243 |
"prepare a scan"
|
|
|
244 |
|
|
|
245 |
errorFlag := false.
|
|
|
246 |
tokenLineNr := 1.
|
|
|
247 |
currentComments := nil.
|
|
|
248 |
saveComments := false.
|
|
7
|
249 |
ignoreErrors := false.
|
|
41
|
250 |
ignoreWarnings := Warnings not.
|
|
49
|
251 |
warnSTXSpecialComment := WarnSTXSpecials.
|
|
60
|
252 |
warnUnderscoreInIdentifier := WarnUnderscoreInIdentifier.
|
|
87
|
253 |
warnOldStyleAssignment := WarnOldStyleAssignment.
|
|
49
|
254 |
ActionArray isNil ifTrue:[
|
|
|
255 |
self class setupActions
|
|
|
256 |
]
|
|
3
|
257 |
!
|
|
|
258 |
|
|
0
|
259 |
notifying:anObject
|
|
|
260 |
"set the requestor to be notified"
|
|
|
261 |
|
|
|
262 |
requestor := anObject
|
|
|
263 |
!
|
|
|
264 |
|
|
76
|
265 |
collectedSource
|
|
|
266 |
^ collectedSource
|
|
|
267 |
!
|
|
|
268 |
|
|
7
|
269 |
ignoreErrors
|
|
|
270 |
"turn off notification of errors"
|
|
|
271 |
|
|
|
272 |
ignoreErrors := true
|
|
|
273 |
!
|
|
|
274 |
|
|
17
|
275 |
ignoreWarnings
|
|
|
276 |
"turn off notification of warnings"
|
|
|
277 |
|
|
|
278 |
ignoreWarnings := true
|
|
|
279 |
!
|
|
|
280 |
|
|
0
|
281 |
backupPosition
|
|
|
282 |
"if reading from a stream, at the end we might have read
|
|
|
283 |
one token too many"
|
|
|
284 |
|
|
|
285 |
(tokenType == #EOF) ifFalse:[
|
|
45
|
286 |
source position:tokenPosition
|
|
0
|
287 |
]
|
|
20
|
288 |
!
|
|
|
289 |
|
|
|
290 |
beginComment
|
|
|
291 |
^ self
|
|
|
292 |
!
|
|
|
293 |
|
|
|
294 |
endComment:comment
|
|
|
295 |
saveComments ifTrue:[
|
|
45
|
296 |
currentComments isNil ifTrue:[
|
|
|
297 |
currentComments := OrderedCollection with:comment
|
|
|
298 |
] ifFalse:[
|
|
|
299 |
currentComments add:comment
|
|
|
300 |
]
|
|
20
|
301 |
].
|
|
81
|
302 |
! !
|
|
|
303 |
|
|
|
304 |
!Scanner methodsFor:'directives'!
|
|
|
305 |
|
|
|
306 |
parseDirectiveStringArg
|
|
|
307 |
hereChar == $' ifTrue:[
|
|
|
308 |
self nextString.
|
|
|
309 |
tokenType == #String ifTrue:[
|
|
|
310 |
^ tokenValue
|
|
|
311 |
]
|
|
|
312 |
].
|
|
|
313 |
^ nil
|
|
|
314 |
!
|
|
|
315 |
|
|
|
316 |
parseDirectiveStringListArg
|
|
|
317 |
|list|
|
|
|
318 |
|
|
|
319 |
list := OrderedCollection new.
|
|
|
320 |
|
|
|
321 |
[hereChar == $'] whileTrue:[
|
|
|
322 |
list addLast:self parseDirectiveStringArg.
|
|
|
323 |
source skipSeparatorsExceptCR.
|
|
|
324 |
hereChar := source peek.
|
|
|
325 |
(hereChar == $,) ifTrue:[
|
|
|
326 |
source next.
|
|
|
327 |
source skipSeparatorsExceptCR.
|
|
|
328 |
hereChar := source peek.
|
|
|
329 |
].
|
|
|
330 |
].
|
|
|
331 |
^ list
|
|
74
|
332 |
!
|
|
|
333 |
|
|
|
334 |
parseDirective
|
|
81
|
335 |
|directive packageName namespace list|
|
|
74
|
336 |
|
|
|
337 |
source next.
|
|
|
338 |
source skipSeparatorsExceptCR.
|
|
|
339 |
hereChar := source peek.
|
|
|
340 |
hereChar isLetter ifTrue:[
|
|
|
341 |
directive := source nextWord.
|
|
|
342 |
source peek == $: ifTrue:[
|
|
|
343 |
source next.
|
|
|
344 |
source skipSeparatorsExceptCR.
|
|
|
345 |
hereChar := source peek.
|
|
|
346 |
|
|
|
347 |
"
|
|
|
348 |
Package: 'name-of-package'
|
|
|
349 |
"
|
|
|
350 |
directive = 'Package' ifTrue:[
|
|
81
|
351 |
packageName := self parseDirectiveStringArg.
|
|
|
352 |
packageName notNil ifTrue:[
|
|
|
353 |
(requestor respondsTo:#setPackage:) ifTrue:[
|
|
|
354 |
requestor setPackage:packageName
|
|
|
355 |
].
|
|
74
|
356 |
].
|
|
|
357 |
].
|
|
|
358 |
|
|
|
359 |
"
|
|
81
|
360 |
Namespace: 'name-of-package'
|
|
|
361 |
"
|
|
|
362 |
directive = 'Namespace' ifTrue:[
|
|
|
363 |
namespace := self parseDirectiveStringArg.
|
|
|
364 |
namespace notNil ifTrue:[
|
|
|
365 |
(requestor respondsTo:#setNameSpace:) ifTrue:[
|
|
|
366 |
requestor setNameSpace:namespace
|
|
|
367 |
].
|
|
|
368 |
].
|
|
|
369 |
].
|
|
|
370 |
|
|
|
371 |
"
|
|
|
372 |
Uses: 'nameSpace1', ... , 'nameSpace2'
|
|
|
373 |
"
|
|
|
374 |
directive = 'Uses' ifTrue:[
|
|
|
375 |
list := self parseDirectiveStringListArg.
|
|
|
376 |
(requestor respondsTo:#useNameSpaces:) ifTrue:[
|
|
|
377 |
requestor useNameSpaces:list
|
|
|
378 |
].
|
|
|
379 |
].
|
|
|
380 |
"
|
|
74
|
381 |
Prerequisites: 'name-of-package', ... , 'name-of-package'
|
|
|
382 |
"
|
|
|
383 |
directive = 'Prerequisites' ifTrue:[
|
|
81
|
384 |
list := self parseDirectiveStringListArg.
|
|
|
385 |
(requestor respondsTo:#requirePackages:) ifTrue:[
|
|
|
386 |
requestor requirePackages:list
|
|
74
|
387 |
].
|
|
|
388 |
].
|
|
|
389 |
]
|
|
|
390 |
].
|
|
|
391 |
hereChar := source peek.
|
|
0
|
392 |
! !
|
|
|
393 |
|
|
|
394 |
!Scanner methodsFor:'error handling'!
|
|
|
395 |
|
|
|
396 |
showErrorMessage:aMessage position:pos
|
|
|
397 |
"show an errormessage on the Transcript"
|
|
|
398 |
|
|
7
|
399 |
ignoreErrors ifFalse:[
|
|
45
|
400 |
Smalltalk silentLoading == true ifFalse:[
|
|
|
401 |
Transcript showCr:(pos printString , ' ' , aMessage)
|
|
|
402 |
]
|
|
7
|
403 |
]
|
|
0
|
404 |
!
|
|
|
405 |
|
|
|
406 |
notifyError:aMessage position:position to:endPos
|
|
|
407 |
"notify requestor of an error - if there is no requestor
|
|
53
|
408 |
put it on the transcript. Requestor is typically the CodeView
|
|
|
409 |
in which the accept/doIt was triggered, or the PositionableStream
|
|
|
410 |
which does the fileIn. The requestor may decide how to highlight the
|
|
|
411 |
error (and/or to abort the compile).
|
|
0
|
412 |
Return the result passed back by the requestor."
|
|
|
413 |
|
|
53
|
414 |
ignoreErrors ifFalse:[
|
|
|
415 |
requestor isNil ifTrue:[
|
|
|
416 |
self showErrorMessage:aMessage position:position.
|
|
|
417 |
^ false
|
|
|
418 |
].
|
|
83
|
419 |
^ requestor error:aMessage position:position to:endPos from:self
|
|
0
|
420 |
].
|
|
53
|
421 |
^ false
|
|
0
|
422 |
!
|
|
|
423 |
|
|
|
424 |
notifyWarning:aMessage position:position to:endPos
|
|
|
425 |
"notify requestor of an warning - if there is no requestor
|
|
|
426 |
put it on the transcript.
|
|
|
427 |
Return the result passed back by the requestor."
|
|
|
428 |
|
|
53
|
429 |
ignoreWarnings ifFalse:[
|
|
|
430 |
requestor isNil ifTrue:[
|
|
45
|
431 |
self showErrorMessage:aMessage position:position.
|
|
53
|
432 |
^ false
|
|
45
|
433 |
].
|
|
83
|
434 |
^ requestor warning:aMessage position:position to:endPos from:self
|
|
0
|
435 |
].
|
|
53
|
436 |
^ false
|
|
0
|
437 |
!
|
|
|
438 |
|
|
|
439 |
syntaxError:aMessage position:position to:endPos
|
|
|
440 |
"a syntax error happened"
|
|
|
441 |
|
|
91
|
442 |
self notifyError:('Error:' , aMessage) position:position to:endPos.
|
|
0
|
443 |
exitBlock notNil ifTrue:[exitBlock value].
|
|
|
444 |
^ false
|
|
|
445 |
!
|
|
|
446 |
|
|
|
447 |
syntaxError:aMessage position:position
|
|
|
448 |
"a syntax error happened - only start position is known"
|
|
|
449 |
|
|
|
450 |
^ self syntaxError:aMessage position:position to:nil
|
|
|
451 |
!
|
|
|
452 |
|
|
|
453 |
syntaxError:aMessage
|
|
|
454 |
"a syntax error happened - position is not known"
|
|
|
455 |
|
|
|
456 |
^ self syntaxError:aMessage position:tokenPosition
|
|
|
457 |
!
|
|
|
458 |
|
|
|
459 |
warning:aMessage position:position to:endPos
|
|
|
460 |
"a warning"
|
|
|
461 |
|
|
30
|
462 |
^ self notifyWarning:('Warning: ' , aMessage) position:position to:endPos
|
|
0
|
463 |
!
|
|
|
464 |
|
|
|
465 |
warning:aMessage position:position
|
|
|
466 |
"a warning - only start position is known"
|
|
|
467 |
|
|
|
468 |
^ self warning:aMessage position:position to:nil
|
|
|
469 |
!
|
|
|
470 |
|
|
|
471 |
warning:aMessage
|
|
|
472 |
"a warning - position is not known"
|
|
|
473 |
|
|
|
474 |
^ self warning:aMessage position:tokenPosition
|
|
87
|
475 |
!
|
|
|
476 |
|
|
|
477 |
warnOldStyleAssignmentAt:position
|
|
|
478 |
ignoreWarnings ifFalse:[
|
|
|
479 |
warnOldStyleAssignment ifTrue:[
|
|
|
480 |
self
|
|
|
481 |
warning:'old style assignment - please change to use '':='''
|
|
|
482 |
position:position to:position.
|
|
|
483 |
"
|
|
|
484 |
only warn once (per method)
|
|
|
485 |
"
|
|
|
486 |
warnOldStyleAssignment := false
|
|
|
487 |
]
|
|
|
488 |
]
|
|
|
489 |
!
|
|
|
490 |
|
|
|
491 |
warnUnderscoreAt:position
|
|
|
492 |
ignoreWarnings ifFalse:[
|
|
|
493 |
warnUnderscoreInIdentifier ifTrue:[
|
|
|
494 |
self
|
|
|
495 |
warning:'underscores in identifiers/symbols are nonportable'
|
|
|
496 |
position:position to:position.
|
|
|
497 |
"
|
|
|
498 |
only warn once (per method)
|
|
|
499 |
"
|
|
|
500 |
warnUnderscoreInIdentifier := false
|
|
|
501 |
]
|
|
|
502 |
]
|
|
0
|
503 |
! !
|
|
|
504 |
|
|
17
|
505 |
!Scanner methodsFor:'general scanning'!
|
|
|
506 |
|
|
|
507 |
scanPositionsFor:aTokenString inString:aSourceString
|
|
|
508 |
"scan aSourceString for occurrances of aTokenString.
|
|
|
509 |
Return a collection of start positions.
|
|
|
510 |
Added for VW compatibility (to support syntax-highlight)."
|
|
|
511 |
|
|
|
512 |
|searchType searchName searchValue positions t|
|
|
|
513 |
|
|
|
514 |
"
|
|
|
515 |
first, look what kind of token we have to search for
|
|
|
516 |
"
|
|
|
517 |
self initializeFor:(ReadStream on:aTokenString).
|
|
|
518 |
self nextToken.
|
|
|
519 |
searchType := tokenType.
|
|
|
520 |
searchName := tokenName.
|
|
|
521 |
searchValue := tokenValue.
|
|
|
522 |
|
|
|
523 |
"
|
|
|
524 |
start the real work ...
|
|
|
525 |
"
|
|
|
526 |
self initializeFor:(ReadStream on:aSourceString).
|
|
|
527 |
positions := OrderedCollection new.
|
|
|
528 |
|
|
|
529 |
[(t := self nextToken) ~~ #EOF] whileTrue:[
|
|
45
|
530 |
searchType == t ifTrue:[
|
|
|
531 |
(searchName isNil or:[tokenName = searchName]) ifTrue:[
|
|
|
532 |
(searchValue isNil or:[tokenValue = searchValue]) ifTrue:[
|
|
|
533 |
positions add:tokenPosition.
|
|
|
534 |
]
|
|
|
535 |
]
|
|
|
536 |
]
|
|
17
|
537 |
].
|
|
|
538 |
|
|
|
539 |
^ positions
|
|
|
540 |
|
|
|
541 |
"
|
|
30
|
542 |
Scanner new scanPositionsFor:'hello' inString:'foo bar hello baz hello'
|
|
|
543 |
Scanner new scanPositionsFor:'3.14' inString:'foo 3.145 bar hello 3.14 baz hello 3.14'
|
|
|
544 |
Scanner new scanPositionsFor:'16' inString:'foo 16 bar hello 16r10 baz hello 2r10000'
|
|
17
|
545 |
"
|
|
|
546 |
! !
|
|
|
547 |
|
|
0
|
548 |
!Scanner methodsFor:'reading next token'!
|
|
|
549 |
|
|
3
|
550 |
skipComment
|
|
69
|
551 |
|commentStream startPos|
|
|
3
|
552 |
|
|
69
|
553 |
saveComments ifTrue:[
|
|
|
554 |
commentStream := WriteStream on:''.
|
|
|
555 |
self beginComment.
|
|
|
556 |
].
|
|
3
|
557 |
|
|
20
|
558 |
outStream notNil ifTrue:[
|
|
45
|
559 |
outStream nextPut:Character doubleQuote.
|
|
|
560 |
outCol := outCol + 1
|
|
20
|
561 |
].
|
|
|
562 |
|
|
7
|
563 |
startPos := source position.
|
|
3
|
564 |
source next.
|
|
64
|
565 |
hereChar := source peek.
|
|
15
|
566 |
|
|
74
|
567 |
"
|
|
|
568 |
special ST/X addition:
|
|
17
|
569 |
a $/ right after the initial double quote makes it an up-to-end-of-line comment,
|
|
15
|
570 |
which is very useful to comment out parts of filed-in source code.
|
|
17
|
571 |
Since this is non-standard, use it in very rare cases only.
|
|
|
572 |
(maybe the upcoming ansi-standard adds something similar - in this case, I will
|
|
|
573 |
change it without notice)"
|
|
15
|
574 |
|
|
64
|
575 |
hereChar == $/ ifTrue:[
|
|
|
576 |
[hereChar notNil and:[hereChar ~~ Character cr]] whileTrue:[
|
|
45
|
577 |
saveComments ifTrue:[
|
|
69
|
578 |
commentStream nextPut:hereChar
|
|
45
|
579 |
].
|
|
|
580 |
outStream notNil ifTrue:[
|
|
64
|
581 |
outStream nextPut:hereChar.
|
|
45
|
582 |
outCol := outCol + 1
|
|
|
583 |
].
|
|
64
|
584 |
hereChar := source nextPeek.
|
|
45
|
585 |
].
|
|
|
586 |
tokenLineNr := tokenLineNr + 1.
|
|
|
587 |
ignoreWarnings ifFalse:[
|
|
|
588 |
warnSTXSpecialComment ifTrue:[
|
|
|
589 |
self warning:'end-of-line comments are a nonstandard feature of ST/X'
|
|
|
590 |
position:startPos to:(source position).
|
|
|
591 |
"
|
|
|
592 |
only warn once
|
|
|
593 |
"
|
|
|
594 |
warnSTXSpecialComment := false
|
|
|
595 |
]
|
|
|
596 |
].
|
|
|
597 |
outStream notNil ifTrue:[
|
|
|
598 |
outStream cr.
|
|
|
599 |
outCol := 1
|
|
|
600 |
].
|
|
15
|
601 |
] ifFalse:[
|
|
74
|
602 |
hereChar == ${ ifTrue:[
|
|
|
603 |
"
|
|
|
604 |
special ST/X addition:
|
|
|
605 |
a ${ right after the initial double quote starts a directive
|
|
|
606 |
"
|
|
|
607 |
self parseDirective
|
|
|
608 |
].
|
|
|
609 |
|
|
64
|
610 |
[hereChar notNil and:[hereChar ~~ (Character doubleQuote)]] whileTrue:[
|
|
|
611 |
hereChar == (Character cr) ifTrue:[
|
|
45
|
612 |
tokenLineNr := tokenLineNr + 1.
|
|
|
613 |
].
|
|
|
614 |
saveComments ifTrue:[
|
|
69
|
615 |
commentStream nextPut:hereChar
|
|
45
|
616 |
].
|
|
|
617 |
outStream notNil ifTrue:[
|
|
64
|
618 |
outStream nextPut:hereChar.
|
|
45
|
619 |
outCol := outCol + 1
|
|
|
620 |
].
|
|
64
|
621 |
hereChar := source nextPeek
|
|
45
|
622 |
].
|
|
64
|
623 |
hereChar isNil ifTrue:[
|
|
45
|
624 |
self warning:'unclosed comment' position:startPos to:(source position)
|
|
|
625 |
] ifFalse:[
|
|
|
626 |
outStream notNil ifTrue:[
|
|
|
627 |
outStream nextPut:(Character doubleQuote).
|
|
|
628 |
outCol := outCol + 1
|
|
|
629 |
].
|
|
|
630 |
]
|
|
3
|
631 |
].
|
|
15
|
632 |
|
|
20
|
633 |
"skip final dQuote or cr"
|
|
3
|
634 |
source next.
|
|
20
|
635 |
|
|
69
|
636 |
saveComments ifTrue:[
|
|
|
637 |
self endComment:commentStream contents.
|
|
|
638 |
].
|
|
3
|
639 |
!
|
|
|
640 |
|
|
0
|
641 |
nextToken
|
|
|
642 |
"return the next token from the source-stream"
|
|
|
643 |
|
|
41
|
644 |
|skipping actionBlock|
|
|
0
|
645 |
|
|
|
646 |
peekChar notNil ifTrue:[
|
|
64
|
647 |
hereChar := peekChar.
|
|
45
|
648 |
peekChar := nil
|
|
0
|
649 |
] ifFalse:[
|
|
45
|
650 |
skipping := true.
|
|
|
651 |
[skipping] whileTrue:[
|
|
|
652 |
outStream notNil ifTrue:[
|
|
64
|
653 |
[(hereChar := source peek) == Character space] whileTrue:[
|
|
45
|
654 |
source next.
|
|
|
655 |
outStream space.
|
|
|
656 |
outCol := outCol + 1.
|
|
|
657 |
]
|
|
|
658 |
] ifFalse:[
|
|
64
|
659 |
hereChar := source skipSeparatorsExceptCR.
|
|
45
|
660 |
].
|
|
64
|
661 |
hereChar == (Character cr) ifTrue:[
|
|
45
|
662 |
tokenLineNr := tokenLineNr + 1.
|
|
|
663 |
source next.
|
|
|
664 |
outStream notNil ifTrue:[
|
|
|
665 |
outStream cr.
|
|
|
666 |
outCol := 1
|
|
|
667 |
]
|
|
|
668 |
] ifFalse:[
|
|
64
|
669 |
hereChar == (Character doubleQuote) ifTrue:[
|
|
45
|
670 |
"start of a comment"
|
|
3
|
671 |
|
|
45
|
672 |
self skipComment.
|
|
64
|
673 |
hereChar := source peek.
|
|
45
|
674 |
] ifFalse:[
|
|
|
675 |
skipping := false
|
|
|
676 |
]
|
|
|
677 |
]
|
|
|
678 |
].
|
|
64
|
679 |
hereChar isNil ifTrue:[
|
|
45
|
680 |
tokenType := #EOF.
|
|
|
681 |
^ tokenType
|
|
|
682 |
]
|
|
0
|
683 |
].
|
|
|
684 |
tokenPosition := source position.
|
|
|
685 |
|
|
64
|
686 |
actionBlock := ActionArray at:(hereChar asciiValue).
|
|
0
|
687 |
actionBlock notNil ifTrue:[
|
|
64
|
688 |
^ actionBlock value:self value:hereChar
|
|
0
|
689 |
].
|
|
|
690 |
|
|
64
|
691 |
self syntaxError:('invalid character: ''' , hereChar asString , ''' ',
|
|
|
692 |
'(' , hereChar asciiValue printString , ')')
|
|
45
|
693 |
position:tokenPosition to:tokenPosition.
|
|
0
|
694 |
tokenType := #Error.
|
|
|
695 |
^ #Error
|
|
|
696 |
!
|
|
|
697 |
|
|
|
698 |
nextToken:aCharacter
|
|
|
699 |
tokenType := aCharacter.
|
|
|
700 |
source next.
|
|
|
701 |
^ tokenType
|
|
|
702 |
!
|
|
|
703 |
|
|
|
704 |
nextColonOrAssign
|
|
|
705 |
"colon has been read - look for = to make it an assign"
|
|
|
706 |
|
|
|
707 |
(source nextPeek == $=) ifTrue:[
|
|
45
|
708 |
source next.
|
|
|
709 |
tokenType := $_
|
|
0
|
710 |
] ifFalse:[
|
|
45
|
711 |
tokenType := $:
|
|
0
|
712 |
].
|
|
|
713 |
^ tokenType
|
|
|
714 |
!
|
|
|
715 |
|
|
|
716 |
nextSpecial
|
|
|
717 |
"a special character has been read, look for another one.
|
|
|
718 |
also -number is handled here"
|
|
|
719 |
|
|
|
720 |
|firstChar secondChar thirdChar string p|
|
|
|
721 |
|
|
|
722 |
firstChar := source next.
|
|
|
723 |
secondChar := source peek.
|
|
|
724 |
(firstChar == $-) ifTrue:[
|
|
45
|
725 |
secondChar isDigit ifTrue:[
|
|
|
726 |
self nextNumber.
|
|
|
727 |
tokenValue := tokenValue negated.
|
|
|
728 |
^ tokenType
|
|
|
729 |
]
|
|
0
|
730 |
].
|
|
|
731 |
string := firstChar asString.
|
|
20
|
732 |
secondChar notNil ifTrue:[
|
|
45
|
733 |
((TypeArray at:(secondChar asciiValue)) == #special) ifTrue:[
|
|
|
734 |
(secondChar == $-) ifTrue:[
|
|
|
735 |
"special- look if minus belongs to number following"
|
|
|
736 |
p := source position.
|
|
|
737 |
source next.
|
|
|
738 |
thirdChar := source peek.
|
|
|
739 |
source position:p.
|
|
|
740 |
thirdChar isDigit ifTrue:[
|
|
|
741 |
tokenName := string.
|
|
|
742 |
tokenType := #BinaryOperator.
|
|
|
743 |
^ tokenType
|
|
|
744 |
]
|
|
|
745 |
].
|
|
|
746 |
source next.
|
|
|
747 |
string := string copyWith:secondChar
|
|
|
748 |
].
|
|
0
|
749 |
].
|
|
|
750 |
tokenName := string.
|
|
|
751 |
tokenType := #BinaryOperator.
|
|
|
752 |
^ tokenType
|
|
|
753 |
!
|
|
|
754 |
|
|
|
755 |
nextCharacter
|
|
|
756 |
"a $ has been read - return a character token"
|
|
|
757 |
|
|
|
758 |
|nextChar|
|
|
|
759 |
|
|
|
760 |
source next.
|
|
|
761 |
nextChar := source next.
|
|
|
762 |
nextChar notNil ifTrue:[
|
|
45
|
763 |
tokenValue := nextChar.
|
|
|
764 |
tokenType := #Character
|
|
0
|
765 |
] ifFalse:[
|
|
45
|
766 |
tokenType := #EOF
|
|
0
|
767 |
].
|
|
|
768 |
^ tokenType
|
|
|
769 |
!
|
|
|
770 |
|
|
|
771 |
nextMantissa:radix
|
|
|
772 |
|nextChar value factor|
|
|
|
773 |
|
|
|
774 |
value := 0.
|
|
|
775 |
factor := 1.0 / radix.
|
|
|
776 |
nextChar := source peek.
|
|
|
777 |
[(nextChar notNil and:[nextChar isDigitRadix:radix])] whileTrue:[
|
|
45
|
778 |
value := value + (nextChar digitValue * factor).
|
|
|
779 |
factor := factor / radix.
|
|
|
780 |
nextChar := source nextPeek
|
|
0
|
781 |
].
|
|
|
782 |
^ value
|
|
|
783 |
!
|
|
|
784 |
|
|
|
785 |
nextNumber
|
|
30
|
786 |
|nextChar value s|
|
|
0
|
787 |
|
|
30
|
788 |
tokenRadix := 10.
|
|
|
789 |
value := Integer readFrom:source radix:tokenRadix.
|
|
0
|
790 |
nextChar := source peek.
|
|
|
791 |
(nextChar == $r) ifTrue:[
|
|
45
|
792 |
tokenRadix := value.
|
|
|
793 |
source next.
|
|
|
794 |
s := 1.
|
|
|
795 |
source peek == $- ifTrue:[
|
|
|
796 |
source next.
|
|
|
797 |
s := -1
|
|
|
798 |
].
|
|
|
799 |
value := Integer readFrom:source radix:tokenRadix.
|
|
|
800 |
value := value * s.
|
|
|
801 |
nextChar := source peek
|
|
0
|
802 |
].
|
|
|
803 |
(nextChar == $.) ifTrue:[
|
|
45
|
804 |
nextChar := source nextPeek.
|
|
|
805 |
(nextChar notNil and:[nextChar isDigitRadix:tokenRadix]) ifTrue:[
|
|
|
806 |
value := value asFloat + (self nextMantissa:tokenRadix).
|
|
|
807 |
nextChar := source peek
|
|
|
808 |
] ifFalse:[
|
|
|
809 |
nextChar == (Character cr) ifTrue:[
|
|
|
810 |
tokenLineNr := tokenLineNr + 1.
|
|
|
811 |
].
|
|
|
812 |
peekChar := $.
|
|
|
813 |
]
|
|
0
|
814 |
].
|
|
30
|
815 |
((nextChar == $e) or:[nextChar == $E]) ifTrue:[
|
|
45
|
816 |
nextChar := source nextPeek.
|
|
|
817 |
(nextChar notNil and:[(nextChar isDigitRadix:tokenRadix) or:['+-' includes:nextChar]]) ifTrue:[
|
|
|
818 |
s := 1.
|
|
|
819 |
(nextChar == $+) ifTrue:[
|
|
|
820 |
nextChar := source nextPeek
|
|
|
821 |
] ifFalse:[
|
|
|
822 |
(nextChar == $-) ifTrue:[
|
|
|
823 |
nextChar := source nextPeek.
|
|
|
824 |
s := s negated
|
|
|
825 |
]
|
|
|
826 |
].
|
|
|
827 |
value := value asFloat
|
|
|
828 |
* (10.0 raisedToInteger:((Integer readFrom:source radix:tokenRadix) * s))
|
|
|
829 |
]
|
|
0
|
830 |
].
|
|
|
831 |
tokenValue := value.
|
|
|
832 |
(value isMemberOf:Float) ifTrue:[
|
|
45
|
833 |
tokenType := #Float
|
|
0
|
834 |
] ifFalse:[
|
|
45
|
835 |
tokenType := #Integer
|
|
0
|
836 |
].
|
|
|
837 |
^ tokenType
|
|
|
838 |
!
|
|
|
839 |
|
|
|
840 |
nextId
|
|
|
841 |
|nextChar string oldString
|
|
|
842 |
index "{ Class: SmallInteger }"
|
|
|
843 |
max "{ Class: SmallInteger }" |
|
|
|
844 |
|
|
|
845 |
nextChar := source peek.
|
|
96
|
846 |
string := String basicNew:20.
|
|
0
|
847 |
index := 0.
|
|
|
848 |
max := 10.
|
|
|
849 |
[true] whileTrue:[
|
|
45
|
850 |
(nextChar notNil and:[nextChar isLetterOrDigit]) ifFalse:[
|
|
|
851 |
^ string copyTo:index
|
|
|
852 |
].
|
|
|
853 |
(index == max) ifTrue:[
|
|
|
854 |
oldString := string.
|
|
96
|
855 |
string := String basicNew:(max * 2).
|
|
45
|
856 |
string replaceFrom:1 to:max with:oldString.
|
|
|
857 |
max := max * 2
|
|
|
858 |
].
|
|
|
859 |
index := index + 1.
|
|
|
860 |
string at:index put:nextChar.
|
|
|
861 |
nextChar := source nextPeek
|
|
0
|
862 |
]
|
|
|
863 |
!
|
|
|
864 |
|
|
|
865 |
nextIdentifier
|
|
97
|
866 |
|nextChar string firstChar|
|
|
0
|
867 |
|
|
87
|
868 |
hereChar == $_ ifTrue:[
|
|
|
869 |
"/
|
|
|
870 |
"/ no need to check for AllowUnderscoreInIdentifier here;
|
|
|
871 |
"/ could not arrive here if it was off
|
|
|
872 |
"/
|
|
|
873 |
nextChar := source nextPeek.
|
|
|
874 |
(nextChar isAlphaNumeric or:[nextChar == $_]) ifFalse:[
|
|
|
875 |
"oops: a single underscore is an old-style assignement"
|
|
|
876 |
self warnOldStyleAssignmentAt:tokenPosition.
|
|
|
877 |
source next.
|
|
|
878 |
tokenType := $_.
|
|
|
879 |
^ tokenType
|
|
|
880 |
].
|
|
|
881 |
string := '_'.
|
|
|
882 |
self warnUnderscoreAt:tokenPosition.
|
|
|
883 |
[nextChar == '_'] whileTrue:[
|
|
|
884 |
string := string copyWith:$_.
|
|
|
885 |
nextChar := source nextPeek.
|
|
|
886 |
].
|
|
|
887 |
string := string , source nextAlphaNumericWord.
|
|
|
888 |
] ifFalse:[
|
|
|
889 |
string := source nextAlphaNumericWord "self nextId".
|
|
|
890 |
].
|
|
0
|
891 |
nextChar := source peek.
|
|
87
|
892 |
|
|
60
|
893 |
AllowUnderscoreInIdentifier ifTrue:[
|
|
|
894 |
nextChar == $_ ifTrue:[
|
|
87
|
895 |
self warnUnderscoreAt:(source position).
|
|
60
|
896 |
].
|
|
|
897 |
[nextChar == $_] whileTrue:[
|
|
|
898 |
string := string copyWith:nextChar.
|
|
|
899 |
nextChar := source nextPeek.
|
|
|
900 |
(nextChar isAlphaNumeric) ifTrue:[
|
|
|
901 |
string := string , source nextAlphaNumericWord.
|
|
|
902 |
nextChar := source peek.
|
|
|
903 |
]
|
|
|
904 |
].
|
|
|
905 |
].
|
|
|
906 |
|
|
0
|
907 |
(nextChar == $:) ifTrue:[
|
|
45
|
908 |
source next.
|
|
|
909 |
(source peek == $=) ifFalse:[
|
|
|
910 |
tokenName := string copyWith:nextChar.
|
|
|
911 |
tokenType := #Keyword.
|
|
|
912 |
^ self
|
|
|
913 |
].
|
|
|
914 |
peekChar := $_
|
|
0
|
915 |
].
|
|
20
|
916 |
tokenName := string.
|
|
0
|
917 |
firstChar := string at:1.
|
|
|
918 |
(firstChar == $s) ifTrue:[
|
|
45
|
919 |
(string = 'self') ifTrue:[tokenType := #Self. ^self].
|
|
|
920 |
(string = 'super') ifTrue:[tokenType := #Super. ^self]
|
|
0
|
921 |
].
|
|
|
922 |
(firstChar == $n) ifTrue:[
|
|
45
|
923 |
(string = 'nil') ifTrue:[tokenType := #Nil. ^self]
|
|
0
|
924 |
].
|
|
|
925 |
(firstChar == $t) ifTrue:[
|
|
45
|
926 |
(string = 'true') ifTrue:[tokenType := #True. ^self].
|
|
|
927 |
(string = 'thisContext') ifTrue:[tokenType := #ThisContext. ^self]
|
|
0
|
928 |
].
|
|
|
929 |
(firstChar == $f) ifTrue:[
|
|
45
|
930 |
(string = 'false') ifTrue:[tokenType := #False. ^self]
|
|
0
|
931 |
].
|
|
|
932 |
tokenType := #Identifier.
|
|
|
933 |
^ tokenType
|
|
|
934 |
!
|
|
|
935 |
|
|
|
936 |
nextPrimitive
|
|
|
937 |
|nextChar inPrimitive string
|
|
|
938 |
index "{ Class: SmallInteger }"
|
|
|
939 |
len "{ Class: SmallInteger }" |
|
|
|
940 |
|
|
|
941 |
nextChar := source nextPeek.
|
|
|
942 |
string := String new:500.
|
|
|
943 |
len := 500.
|
|
|
944 |
index := 1.
|
|
|
945 |
(nextChar == ${) ifTrue:[
|
|
45
|
946 |
nextChar := source nextPeek.
|
|
|
947 |
inPrimitive := true.
|
|
|
948 |
[inPrimitive] whileTrue:[
|
|
|
949 |
[nextChar == $%] whileFalse:[
|
|
|
950 |
string at:index put:nextChar.
|
|
|
951 |
(index == len) ifTrue:[
|
|
|
952 |
string := string , (String new:len).
|
|
|
953 |
len := len * 2
|
|
|
954 |
].
|
|
|
955 |
index := index + 1.
|
|
|
956 |
nextChar := source next
|
|
|
957 |
].
|
|
|
958 |
(source peek == $}) ifTrue:[
|
|
|
959 |
inPrimitive := false
|
|
|
960 |
] ifFalse:[
|
|
|
961 |
string at:index put:nextChar.
|
|
|
962 |
(index == len) ifTrue:[
|
|
|
963 |
string := string , (String new:len).
|
|
|
964 |
len := len * 2
|
|
|
965 |
].
|
|
|
966 |
index := index + 1.
|
|
|
967 |
nextChar := source next
|
|
|
968 |
]
|
|
|
969 |
].
|
|
|
970 |
source next.
|
|
|
971 |
tokenValue := string copyTo:(index - 1).
|
|
|
972 |
tokenType := #Primitive.
|
|
|
973 |
tokenLineNr := tokenLineNr + (tokenValue occurrencesOf:(Character cr)).
|
|
|
974 |
^ tokenType
|
|
0
|
975 |
].
|
|
|
976 |
|
|
|
977 |
"a % alone is a binary operator"
|
|
|
978 |
tokenName := '%'.
|
|
|
979 |
tokenType := #BinaryOperator.
|
|
|
980 |
^ tokenType.
|
|
|
981 |
"
|
|
|
982 |
self syntaxError:('invalid character: ''' , nextChar asString , '''')
|
|
45
|
983 |
position:tokenPosition to:(tokenPosition + 1).
|
|
0
|
984 |
^ #Error
|
|
|
985 |
"
|
|
|
986 |
!
|
|
|
987 |
|
|
|
988 |
nextHash
|
|
|
989 |
|nextChar string|
|
|
|
990 |
|
|
|
991 |
nextChar := source nextPeek.
|
|
|
992 |
nextChar notNil ifTrue:[
|
|
45
|
993 |
nextChar isLetterOrDigit ifTrue:[
|
|
|
994 |
string := ''.
|
|
|
995 |
[nextChar notNil and:[nextChar isLetterOrDigit]] whileTrue:[
|
|
60
|
996 |
string := string , (source nextAlphaNumericWord "self nextId").
|
|
45
|
997 |
nextChar := source peek.
|
|
60
|
998 |
AllowUnderscoreInIdentifier == true ifTrue:[
|
|
|
999 |
nextChar == $_ ifTrue:[
|
|
87
|
1000 |
self warnUnderscoreAt:source position.
|
|
60
|
1001 |
].
|
|
|
1002 |
[nextChar == $_] whileTrue:[
|
|
|
1003 |
string := string copyWith:nextChar.
|
|
|
1004 |
nextChar := source nextPeek.
|
|
|
1005 |
(nextChar isAlphaNumeric) ifTrue:[
|
|
|
1006 |
string := string , source nextAlphaNumericWord.
|
|
|
1007 |
nextChar := source peek.
|
|
|
1008 |
]
|
|
|
1009 |
].
|
|
|
1010 |
].
|
|
45
|
1011 |
(nextChar == $:) ifFalse:[
|
|
|
1012 |
tokenValue := string asSymbol.
|
|
|
1013 |
tokenType := #Symbol.
|
|
|
1014 |
^ tokenType
|
|
|
1015 |
].
|
|
|
1016 |
string := string copyWith:nextChar.
|
|
|
1017 |
nextChar := source nextPeek
|
|
|
1018 |
].
|
|
|
1019 |
tokenValue := string asSymbol.
|
|
|
1020 |
tokenType := #Symbol.
|
|
|
1021 |
^ tokenType
|
|
|
1022 |
].
|
|
|
1023 |
(nextChar == $( ) ifTrue:[
|
|
|
1024 |
source next.
|
|
|
1025 |
tokenType := #HashLeftParen.
|
|
|
1026 |
^ tokenType
|
|
|
1027 |
].
|
|
|
1028 |
(nextChar == $[ ) ifTrue:[
|
|
|
1029 |
"it seems that ST-80 supports Constant ByteArrays as #[...]
|
|
|
1030 |
(seen in a PD program)"
|
|
|
1031 |
source next.
|
|
|
1032 |
tokenType := #HashLeftBrack.
|
|
|
1033 |
^ tokenType
|
|
|
1034 |
].
|
|
|
1035 |
(nextChar == $' ) ifTrue:[
|
|
|
1036 |
"it seems that ST-80 supports arbitrary symbols as #'...'
|
|
|
1037 |
(seen in a PD program)"
|
|
|
1038 |
self nextString.
|
|
|
1039 |
tokenValue := tokenValue asSymbol.
|
|
|
1040 |
tokenType := #Symbol.
|
|
|
1041 |
^ tokenType
|
|
|
1042 |
].
|
|
|
1043 |
((TypeArray at:(nextChar asciiValue)) == #special) ifTrue:[
|
|
|
1044 |
string := source next asString.
|
|
|
1045 |
nextChar := source peek.
|
|
|
1046 |
nextChar notNil ifTrue:[
|
|
|
1047 |
((TypeArray at:(nextChar asciiValue)) == #special) ifTrue:[
|
|
|
1048 |
source next.
|
|
|
1049 |
string := string copyWith:nextChar
|
|
|
1050 |
]
|
|
|
1051 |
].
|
|
|
1052 |
tokenValue := string asSymbol.
|
|
|
1053 |
tokenType := #Symbol.
|
|
|
1054 |
^ tokenType
|
|
|
1055 |
]
|
|
0
|
1056 |
].
|
|
3
|
1057 |
"this allows hash to be used as binop -
|
|
|
1058 |
I dont know, if this is correct ..."
|
|
|
1059 |
|
|
|
1060 |
tokenName := '#'.
|
|
45
|
1061 |
tokenType := #BinaryOperator.
|
|
3
|
1062 |
^ tokenType
|
|
|
1063 |
"
|
|
0
|
1064 |
self syntaxError:'unexpected end-of-input in Symbol'
|
|
45
|
1065 |
position:tokenPosition to:(tokenPosition + 1).
|
|
0
|
1066 |
^ #Error
|
|
3
|
1067 |
"
|
|
0
|
1068 |
!
|
|
|
1069 |
|
|
|
1070 |
nextString
|
|
|
1071 |
|nextChar string pos
|
|
|
1072 |
index "{ Class: SmallInteger }"
|
|
|
1073 |
len "{ Class: SmallInteger }"
|
|
|
1074 |
inString|
|
|
|
1075 |
|
|
96
|
1076 |
string := String basicNew:20.
|
|
0
|
1077 |
len := 20.
|
|
|
1078 |
index := 1.
|
|
|
1079 |
pos := source position.
|
|
|
1080 |
source next.
|
|
|
1081 |
nextChar := source next.
|
|
|
1082 |
inString := true.
|
|
|
1083 |
|
|
|
1084 |
[inString] whileTrue:[
|
|
45
|
1085 |
nextChar isNil ifTrue:[
|
|
|
1086 |
self syntaxError:'unexpected end-of-input in String'
|
|
|
1087 |
position:pos to:(source position - 1).
|
|
|
1088 |
tokenType := #EOF.
|
|
|
1089 |
^ tokenType
|
|
|
1090 |
].
|
|
|
1091 |
(nextChar == Character cr) ifTrue:[
|
|
|
1092 |
tokenLineNr := tokenLineNr + 1
|
|
|
1093 |
].
|
|
|
1094 |
(nextChar == Character quote) ifTrue:[
|
|
|
1095 |
(source peek == Character quote) ifTrue:[
|
|
|
1096 |
source next
|
|
|
1097 |
] ifFalse:[
|
|
|
1098 |
inString := false
|
|
|
1099 |
]
|
|
|
1100 |
].
|
|
|
1101 |
inString ifTrue:[
|
|
|
1102 |
string at:index put:nextChar.
|
|
|
1103 |
(index == len) ifTrue:[
|
|
|
1104 |
string := string , (String new:len).
|
|
|
1105 |
len := len * 2
|
|
|
1106 |
].
|
|
|
1107 |
index := index + 1.
|
|
|
1108 |
nextChar := source next
|
|
|
1109 |
]
|
|
0
|
1110 |
].
|
|
15
|
1111 |
tokenValue := string copyTo:(index - 1).
|
|
0
|
1112 |
tokenType := #String.
|
|
|
1113 |
^ tokenType
|
|
|
1114 |
! !
|