|
1 " |
|
2 COPYRIGHT (c) 2003 by Claus Gittinger |
|
3 All Rights Reserved |
|
4 |
|
5 This software is furnished under a license and may be used |
|
6 only in accordance with the terms of that license and with the |
|
7 inclusion of the above copyright notice. This software may not |
|
8 be provided or otherwise made available to, or used by, any |
|
9 other person. No title to or ownership of the software is |
|
10 hereby transferred. |
|
11 " |
|
12 "{ Package: 'jv:dart/compiler' }" |
|
13 |
|
14 "{ NameSpace: Dart }" |
|
15 |
|
16 ScannerBase subclass:#Scanner |
|
17 instanceVariableNames:'allowDegeneratedMantissa keywordTable' |
|
18 classVariableNames:'Verbose' |
|
19 poolDictionaries:'' |
|
20 category:'Languages-Dart-Parser' |
|
21 ! |
|
22 |
|
23 Object subclass:#Token |
|
24 instanceVariableNames:'type value startPosition endPosition' |
|
25 classVariableNames:'' |
|
26 poolDictionaries:'' |
|
27 privateIn:Scanner |
|
28 ! |
|
29 |
|
30 !Scanner class methodsFor:'documentation'! |
|
31 |
|
32 copyright |
|
33 " |
|
34 COPYRIGHT (c) 2003 by Claus Gittinger |
|
35 All Rights Reserved |
|
36 |
|
37 This software is furnished under a license and may be used |
|
38 only in accordance with the terms of that license and with the |
|
39 inclusion of the above copyright notice. This software may not |
|
40 be provided or otherwise made available to, or used by, any |
|
41 other person. No title to or ownership of the software is |
|
42 hereby transferred. |
|
43 " |
|
44 ! |
|
45 |
|
46 examples |
|
47 " |
|
48 |s in| |
|
49 |
|
50 in := ' |
|
51 function scalefactor(value) { |
|
52 scalevector[0]=value; |
|
53 scalevector[1]=1.; |
|
54 scalevector[2]=1.; |
|
55 } |
|
56 '. |
|
57 |
|
58 s := JavaScanner for:in readStream. |
|
59 s nextToken |
|
60 |
|
61 |
|
62 |s in| |
|
63 |
|
64 in := ' |
|
65 function scalefactor(value) { |
|
66 scalevector[0]=value; |
|
67 scalevector[1]=1.; |
|
68 scalevector[2]=1.; |
|
69 } |
|
70 '. |
|
71 s := JavaScanner new. |
|
72 s scanTokens:(in readStream). |
|
73 |
|
74 |
|
75 |s in| |
|
76 |
|
77 in := ' |
|
78 function scalefactor(value) { |
|
79 scalevector[0]=value; |
|
80 scalevector[1]=1.; |
|
81 scalevector[2]=1.; |
|
82 } |
|
83 '. |
|
84 in := in readStream. |
|
85 s := JavaScanner for:in. |
|
86 [in atEnd] whileFalse:[ |
|
87 Transcript showCR:s nextToken |
|
88 ] |
|
89 |
|
90 " |
|
91 |
|
92 "Created: / 13.5.1998 / 14:54:06 / cg" |
|
93 ! ! |
|
94 |
|
95 !Scanner class methodsFor:'initialization'! |
|
96 |
|
97 setupActions |
|
98 "initialize the scanners actionTables - these are used to dispatch |
|
99 into scanner methods as characters are read" |
|
100 |
|
101 |block| |
|
102 |
|
103 self setupKeywordTable. |
|
104 |
|
105 ActionArray := Array new:256. |
|
106 TypeArray := Array new:256. |
|
107 |
|
108 block := [:s :char | s nextNumber]. |
|
109 ($0 asciiValue) to:($9 asciiValue) do:[:index | |
|
110 ActionArray at:index put:block |
|
111 ]. |
|
112 |
|
113 block := [:s :char | s nextSingleCharacterToken:char]. |
|
114 #( $: $; $, ${ $} $( $) $[ $] $_ $? $@) do:[:ch | |
|
115 ActionArray at:(ch asciiValue) put:block |
|
116 ]. |
|
117 |
|
118 block := [:s :char | s nextIdentifier]. |
|
119 ($a asciiValue) to:($z asciiValue) do:[:index | |
|
120 ActionArray at:index put:block |
|
121 ]. |
|
122 ($A asciiValue) to:($Z asciiValue) do:[:index | |
|
123 ActionArray at:index put:block |
|
124 ]. |
|
125 ActionArray at:$_ asciiValue put:block. |
|
126 |
|
127 ActionArray at:$$ asciiValue put:block. |
|
128 |
|
129 ActionArray at:($. asciiValue) put:[:s :char | s nextDotOrFloatOrEllipsis]. |
|
130 |
|
131 ActionArray at:($' asciiValue) put:[:s :char | s nextString:$' character:true]. |
|
132 ActionArray at:($" asciiValue) put:[:s :char | s nextString:$" character:false]. |
|
133 ActionArray at:($!! asciiValue) put:[:s :char | s nextMulti:#(($= #'!!=')) after:char]. |
|
134 ActionArray at:($= asciiValue) put:[:s :char | s nextMulti:#(($= #'==')) after:char]. |
|
135 ActionArray at:($< asciiValue) put:[:s :char | s nextMulti:#(($= #'<=') ($< #'<<')) after:char]. |
|
136 ActionArray at:($> asciiValue) put:[:s :char | s nextMulti:#(($= #'>=') ($> #'>>' $> #'>>>' $= #'>>>=')) after:char]. |
|
137 |
|
138 ActionArray at:($- asciiValue) put:[:s :char | s nextMulti:#(($- #'--') ($= #'-=')) after:char]. |
|
139 ActionArray at:($+ asciiValue) put:[:s :char | s nextMulti:#(($+ #'++') ($= #'+=')) after:char]. |
|
140 ActionArray at:($* asciiValue) put:[:s :char | s nextMulti:#(($= #'*=')) after:char]. |
|
141 ActionArray at:($/ asciiValue) put:[:s :char | s nextMulti:#(($= #'/=') ($/ nil #skipEOLComment) ($* nil #skipComment)) after:char]. |
|
142 ActionArray at:($% asciiValue) put:[:s :char | s nextMulti:#(($= #'%=')) after:char]. |
|
143 ActionArray at:($& asciiValue) put:[:s :char | s nextMulti:#(($= #'&=') ($& #'&&')) after:char]. |
|
144 ActionArray at:($^ asciiValue) put:[:s :char | s nextMulti:#(($= #'^=')) after:char]. |
|
145 ActionArray at:($~ asciiValue) put:[:s :char | s nextMulti:#(($= #'~=')) after:char]. |
|
146 ActionArray at:($| asciiValue) put:[:s :char | s nextMulti:#(($= #'|=') ($| #'||')) after:char]. |
|
147 |
|
148 " |
|
149 self setupActions |
|
150 " |
|
151 |
|
152 "Created: / 14-05-1998 / 15:48:03 / cg" |
|
153 "Modified: / 17-05-1998 / 21:03:37 / cg" |
|
154 "Modified: / 16-03-2012 / 23:49:40 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
|
155 ! |
|
156 |
|
157 setupKeywordTable |
|
158 "initialize the scanners actionTables - these are used to dispatch |
|
159 into scanner methods as characters are read" |
|
160 |
|
161 KeywordTable := Dictionary new. |
|
162 |
|
163 #( |
|
164 'abstract' abstract |
|
165 'assert' assert |
|
166 'class' class |
|
167 'extends' extends |
|
168 'factory' factory |
|
169 'get' get |
|
170 'implements' implements |
|
171 'import' import |
|
172 'interface' interface |
|
173 'is' is |
|
174 'library' library |
|
175 'native' native |
|
176 'negate' negate |
|
177 'operator' operator |
|
178 'set' set |
|
179 'source' source |
|
180 'static' static |
|
181 'typedef' typedef |
|
182 'this' this |
|
183 'super' super |
|
184 'null' null |
|
185 'true' #true |
|
186 'false' #false |
|
187 'const' const |
|
188 'new' new |
|
189 'void' void |
|
190 'final' final |
|
191 'var' var |
|
192 'while' while |
|
193 'do' do |
|
194 'for' for |
|
195 'in' in |
|
196 'if' if |
|
197 'else' else |
|
198 'switch' switch |
|
199 'case' case |
|
200 'default' default |
|
201 'try' try |
|
202 'catch' catch |
|
203 'finally' finally |
|
204 'break' break |
|
205 'continue' continue |
|
206 'return' return |
|
207 'throw' throw |
|
208 ) pairWiseDo:[:s :kw | |
|
209 KeywordTable at:s put:kw |
|
210 ]. |
|
211 |
|
212 " |
|
213 NewJavaScanner setupKeywordTable |
|
214 " |
|
215 |
|
216 "Created: / 14-05-1998 / 15:48:03 / cg" |
|
217 "Modified: / 17-05-1998 / 21:03:37 / cg" |
|
218 "Modified: / 10-01-2013 / 10:51:58 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
|
219 ! ! |
|
220 |
|
221 !Scanner methodsFor:'accessing'! |
|
222 |
|
223 token |
|
224 "the previously scanned token" |
|
225 |
|
226 ^Token new |
|
227 type: tokenType; |
|
228 value: tokenValue; |
|
229 startPosition: tokenStartPosition; |
|
230 endPosition: tokenEndPosition; |
|
231 yourself |
|
232 |
|
233 "Created: / 17-03-2012 / 13:32:09 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
|
234 "Modified: / 10-01-2013 / 11:20:19 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
|
235 ! ! |
|
236 |
|
237 !Scanner methodsFor:'converting'! |
|
238 |
|
239 asPetitStream |
|
240 |
|
241 ^self |
|
242 |
|
243 "Created: / 14-03-2012 / 22:51:43 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
|
244 ! ! |
|
245 |
|
246 !Scanner methodsFor:'error handling'! |
|
247 |
|
248 errorMessagePrefix |
|
249 ^ 'Dart Error:' |
|
250 |
|
251 "Modified: / 10-01-2013 / 11:21:14 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
|
252 ! |
|
253 |
|
254 warningMessagePrefix |
|
255 ^ 'Dart Warning:' |
|
256 |
|
257 "Modified: / 10-01-2013 / 11:21:18 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
|
258 ! ! |
|
259 |
|
260 !Scanner methodsFor:'initialization'! |
|
261 |
|
262 initialize |
|
263 "initialize the scanner" |
|
264 |
|
265 super initialize. |
|
266 |
|
267 allowDegeneratedMantissa := true. "/ something like 123. |
|
268 keywordTable := self class keywordTable. |
|
269 ! ! |
|
270 |
|
271 !Scanner methodsFor:'private'! |
|
272 |
|
273 checkForKeyword:string |
|
274 "check if string is a keyword (as opposed to an identifier)." |
|
275 |
|
276 |tok| |
|
277 |
|
278 (tok := keywordTable at:string ifAbsent:nil) notNil ifTrue:[ |
|
279 tokenType := tok. |
|
280 ^ true |
|
281 ]. |
|
282 ^ false |
|
283 ! |
|
284 |
|
285 isCommentCharacter:ch |
|
286 "return true, if ch is the comment-start character" |
|
287 |
|
288 ^ false |
|
289 |
|
290 "Created: / 14.5.1998 / 20:53:33 / cg" |
|
291 ! |
|
292 |
|
293 rememberTokenStartPosition |
|
294 self rememberTokenStartPosition:0 |
|
295 |
|
296 "Created: / 17-03-2012 / 00:19:57 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
|
297 ! |
|
298 |
|
299 rememberTokenStartPosition: offset |
|
300 tokenStartPosition := source position - offset |
|
301 |
|
302 "Created: / 17-03-2012 / 17:39:25 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
|
303 ! ! |
|
304 |
|
305 !Scanner methodsFor:'reading next token'! |
|
306 |
|
307 characterEscape:char |
|
308 |ascii c | |
|
309 |
|
310 char == $" ifTrue:[ |
|
311 ^ $". |
|
312 ]. |
|
313 |
|
314 char == $b ifTrue:[ |
|
315 ^ Character backspace |
|
316 ]. |
|
317 char == $t ifTrue:[ |
|
318 ^ Character tab |
|
319 ]. |
|
320 char == $n ifTrue:[ |
|
321 ^ Character cr |
|
322 ]. |
|
323 char == $r ifTrue:[ |
|
324 ^ Character return |
|
325 ]. |
|
326 char == $f ifTrue:[ |
|
327 ^ Character newPage |
|
328 ]. |
|
329 |
|
330 char == $u ifTrue:[ |
|
331 ascii := 0. |
|
332 c := source peekOrNil. |
|
333 4 timesRepeat:[ |
|
334 (c isDigitRadix:16) ifFalse:[ |
|
335 self syntaxError:'invalid hex character constant' |
|
336 position:source position-2 to:(source position - 1). |
|
337 ^ Character value:ascii |
|
338 ]. |
|
339 ascii := (ascii bitShift:4). |
|
340 ascii := ascii + c digitValue. |
|
341 source next. c := source peekOrNil. |
|
342 ]. |
|
343 ^ Character value:ascii |
|
344 ]. |
|
345 "/ char == $x ifTrue:[ |
|
346 "/ ascii := 0. |
|
347 "/ c := source peekOrNil. |
|
348 "/ 2 timesRepeat:[ |
|
349 "/ (c isDigitRadix:16) ifFalse:[ |
|
350 "/ self syntaxError:'invalid hex character constant' |
|
351 "/ position:source position-2 to:(source position - 1). |
|
352 "/ ^ Character value:ascii |
|
353 "/ ]. |
|
354 "/ ascii := (ascii bitShift:4). |
|
355 "/ ascii := ascii + c digitValue. |
|
356 "/ source next. c := source peekOrNil. |
|
357 "/ ]. |
|
358 "/ ^ Character value:ascii |
|
359 "/ ]. |
|
360 "OctalEscape ::= \ OctalDigit | |
|
361 \ OctalDigit OctalDigit |
|
362 \ ZeroToThree OctalDigit OctalDigit" |
|
363 |
|
364 (char between:$0 and:$3) ifTrue:[ |
|
365 ascii := char digitValue. |
|
366 c := source peekOrNil. |
|
367 (c between: $0 and: $7) ifTrue:[ |
|
368 source next. |
|
369 ascii := (ascii bitShift:3). |
|
370 ascii := ascii + c digitValue. |
|
371 ]. |
|
372 c := source peekOrNil. |
|
373 (c between: $0 and: $7) ifTrue:[ |
|
374 source next. |
|
375 ascii := (ascii bitShift:3). |
|
376 ascii := ascii + c digitValue. |
|
377 ]. |
|
378 ^ Character value:ascii |
|
379 ]. |
|
380 (char between:$4 and: $7) ifTrue:[ |
|
381 ascii := char digitValue. |
|
382 c := source peekOrNil. |
|
383 (c between: $0 and: $7) ifTrue:[ |
|
384 source next. |
|
385 ascii := (ascii bitShift:3). |
|
386 ascii := ascii + c digitValue. |
|
387 ]. |
|
388 ^ Character value:ascii |
|
389 ]. |
|
390 |
|
391 ^ char |
|
392 |
|
393 "Modified: / 16-03-2012 / 10:07:10 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
|
394 ! |
|
395 |
|
396 exponentPart:initialValue |
|
397 |nextChar value s| |
|
398 |
|
399 value := initialValue. |
|
400 nextChar := source peekOrNil. |
|
401 |
|
402 ((nextChar == $e) or:[nextChar == $E]) ifTrue:[ |
|
403 nextChar := source nextPeek. |
|
404 (nextChar notNil and:[(nextChar isDigitRadix:10) or:['+-' includes:nextChar]]) ifTrue:[ |
|
405 s := 1. |
|
406 (nextChar == $+) ifTrue:[ |
|
407 nextChar := source nextPeek |
|
408 ] ifFalse:[ |
|
409 (nextChar == $-) ifTrue:[ |
|
410 nextChar := source nextPeek. |
|
411 s := s negated |
|
412 ] |
|
413 ]. |
|
414 value := value asFloat |
|
415 * (10.0 raisedToInteger:((Integer readFrom:source radix:10) * s)) |
|
416 ] |
|
417 ]. |
|
418 ^ value |
|
419 ! |
|
420 |
|
421 hexponentPart:initialValue |
|
422 |nextChar value s| |
|
423 |
|
424 value := initialValue. |
|
425 nextChar := source peekOrNil. |
|
426 |
|
427 ((nextChar == $p) or:[nextChar == $P]) ifTrue:[ |
|
428 nextChar := source nextPeek. |
|
429 (nextChar notNil and:[(nextChar isDigitRadix:16) or:['+-' includes:nextChar]]) ifTrue:[ |
|
430 s := 1. |
|
431 (nextChar == $+) ifTrue:[ |
|
432 nextChar := source nextPeek |
|
433 ] ifFalse:[ |
|
434 (nextChar == $-) ifTrue:[ |
|
435 nextChar := source nextPeek. |
|
436 s := s negated |
|
437 ] |
|
438 ]. |
|
439 value := value asFloat |
|
440 * (16.0 raisedToInteger:((Integer readFrom:source radix:16) * s)) |
|
441 ] |
|
442 ]. |
|
443 ^ value |
|
444 |
|
445 "Created: / 16-03-2012 / 00:00:09 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
|
446 ! |
|
447 |
|
448 nextDotOrFloat |
|
449 |nextChar| |
|
450 |
|
451 nextChar := source nextPeek. |
|
452 nextChar isDigit ifTrue:[ |
|
453 ^ self nextFractionalPart:0. |
|
454 ]. |
|
455 tokenType := tokenValue := $. . |
|
456 ^ tokenType |
|
457 ! |
|
458 |
|
459 nextDotOrFloatOrEllipsis |
|
460 |nextChar nextChar2| |
|
461 |
|
462 nextChar := source nextPeek. |
|
463 nextChar isDigit ifTrue:[ |
|
464 ^ self nextFractionalPart:0. |
|
465 ]. |
|
466 nextChar == $. ifTrue:[ |
|
467 nextChar2 := source nextPeek. |
|
468 nextChar2 == $. ifTrue:[ |
|
469 source next. |
|
470 tokenType := #Ellipsis. |
|
471 tokenValue := '...'. |
|
472 ^tokenType. |
|
473 ] ifFalse:[ |
|
474 source skip: -1. |
|
475 ]. |
|
476 ]. |
|
477 tokenType := tokenValue := $. . |
|
478 ^ tokenType |
|
479 |
|
480 "Created: / 15-03-2012 / 10:08:13 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
|
481 ! |
|
482 |
|
483 nextFractionalPart:intValue |
|
484 |nextChar value| |
|
485 |
|
486 value := intValue. |
|
487 nextChar := source peekOrNil. |
|
488 |
|
489 (nextChar notNil and:[nextChar isDigitRadix:10]) ifTrue:[ |
|
490 value := value asFloat + (self nextMantissa:10). |
|
491 nextChar := source peekOrNil |
|
492 ] ifFalse:[ |
|
493 allowDegeneratedMantissa == true ifTrue:[ |
|
494 self warning:'degenerated float constant: ' , value printString , '.' . |
|
495 tokenValue := value asFloat. |
|
496 tokenType := #Float. |
|
497 ^ tokenType |
|
498 ]. |
|
499 nextChar := peekChar := $.. |
|
500 ]. |
|
501 |
|
502 ((nextChar == $e) or:[nextChar == $E]) ifTrue:[ |
|
503 value := self exponentPart:value. |
|
504 nextChar := source peekOrNil |
|
505 ] ifFalse:[ |
|
506 ((nextChar == $p) or:[nextChar == $P]) ifTrue:[ |
|
507 value := self hexponentPart:value. |
|
508 nextChar := source peekOrNil |
|
509 ]. |
|
510 ]. |
|
511 |
|
512 tokenValue := value. |
|
513 |
|
514 (nextChar == $d or:[nextChar == $D]) ifTrue:[ |
|
515 source next. |
|
516 tokenType := #Double. |
|
517 ] ifFalse:[ |
|
518 (nextChar == $f or:[nextChar == $F]) ifTrue:[ |
|
519 source next. |
|
520 ]. |
|
521 tokenType := #Float. |
|
522 ]. |
|
523 |
|
524 ^ tokenType |
|
525 |
|
526 "Created: / 14-05-1998 / 20:00:25 / cg" |
|
527 "Modified: / 16-05-1998 / 15:51:46 / cg" |
|
528 "Modified: / 16-03-2012 / 00:00:58 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
|
529 ! |
|
530 |
|
531 nextHexFractionalPart:intValue |
|
532 |nextChar value| |
|
533 |
|
534 value := intValue. |
|
535 nextChar := source peekOrNil. |
|
536 |
|
537 (nextChar notNil and:[nextChar isDigitRadix:16]) ifTrue:[ |
|
538 value := value asFloat + (self nextMantissa:16). |
|
539 nextChar := source peekOrNil |
|
540 ] ifFalse:[ |
|
541 allowDegeneratedMantissa == true ifTrue:[ |
|
542 self warning:'degenerated float constant: ' , value printString , '.' . |
|
543 tokenValue := value asFloat. |
|
544 tokenType := #Float. |
|
545 ^ tokenType |
|
546 ]. |
|
547 nextChar := peekChar := $.. |
|
548 ]. |
|
549 |
|
550 ((nextChar == $e) or:[nextChar == $E]) ifTrue:[ |
|
551 value := self exponentPart:value. |
|
552 nextChar := source peekOrNil |
|
553 ] ifFalse:[ |
|
554 ((nextChar == $p) or:[nextChar == $P]) ifTrue:[ |
|
555 value := self hexponentPart:value. |
|
556 nextChar := source peekOrNil |
|
557 ]. |
|
558 ]. |
|
559 |
|
560 tokenValue := value. |
|
561 |
|
562 (nextChar == $d or:[nextChar == $D]) ifTrue:[ |
|
563 source next. |
|
564 tokenType := #Double. |
|
565 ] ifFalse:[ |
|
566 (nextChar == $f or:[nextChar == $F]) ifTrue:[ |
|
567 source next. |
|
568 ]. |
|
569 tokenType := #Float. |
|
570 ]. |
|
571 |
|
572 ^ tokenType |
|
573 |
|
574 "Created: / 14-05-1998 / 20:00:25 / cg" |
|
575 "Modified: / 16-05-1998 / 15:51:46 / cg" |
|
576 "Created: / 16-03-2012 / 00:16:43 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
|
577 ! |
|
578 |
|
579 nextIdentifier |
|
580 "an alpha character (or underscore) has been read. |
|
581 Return the next identifier." |
|
582 |
|
583 |nextChar string ok pos| |
|
584 |
|
585 hereChar == $_ ifTrue:[ |
|
586 nextChar := source nextPeek. |
|
587 string := '_'. |
|
588 [nextChar == $_] whileTrue:[ |
|
589 string := string copyWith:$_. |
|
590 nextChar := source nextPeek. |
|
591 ]. |
|
592 nextChar isAlphaNumeric ifTrue:[ |
|
593 string := string , source nextAlphaNumericWord. |
|
594 ] |
|
595 ] ifFalse:[ |
|
596 string := source nextAlphaNumericWord "self nextId". |
|
597 ]. |
|
598 nextChar := source peekOrNil. |
|
599 |
|
600 (nextChar == $_ or:[nextChar == $$]) ifTrue:[ |
|
601 pos := source position. |
|
602 ok := true. |
|
603 [ok] whileTrue:[ |
|
604 string := string copyWith:nextChar. |
|
605 nextChar := source nextPeek. |
|
606 nextChar isNil ifTrue:[ |
|
607 ok := false |
|
608 ] ifFalse:[ |
|
609 (nextChar isAlphaNumeric) ifTrue:[ |
|
610 string := string , source nextAlphaNumericWord. |
|
611 nextChar := source peekOrNil. |
|
612 ]. |
|
613 (nextChar == $_ or:[nextChar == $$]) ifFalse:[ |
|
614 ok := false |
|
615 ] |
|
616 ] |
|
617 ]. |
|
618 ]. |
|
619 |
|
620 "/ (nextChar == $: and:[scanColonAsLabel]) ifTrue:[ |
|
621 "/ source next. |
|
622 "/ ch2 := source peekOrNil. |
|
623 "/ "/ colon follows - care for '::' (nameSpace separator) or ':=' (assignment) |
|
624 "/ (ch2 == $=) ifFalse:[ |
|
625 "/ (ch2 == $:) ifFalse:[ |
|
626 "/ tokenEndPosition := source position - 1. |
|
627 "/ token := string copyWith:nextChar. |
|
628 "/ tokenType := #Keyword. |
|
629 "/ ^ tokenType |
|
630 "/ ]. |
|
631 "/ peekChar := $:. |
|
632 "/ peekChar2 := $:. |
|
633 "/ ] ifTrue:[ |
|
634 "/ peekChar := $:. |
|
635 "/ peekChar2 := $=. |
|
636 "/ ] |
|
637 "/ ]. |
|
638 |
|
639 tokenValue := string. |
|
640 (self checkForKeyword:string) ifFalse:[ |
|
641 tokenType := #Identifier. |
|
642 ]. |
|
643 tokenEndPosition := source position - 1. |
|
644 peekChar2 notNil ifTrue:[ |
|
645 tokenEndPosition := tokenEndPosition - 1 |
|
646 ]. |
|
647 ^ tokenType |
|
648 |
|
649 "Modified: / 15-03-2012 / 20:53:44 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
|
650 ! |
|
651 |
|
652 nextMulti:list after:firstChar |
|
653 "a char has been read - peek ahead in list" |
|
654 |
|
655 |pc| |
|
656 |
|
657 peekChar isNil ifTrue:[ |
|
658 source next. |
|
659 ] ifFalse:[ |
|
660 peekChar := nil. |
|
661 ]. |
|
662 pc := source peek. |
|
663 |
|
664 list do:[:spec | |
|
665 |ch tok idx| |
|
666 |
|
667 ch := spec at:1. |
|
668 tok := spec at:2. |
|
669 idx := 3. |
|
670 |
|
671 pc == ch ifTrue:[ |
|
672 peekChar isNil ifTrue:[ |
|
673 source next. |
|
674 ] ifFalse:[ |
|
675 peekChar := nil. |
|
676 ]. |
|
677 |
|
678 spec size > 2 ifTrue:[ |
|
679 ch := spec at:3. |
|
680 source peek == ch ifTrue:[ |
|
681 source next. |
|
682 tok := spec at:4. |
|
683 idx := 5. |
|
684 ] |
|
685 ]. |
|
686 |
|
687 tok isNil ifTrue:[ |
|
688 ^ self perform:(spec at:idx). |
|
689 ]. |
|
690 |
|
691 tokenType := tokenValue := tok. |
|
692 ^ tokenType |
|
693 ] |
|
694 ]. |
|
695 |
|
696 tokenType := tokenValue := firstChar. |
|
697 ^ tokenType |
|
698 |
|
699 "Created: / 14.5.1998 / 19:19:34 / cg" |
|
700 "Modified: / 16.5.1998 / 19:09:59 / cg" |
|
701 ! |
|
702 |
|
703 nextNumber |
|
704 |nextChar value| |
|
705 |
|
706 value := 0. |
|
707 nextChar := source peekOrNil. |
|
708 nextChar == $0 ifTrue:[ |
|
709 source next. |
|
710 nextChar := source peekOrNil. |
|
711 (nextChar == $x or:[nextChar == $X]) ifTrue:[ |
|
712 source next. |
|
713 value := Integer readFrom:source radix:16. |
|
714 |
|
715 tokenValue := value. |
|
716 numberRadix := 16. |
|
717 |
|
718 nextChar := source peekOrNil. |
|
719 |
|
720 source peekOrNil isNil ifTrue:[ |
|
721 tokenType := #Integer. |
|
722 tokenValue := 0. |
|
723 ^tokenType |
|
724 ]. |
|
725 |
|
726 (nextChar == $L or:[nextChar == $l]) ifTrue:[ |
|
727 source next. |
|
728 tokenType := #LongInteger. |
|
729 ]. |
|
730 nextChar == $. ifTrue:[ |
|
731 source next. |
|
732 ^self nextHexFractionalPart: tokenValue. |
|
733 ]. |
|
734 tokenType := #Integer. |
|
735 ^ tokenType. |
|
736 |
|
737 ]. |
|
738 (nextChar notNil and:[nextChar between:$0 and:$7]) ifTrue:[ |
|
739 value := Integer readFrom:source radix:8. |
|
740 tokenValue := value. |
|
741 numberRadix := 8. |
|
742 |
|
743 nextChar := source peekOrNil. |
|
744 (nextChar == $L or:[nextChar == $l]) ifTrue:[ |
|
745 source next. |
|
746 tokenType := #LongInteger. |
|
747 ] ifFalse:[ |
|
748 tokenType := #Integer. |
|
749 ]. |
|
750 ^ tokenType |
|
751 ]. |
|
752 ]. |
|
753 (nextChar == $L or:[nextChar == $l]) ifTrue:[ |
|
754 source next. |
|
755 tokenValue := value. |
|
756 tokenType := #LongInteger. |
|
757 ^ tokenType |
|
758 ]. |
|
759 (nextChar == $D or:[nextChar == $d]) ifTrue:[ |
|
760 source next. |
|
761 tokenValue := 0.0. |
|
762 tokenType := #Double. |
|
763 ^ tokenType |
|
764 ]. |
|
765 |
|
766 |
|
767 |
|
768 numberRadix := 10. |
|
769 nextChar isDigit ifTrue:[ |
|
770 value := Integer readFrom:source radix:10. |
|
771 nextChar := source peekOrNil. |
|
772 |
|
773 (nextChar == $L or:[nextChar == $l]) ifTrue:[ |
|
774 source next. |
|
775 tokenValue := value. |
|
776 tokenType := #LongInteger. |
|
777 ^ tokenType |
|
778 ]. |
|
779 ]. |
|
780 |
|
781 (nextChar == $.) ifTrue:[ |
|
782 nextChar := source nextPeek. |
|
783 (nextChar notNil and:[nextChar isDigitRadix:10]) ifTrue:[ |
|
784 value := value asFloat + (self nextMantissa:10). |
|
785 nextChar := source peekOrNil |
|
786 ] ifFalse:[ |
|
787 allowDegeneratedMantissa == true ifTrue:[ |
|
788 self warning:'degenerated float constant: ' , value printString , '.' . |
|
789 tokenValue := value asFloat. |
|
790 tokenType := #Float. |
|
791 ^ tokenType |
|
792 ]. |
|
793 |
|
794 "/ nextChar == (Character cr) ifTrue:[ |
|
795 "/ lineNr := lineNr + 1. |
|
796 "/ ]. |
|
797 nextChar := peekChar := $.. |
|
798 ] |
|
799 ]. |
|
800 ((nextChar == $e) or:[nextChar == $E]) ifTrue:[ |
|
801 value := self exponentPart:value. |
|
802 nextChar := source peekOrNil |
|
803 ] ifFalse:[ |
|
804 ((nextChar == $p) or:[nextChar == $P]) ifTrue:[ |
|
805 value := self hexponentPart:value. |
|
806 nextChar := source peekOrNil |
|
807 ] |
|
808 ]. |
|
809 |
|
810 nextChar == $- ifTrue:[ |
|
811 self |
|
812 warnPossibleIncompatibility:'add a space before ''-'' for compatibility with other systems' |
|
813 position:(source position) to:source position. |
|
814 ]. |
|
815 |
|
816 (nextChar == $d or:[nextChar == $D]) ifTrue:[ |
|
817 source next. |
|
818 tokenType := #Double. |
|
819 value := value asFloat. |
|
820 ] ifFalse:[ |
|
821 (nextChar == $f or:[nextChar == $F]) ifTrue:[ |
|
822 source next. |
|
823 tokenType := #Float. |
|
824 value := value asFloat. |
|
825 ] ifFalse:[ |
|
826 (value isMemberOf:Float) ifTrue:[ |
|
827 tokenType := #Float. |
|
828 ] ifFalse:[ |
|
829 tokenType := #Integer. |
|
830 ] |
|
831 ] |
|
832 ]. |
|
833 tokenValue := value. |
|
834 ^ tokenType |
|
835 |
|
836 "Created: / 14-05-1998 / 20:00:25 / cg" |
|
837 "Modified: / 16-05-1998 / 15:51:46 / cg" |
|
838 "Modified: / 16-03-2012 / 23:34:28 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
|
839 ! |
|
840 |
|
841 nextString:delimiter character:isCharacter |
|
842 |s pos nextChar inString| |
|
843 |
|
844 s := (String new:20) writeStream. |
|
845 |
|
846 pos := source position. |
|
847 source next. |
|
848 nextChar := source next. |
|
849 inString := true. |
|
850 |
|
851 [inString] whileTrue:[ |
|
852 nextChar isNil ifTrue:[ |
|
853 self syntaxError:'unexpected end-of-input in String' |
|
854 position:pos to:(source position - 1). |
|
855 tokenValue := nil. |
|
856 tokenType := #EOF. |
|
857 ^ tokenType |
|
858 ]. |
|
859 nextChar == $\ ifTrue:[ |
|
860 nextChar := source next. |
|
861 nextChar := self characterEscape:nextChar. |
|
862 ] ifFalse:[ |
|
863 (nextChar == Character cr) ifTrue:[ |
|
864 lineNr := lineNr + 1 |
|
865 ] ifFalse:[ |
|
866 (nextChar == delimiter) ifTrue:[ |
|
867 (source peekOrNil == delimiter) ifTrue:[ |
|
868 source next |
|
869 ] ifFalse:[ |
|
870 inString := false |
|
871 ] |
|
872 ]. |
|
873 ]. |
|
874 ]. |
|
875 inString ifTrue:[ |
|
876 s nextPut:nextChar. |
|
877 nextChar := source next |
|
878 ] |
|
879 ]. |
|
880 |
|
881 tokenValue := s contents. |
|
882 isCharacter ifTrue:[ |
|
883 tokenValue size ~~ 1 ifTrue:[ |
|
884 self syntaxError:'bad (multi-)character constant' |
|
885 position:pos to:(source position - 1). |
|
886 ]. |
|
887 tokenValue := tokenValue at:1. |
|
888 tokenType := #Character. |
|
889 ] ifFalse:[ |
|
890 tokenType := #String. |
|
891 ]. |
|
892 ^ tokenType |
|
893 |
|
894 "Created: / 16.5.1998 / 19:53:05 / cg" |
|
895 "Modified: / 16.5.1998 / 19:57:16 / cg" |
|
896 ! |
|
897 |
|
898 nextToken |
|
899 |t| |
|
900 |
|
901 [ |
|
902 t := super nextToken. |
|
903 tokenEndPosition := source position - 1. |
|
904 t isNil |
|
905 ] whileTrue. |
|
906 Verbose == true ifTrue:[ |
|
907 Transcript |
|
908 show:'JavaScanner nextToken => '; |
|
909 show: t storeString; |
|
910 show: ' | '; |
|
911 showCR: tokenValue. |
|
912 ]. |
|
913 ^ t |
|
914 |
|
915 "Created: / 14-05-1998 / 15:48:04 / cg" |
|
916 "Modified: / 16-05-1998 / 19:12:29 / cg" |
|
917 "Modified: / 17-03-2012 / 17:35:22 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
|
918 ! |
|
919 |
|
920 skipComment |
|
921 |commentStream commentType startPos| |
|
922 |
|
923 startPos := source position. |
|
924 source next. |
|
925 hereChar := source peekOrNil. |
|
926 |
|
927 [ |
|
928 [hereChar notNil and:[hereChar ~~ $*]] whileTrue:[ |
|
929 hereChar == (Character cr) ifTrue:[ |
|
930 lineNr := lineNr + 1. |
|
931 ]. |
|
932 hereChar := source nextPeek |
|
933 ]. |
|
934 ] doUntil:[ |
|
935 hereChar := source nextPeek. |
|
936 hereChar isNil or:[hereChar == $/]. |
|
937 ]. |
|
938 |
|
939 "skip final /" |
|
940 source next. |
|
941 |
|
942 hereChar isNil ifTrue:[ |
|
943 self warning:'unclosed comment' position:startPos to:(source position) |
|
944 ]. |
|
945 |
|
946 "/ saveComments ifTrue:[ |
|
947 "/ self endComment:(commentStream contents) type:commentType. |
|
948 "/ ]. |
|
949 ^ nil. "/ force nextToken again |
|
950 |
|
951 "Modified: / 31.3.1998 / 23:45:26 / cg" |
|
952 ! |
|
953 |
|
954 skipEOLComment |
|
955 hereChar := source peek. |
|
956 [hereChar notNil and:[hereChar ~~ Character cr]] whileTrue:[ |
|
957 hereChar := source nextPeek. |
|
958 ]. |
|
959 lineNr := lineNr + 1. |
|
960 |
|
961 ^ nil. |
|
962 |
|
963 "Created: / 16.5.1998 / 19:11:05 / cg" |
|
964 "Modified: / 16.5.1998 / 19:15:42 / cg" |
|
965 ! ! |
|
966 |
|
967 !Scanner methodsFor:'stream api'! |
|
968 |
|
969 atEnd |
|
970 ^ source atEnd |
|
971 |
|
972 "Created: / 14-03-2012 / 22:53:39 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
|
973 ! |
|
974 |
|
975 next |
|
976 ^ source next |
|
977 |
|
978 "Created: / 14-03-2012 / 22:53:21 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
|
979 ! |
|
980 |
|
981 next: anInteger |
|
982 "Answer up to anInteger elements of my collection. Overridden for efficiency." |
|
983 |
|
984 ^ source nextAvailable: anInteger |
|
985 |
|
986 "Modified: / 14-03-2012 / 22:55:37 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
|
987 ! |
|
988 |
|
989 peek |
|
990 "An improved version of peek, that is slightly faster than the built in version." |
|
991 |
|
992 ^source peek |
|
993 |
|
994 "Modified: / 14-03-2012 / 22:46:04 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
|
995 ! |
|
996 |
|
997 position |
|
998 ^source position |
|
999 |
|
1000 "Created: / 14-03-2012 / 22:52:51 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
|
1001 ! |
|
1002 |
|
1003 position: anInteger |
|
1004 "The receiver does not check for invalid arguments passed to this method, as it is solely used with valid indexes for backtracking." |
|
1005 |
|
1006 "/ anInteger = 16 ifTrue:[self halt]. |
|
1007 |
|
1008 ^source position: anInteger |
|
1009 |
|
1010 "Modified: / 15-03-2012 / 10:59:28 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
|
1011 ! |
|
1012 |
|
1013 skipSeparators |
|
1014 |
|
1015 source skipSeparators |
|
1016 |
|
1017 "Created: / 15-03-2012 / 10:35:28 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
|
1018 ! |
|
1019 |
|
1020 uncheckedPeek |
|
1021 "An unchecked version of peek that throws an error if we try to peek over the end of the stream, even faster than #peek." |
|
1022 |
|
1023 ^ self peek |
|
1024 |
|
1025 "Modified: / 14-03-2012 / 22:46:22 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
|
1026 ! ! |
|
1027 |
|
1028 !Scanner::Token methodsFor:'accessing'! |
|
1029 |
|
1030 endPosition |
|
1031 ^ endPosition |
|
1032 ! |
|
1033 |
|
1034 endPosition:something |
|
1035 endPosition := something. |
|
1036 ! |
|
1037 |
|
1038 startPosition |
|
1039 ^ startPosition |
|
1040 ! |
|
1041 |
|
1042 startPosition:something |
|
1043 startPosition := something. |
|
1044 ! |
|
1045 |
|
1046 type |
|
1047 ^ type |
|
1048 ! |
|
1049 |
|
1050 type:something |
|
1051 type := something. |
|
1052 ! |
|
1053 |
|
1054 value |
|
1055 ^ value |
|
1056 ! |
|
1057 |
|
1058 value:something |
|
1059 value := something. |
|
1060 ! ! |
|
1061 |
|
1062 !Scanner::Token methodsFor:'printing & storing'! |
|
1063 |
|
1064 printOn:aStream |
|
1065 "append a printed representation if the receiver to the argument, aStream" |
|
1066 |
|
1067 super printOn:aStream. |
|
1068 aStream nextPutAll:'type: '. |
|
1069 type printOn:aStream. |
|
1070 aStream nextPutAll:'value: '. |
|
1071 value printOn:aStream. |
|
1072 aStream nextPutAll:'startPosition: '. |
|
1073 startPosition printOn:aStream. |
|
1074 aStream nextPutAll:'endPosition: '. |
|
1075 endPosition printOn:aStream. |
|
1076 ! ! |
|
1077 |
|
1078 !Scanner class methodsFor:'documentation'! |
|
1079 |
|
1080 version_HG |
|
1081 |
|
1082 ^ '$Changeset: <not expanded> $' |
|
1083 ! ! |