|
1 " |
|
2 COPYRIGHT (c) 2003 by Claus Gittinger |
|
3 All Rights Reserved |
|
4 |
|
5 This software is furnished under a license and may be used |
|
6 only in accordance with the terms of that license and with the |
|
7 inclusion of the above copyright notice. This software may not |
|
8 be provided or otherwise made available to, or used by, any |
|
9 other person. No title to or ownership of the software is |
|
10 hereby transferred. |
|
11 " |
|
12 "{ Package: 'jv:dart/compiler' }" |
|
13 |
|
14 "{ NameSpace: Dart }" |
|
15 |
|
16 Object subclass:#ScannerBase |
|
17 instanceVariableNames:'typeArray actionArray source lineNr tokenType tokenStartPosition |
|
18 tokenEndPosition tokenLineNr numberRadix numberScale hereChar |
|
19 peekChar peekChar2 requestor saveComments currentComments |
|
20 tokenValue scaledMantissaValue parserFlags' |
|
21 classVariableNames:'Warnings' |
|
22 poolDictionaries:'' |
|
23 category:'Languages-Dart-Parser' |
|
24 ! |
|
25 |
|
26 ScannerBase class instanceVariableNames:'TypeArray ActionArray KeywordTable' |
|
27 |
|
28 " |
|
29 No other class instance variables are inherited by this class. |
|
30 " |
|
31 ! |
|
32 |
|
33 !ScannerBase class methodsFor:'documentation'! |
|
34 |
|
35 copyright |
|
36 " |
|
37 COPYRIGHT (c) 2003 by Claus Gittinger |
|
38 All Rights Reserved |
|
39 |
|
40 This software is furnished under a license and may be used |
|
41 only in accordance with the terms of that license and with the |
|
42 inclusion of the above copyright notice. This software may not |
|
43 be provided or otherwise made available to, or used by, any |
|
44 other person. No title to or ownership of the software is |
|
45 hereby transferred. |
|
46 " |
|
47 ! ! |
|
48 |
|
49 !ScannerBase class methodsFor:'initialization'! |
|
50 |
|
51 initialize |
|
52 "initialize the classes defaults. Typically, these are changed |
|
53 later in the 'private.rc' file." |
|
54 |
|
55 "/ ScannerError isLoaded ifFalse:[ |
|
56 "/ ScannerError autoload |
|
57 "/ ]. |
|
58 "/ EmptySourceNotification notifierString:'empty source given to evaluate'. |
|
59 |
|
60 Warnings := false. |
|
61 |
|
62 "ActionArray := nil. |
|
63 TypeArray := nil. |
|
64 self initialize |
|
65 " |
|
66 |
|
67 "Modified: / 15-03-2012 / 00:05:35 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
|
68 ! |
|
69 |
|
70 setupActions |
|
71 "initialize the scanners tables - these are used to dispatch |
|
72 into scanner methods as characters are read" |
|
73 |
|
74 self subclassResponsibility |
|
75 |
|
76 "/ |block| |
|
77 "/ |
|
78 "/ ActionArray := Array new:256. |
|
79 "/ TypeArray := Array new:256. |
|
80 "/ |
|
81 "/ block := [:s :char | s nextNumber]. |
|
82 "/ ($0 asciiValue) to:($9 asciiValue) do:[:index | |
|
83 "/ ActionArray at:index put:block |
|
84 "/ ]. |
|
85 "/ |
|
86 "/ block := [:s :char | s nextIdentifier]. |
|
87 "/ ($a asciiValue) to:($z asciiValue) do:[:index | |
|
88 "/ ActionArray at:index put:block |
|
89 "/ ]. |
|
90 "/ ($A asciiValue) to:($Z asciiValue) do:[:index | |
|
91 "/ ActionArray at:index put:block |
|
92 "/ ]. |
|
93 "/ ActionArray at:$_ asciiValue put:block |
|
94 "/ |
|
95 ! ! |
|
96 |
|
97 !ScannerBase class methodsFor:'instance creation'! |
|
98 |
|
99 for:aStringOrStream |
|
100 "create & return a new scanner reading from aStringOrStream" |
|
101 |
|
102 ^ (super new) initializeFor:aStringOrStream |
|
103 |
|
104 "Modified: 23.5.1997 / 12:08:42 / cg" |
|
105 ! |
|
106 |
|
107 new |
|
108 "create & return a new scanner" |
|
109 |
|
110 ^ self basicNew initialize. |
|
111 |
|
112 "Modified: / 23.5.1997 / 12:08:42 / cg" |
|
113 "Created: / 26.5.1999 / 12:02:16 / stefan" |
|
114 ! ! |
|
115 |
|
116 !ScannerBase class methodsFor:'Signal constants'! |
|
117 |
|
118 emptySourceNotificationSignal |
|
119 ^ EmptySourceNotification |
|
120 |
|
121 "Created: / 16.5.1998 / 15:55:14 / cg" |
|
122 ! |
|
123 |
|
124 errorSignal |
|
125 ^ ScannerError |
|
126 |
|
127 "Created: / 16.5.1998 / 15:55:14 / cg" |
|
128 ! |
|
129 |
|
130 scannerErrorSignal |
|
131 ^ ScannerError |
|
132 |
|
133 "Created: / 16.5.1998 / 15:55:14 / cg" |
|
134 ! |
|
135 |
|
136 warningSignal |
|
137 ^ ScannerWarning |
|
138 |
|
139 "Created: / 16.5.1998 / 15:55:14 / cg" |
|
140 ! ! |
|
141 |
|
142 !ScannerBase class methodsFor:'defaults'! |
|
143 |
|
144 warnings |
|
145 "return true, if any warnings are to be shown" |
|
146 |
|
147 ^ Warnings |
|
148 ! |
|
149 |
|
150 warnings:aBoolean |
|
151 "this allows turning on/off all warnings; the default is on. |
|
152 You can turn off warnings in your 'private.rc' file with |
|
153 Compiler warnings:false |
|
154 " |
|
155 |
|
156 Warnings := aBoolean |
|
157 |
|
158 "Modified: 23.5.1997 / 12:03:05 / cg" |
|
159 ! ! |
|
160 |
|
161 !ScannerBase class methodsFor:'private accessing'! |
|
162 |
|
163 actionArray |
|
164 ^ ActionArray |
|
165 ! |
|
166 |
|
167 keywordTable |
|
168 ^ KeywordTable |
|
169 ! |
|
170 |
|
171 typeArray |
|
172 ^ TypeArray |
|
173 ! ! |
|
174 |
|
175 !ScannerBase class methodsFor:'utility scanning'! |
|
176 |
|
177 scanNumberFrom:aStream |
|
178 "utility - helper for Number>>readSmalltalkSyntaxFrom:" |
|
179 |
|
180 ^ self basicNew scanNumberFrom:aStream |
|
181 |
|
182 " |
|
183 |s| |
|
184 |
|
185 s := '12345abcd' readStream. |
|
186 Transcript showCR:(self scanNumberFrom:s). |
|
187 Transcript showCR:(s upToEnd). |
|
188 " |
|
189 " |
|
190 |s| |
|
191 |
|
192 s := '16rffffxabcd' readStream. |
|
193 Transcript showCR:(self scanNumberFrom:s). |
|
194 Transcript showCR:(s upToEnd). |
|
195 " |
|
196 " |
|
197 |s| |
|
198 |
|
199 s := '1.2345abcd' readStream. |
|
200 Transcript showCR:(self scanNumberFrom:s). |
|
201 Transcript showCR:(s upToEnd). |
|
202 " |
|
203 " |
|
204 |s| |
|
205 |
|
206 s := '1.abcd' readStream. |
|
207 Transcript showCR:(self scanNumberFrom:s). |
|
208 Transcript showCR:(s upToEnd). |
|
209 " |
|
210 |
|
211 "Modified: / 18.6.1998 / 23:10:39 / cg" |
|
212 ! ! |
|
213 |
|
214 !ScannerBase methodsFor:'Compatibility - ST80'! |
|
215 |
|
216 endOfLastToken |
|
217 "return the position of the token which was just read. |
|
218 This method was required by some PD program. |
|
219 It is not maintained and may be removed without notice." |
|
220 |
|
221 ^ source position |
|
222 |
|
223 "Modified: 23.5.1997 / 12:14:27 / cg" |
|
224 ! |
|
225 |
|
226 scan:aStringOrStream |
|
227 "initialize the scanner: set the source-stream and |
|
228 preread the first token" |
|
229 |
|
230 self initializeFor:aStringOrStream. |
|
231 self nextToken |
|
232 |
|
233 "Created: / 30.10.1997 / 16:59:39 / cg" |
|
234 ! |
|
235 |
|
236 scanToken |
|
237 "read the next token from my input stream" |
|
238 |
|
239 ^ self nextToken |
|
240 |
|
241 "Created: / 30.10.1997 / 17:00:16 / cg" |
|
242 ! |
|
243 |
|
244 scanTokens:aStringOrStream |
|
245 "return a collection of symbolic tokens from the passed input" |
|
246 |
|
247 |tokens| |
|
248 |
|
249 self initializeFor:aStringOrStream. |
|
250 tokens := OrderedCollection new. |
|
251 self nextToken. |
|
252 [tokenValue notNil] whileTrue:[ |
|
253 tokens add:tokenValue. |
|
254 self nextToken |
|
255 ]. |
|
256 ^ tokens |
|
257 |
|
258 " |
|
259 Scanner new |
|
260 scanTokens:'Boolean subclass:#True |
|
261 instanceVariableNames:'''' |
|
262 classVariableNames:'''' |
|
263 poolDictionaries:'''' |
|
264 category:''Kernel-Objects'' |
|
265 ' |
|
266 " |
|
267 |
|
268 "Modified: 20.6.1997 / 18:22:58 / cg" |
|
269 ! ! |
|
270 |
|
271 !ScannerBase methodsFor:'accessing'! |
|
272 |
|
273 actionArray |
|
274 ^ actionArray |
|
275 ! |
|
276 |
|
277 actionArray:something |
|
278 actionArray := something. |
|
279 ! |
|
280 |
|
281 comments |
|
282 "if saveComments is on: |
|
283 returns the collection of collected comments (so far) |
|
284 clears the internal collection for the next access" |
|
285 |
|
286 |ret| |
|
287 |
|
288 ret := currentComments ? #(). |
|
289 currentComments := nil. |
|
290 ^ ret |
|
291 |
|
292 "Created: 20.4.1996 / 20:07:01 / cg" |
|
293 "Modified: 23.5.1997 / 12:14:45 / cg" |
|
294 ! |
|
295 |
|
296 lineNumber |
|
297 "the current line number (in the stream)" |
|
298 |
|
299 ^ lineNr |
|
300 ! |
|
301 |
|
302 newSourceStream:aStream |
|
303 source := aStream. |
|
304 self nextToken. |
|
305 |
|
306 "Created: / 29.10.1998 / 21:59:33 / cg" |
|
307 ! |
|
308 |
|
309 numberRadix |
|
310 "the radix of the previously scanned number" |
|
311 |
|
312 ^ numberRadix |
|
313 ! |
|
314 |
|
315 parserFlags:something |
|
316 parserFlags := something. |
|
317 ! |
|
318 |
|
319 saveComments:aBoolean |
|
320 "toggle to turn on/off comment remembering" |
|
321 |
|
322 saveComments := aBoolean |
|
323 |
|
324 "Created: 20.4.1996 / 20:03:56 / cg" |
|
325 "Modified: 23.5.1997 / 12:14:49 / cg" |
|
326 ! |
|
327 |
|
328 sourceStream |
|
329 ^ source |
|
330 |
|
331 "Created: 20.4.1996 / 19:59:58 / cg" |
|
332 ! |
|
333 |
|
334 token |
|
335 "the previously scanned token" |
|
336 |
|
337 "/ generated lazily ... |
|
338 self halt. |
|
339 ! |
|
340 |
|
341 tokenEndPosition |
|
342 "the previously scanned tokens last character position" |
|
343 |
|
344 ^ tokenEndPosition |
|
345 ! |
|
346 |
|
347 tokenLineNumber |
|
348 "the previously scanned tokens line number" |
|
349 |
|
350 ^ tokenLineNr |
|
351 ! |
|
352 |
|
353 tokenStartPosition |
|
354 "the previously scanned tokens first character position" |
|
355 |
|
356 ^ tokenStartPosition |
|
357 ! |
|
358 |
|
359 tokenType |
|
360 "the type (symbolic) of the previously scanned token" |
|
361 |
|
362 ^ tokenType |
|
363 ! |
|
364 |
|
365 tokenValue |
|
366 "the value (string or number) of the previously scanned token" |
|
367 |
|
368 ^ tokenValue |
|
369 ! |
|
370 |
|
371 typeArray |
|
372 ^ typeArray |
|
373 ! |
|
374 |
|
375 typeArray:something |
|
376 typeArray := something. |
|
377 ! ! |
|
378 |
|
379 !ScannerBase methodsFor:'error handling'! |
|
380 |
|
381 correctableError:message position:pos1 to:pos2 |
|
382 "report an error which can be corrected by compiler - |
|
383 return non-false, if correction is wanted (there is more than |
|
384 true/false returned here)" |
|
385 |
|
386 |correctIt| |
|
387 |
|
388 requestor isNil ifTrue:[ |
|
389 "/ self showErrorMessage:message position:pos1. |
|
390 correctIt := false |
|
391 ] ifFalse:[ |
|
392 correctIt := requestor correctableError:message position:pos1 to:pos2 from:self |
|
393 ]. |
|
394 ^ correctIt |
|
395 |
|
396 "Created: / 13.5.1998 / 16:45:56 / cg" |
|
397 ! |
|
398 |
|
399 errorMessagePrefix |
|
400 ^ 'Error:' |
|
401 ! |
|
402 |
|
403 ignoreWarnings |
|
404 ^ Warnings == false |
|
405 |
|
406 "Modified: / 14-03-2012 / 22:36:11 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
|
407 ! |
|
408 |
|
409 lastTokenLineNumber |
|
410 "return the line number of the token which was just read." |
|
411 |
|
412 ^ tokenLineNr |
|
413 |
|
414 "Created: 8.11.1996 / 18:46:36 / cg" |
|
415 "Modified: 23.5.1997 / 12:16:12 / cg" |
|
416 ! |
|
417 |
|
418 notifyError:aMessage position:position to:endPos |
|
419 "notify requestor of an error - if there is no requestor |
|
420 put it on the transcript. Requestor is typically the CodeView |
|
421 in which the accept/doIt was triggered, or the PositionableStream |
|
422 which does the fileIn. The requestor may decide how to highlight the |
|
423 error (and/or to abort the compile). |
|
424 Return the result passed back by the requestor." |
|
425 |
|
426 requestor isNil ifTrue:[ |
|
427 self showErrorMessage:aMessage position:position. |
|
428 ^ false |
|
429 ]. |
|
430 ^ requestor error:aMessage position:position to:endPos from:self |
|
431 ! |
|
432 |
|
433 notifyWarning:aMessage position:position to:endPos |
|
434 "notify requestor of an warning - if there is no requestor |
|
435 put it on the transcript. |
|
436 Return the result passed back by the requestor." |
|
437 |
|
438 |warn| |
|
439 |
|
440 self ignoreWarnings ifFalse:[ |
|
441 requestor isNil ifTrue:[ |
|
442 warn := ScannerWarning new. |
|
443 warn startPosition:position. |
|
444 warn endPosition:endPos. |
|
445 warn lineNumber:tokenLineNr. |
|
446 warn errorString:((self warningMessagePrefix) , ' ' , aMessage). |
|
447 warn raiseRequest. |
|
448 ^ false |
|
449 ]. |
|
450 ^ requestor warning:aMessage position:position to:endPos from:self |
|
451 ]. |
|
452 ^ false |
|
453 ! |
|
454 |
|
455 parseError:aMessage |
|
456 "report an error" |
|
457 |
|
458 ^ self parseError:aMessage position:tokenStartPosition to:nil |
|
459 |
|
460 "Created: / 13.5.1998 / 16:45:13 / cg" |
|
461 ! |
|
462 |
|
463 parseError:aMessage position:position |
|
464 "report an error" |
|
465 |
|
466 ^ self parseError:aMessage position:position to:nil |
|
467 |
|
468 "Created: / 13.5.1998 / 16:45:05 / cg" |
|
469 ! |
|
470 |
|
471 parseError:aMessage position:position to:endPos |
|
472 "report an error" |
|
473 |
|
474 |m| |
|
475 |
|
476 m := (self errorMessagePrefix) , ' ' , (aMessage ? '???'). |
|
477 self notifyError:m position:position to:endPos. |
|
478 ^ false |
|
479 |
|
480 "Created: / 13.5.1998 / 16:44:55 / cg" |
|
481 "Modified: / 28.9.1998 / 19:29:27 / cg" |
|
482 ! |
|
483 |
|
484 showErrorMessage:aMessage position:pos |
|
485 "show an errormessage on the Transcript" |
|
486 |
|
487 Transcript showCR:(pos printString , ' [line: ' , tokenLineNr printString , '] ' , aMessage) |
|
488 ! |
|
489 |
|
490 syntaxError:aMessage |
|
491 "a syntax error happened - position is not known" |
|
492 |
|
493 ^ self syntaxError:aMessage position:tokenStartPosition |
|
494 ! |
|
495 |
|
496 syntaxError:aMessage position:position |
|
497 "a syntax error happened - only start position is known" |
|
498 |
|
499 ^ self syntaxError:aMessage position:position to:nil |
|
500 ! |
|
501 |
|
502 syntaxError:aMessage position:position to:endPos |
|
503 "a syntax error happened" |
|
504 |
|
505 |err| |
|
506 |
|
507 err := ScannerError new. |
|
508 err startPosition:tokenStartPosition. |
|
509 err endPosition:tokenEndPosition. |
|
510 err lineNumber:tokenLineNr. |
|
511 err errorString:((self errorMessagePrefix) , ' ' , aMessage). |
|
512 err raiseRequest |
|
513 ! |
|
514 |
|
515 warning:aMessage |
|
516 "a warning - position is not known" |
|
517 |
|
518 ^ self warning:aMessage position:tokenStartPosition |
|
519 ! |
|
520 |
|
521 warning:aMessage position:position |
|
522 "a warning - only start position is known" |
|
523 |
|
524 ^ self warning:aMessage position:position to:nil |
|
525 ! |
|
526 |
|
527 warning:aMessage position:position to:endPos |
|
528 "a warning" |
|
529 |
|
530 ^ self notifyWarning:((self warningMessagePrefix) , ' ' , aMessage) position:position to:endPos |
|
531 ! |
|
532 |
|
533 warningMessagePrefix |
|
534 ^ 'Warning:' |
|
535 ! ! |
|
536 |
|
537 !ScannerBase methodsFor:'general scanning'! |
|
538 |
|
539 scanPositionsFor:aTokenString inString:aSourceString |
|
540 "scan aSourceString for occurrances of aTokenString. |
|
541 Return a collection of start positions. |
|
542 Added for VW compatibility (to support syntax-highlight)." |
|
543 |
|
544 |searchType searchToken positions t| |
|
545 |
|
546 " |
|
547 first, look what kind of token we have to search for |
|
548 " |
|
549 self initializeFor:(ReadStream on:aTokenString). |
|
550 self nextToken. |
|
551 searchType := tokenType. |
|
552 searchToken := tokenValue. |
|
553 |
|
554 " |
|
555 start the real work ... |
|
556 " |
|
557 self initializeFor:(ReadStream on:aSourceString). |
|
558 positions := OrderedCollection new. |
|
559 |
|
560 [(t := self nextToken) ~~ #EOF] whileTrue:[ |
|
561 searchType == t ifTrue:[ |
|
562 (searchToken isNil or:[tokenValue = searchToken]) ifTrue:[ |
|
563 positions add:tokenStartPosition. |
|
564 ] |
|
565 ] |
|
566 ]. |
|
567 |
|
568 ^ positions |
|
569 |
|
570 " |
|
571 Scanner new scanPositionsFor:'hello' inString:'foo bar hello baz hello helloWorld' |
|
572 Scanner new scanPositionsFor:'3.14' inString:'foo 3.145 bar hello 3.14 baz hello 3.14 ''3.14''' |
|
573 Scanner new scanPositionsFor:'''3.14''' inString:'foo 3.145 bar hello 3.14 baz hello 3.14 ''3.14'' aaa' |
|
574 Scanner new scanPositionsFor:'16' inString:'foo 16 bar hello 16r10 baz hello 2r10000' |
|
575 " |
|
576 ! ! |
|
577 |
|
578 !ScannerBase methodsFor:'initialization'! |
|
579 |
|
580 initialize |
|
581 "initialize the scanner" |
|
582 |
|
583 "/actionArray notNil ifTrue:[ self halt ]. |
|
584 |
|
585 saveComments := false. |
|
586 parserFlags := ParserFlags new. |
|
587 |
|
588 (actionArray := self class actionArray) isNil ifTrue:[ |
|
589 self class setupActions. |
|
590 actionArray := self class actionArray |
|
591 ]. |
|
592 typeArray := self class typeArray. |
|
593 |
|
594 "Modified: / 14-03-2012 / 22:35:21 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
|
595 ! |
|
596 |
|
597 initializeFor:aStringOrStream |
|
598 "initialize the new scanner & prepare for reading from aStringOrStream" |
|
599 |
|
600 self initialize. |
|
601 self source:aStringOrStream. |
|
602 ! |
|
603 |
|
604 source:aStringOrStream |
|
605 "prepare for reading from aStringOrStream" |
|
606 |
|
607 tokenStartPosition := 1. |
|
608 tokenLineNr := lineNr := 1. |
|
609 currentComments := nil. |
|
610 |
|
611 aStringOrStream isStream ifFalse:[ |
|
612 source := ReadStream on:aStringOrStream |
|
613 ] ifTrue:[ |
|
614 source := aStringOrStream. |
|
615 ]. |
|
616 |
|
617 "Modified: / 26.5.1999 / 12:02:16 / stefan" |
|
618 ! ! |
|
619 |
|
620 !ScannerBase methodsFor:'private'! |
|
621 |
|
622 addComment:comment |
|
623 saveComments ifTrue:[ |
|
624 currentComments isNil ifTrue:[ |
|
625 currentComments := OrderedCollection with:comment |
|
626 ] ifFalse:[ |
|
627 currentComments add:comment |
|
628 ] |
|
629 ]. |
|
630 ! |
|
631 |
|
632 backupPosition |
|
633 "if reading from a stream, at the end we might have read |
|
634 one token too many" |
|
635 |
|
636 (tokenType == #EOF) ifFalse:[ |
|
637 source position:tokenStartPosition |
|
638 ] |
|
639 ! |
|
640 |
|
641 beginComment |
|
642 ^ self |
|
643 ! |
|
644 |
|
645 requestor:anObject |
|
646 "set the requestor to be notified about errors" |
|
647 |
|
648 requestor := anObject |
|
649 ! ! |
|
650 |
|
651 !ScannerBase methodsFor:'reading next token'! |
|
652 |
|
653 atEnd |
|
654 "true if at the end" |
|
655 |
|
656 ^ tokenType == #EOF. |
|
657 |
|
658 "Created: / 30-04-2011 / 11:24:13 / cg" |
|
659 ! |
|
660 |
|
661 isCommentCharacter:aCharacter |
|
662 self subclassResponsibility |
|
663 ! |
|
664 |
|
665 nextSingleCharacterToken:aCharacter |
|
666 "return a character token" |
|
667 |
|
668 tokenEndPosition := tokenStartPosition. |
|
669 tokenType := tokenValue := aCharacter. |
|
670 hereChar notNil ifTrue:[source next]. |
|
671 ^ tokenType |
|
672 |
|
673 "Modified: / 13.5.1998 / 15:10:23 / cg" |
|
674 ! |
|
675 |
|
676 nextToken |
|
677 "scan the next token from the source-stream; |
|
678 as a side effect, leave info in: |
|
679 tokenType - a symbol describing the kind of token |
|
680 token - its value as string or number |
|
681 tokenStartPosition - the tokens first characters position in the input stream |
|
682 tokenEndPosition - the tokens last characters position in the input stream |
|
683 tokenLineNr - the tokens first characters lineNumber in the input stream |
|
684 returns the tokenType. |
|
685 " |
|
686 |
|
687 |skipping actionBlock v ch tok| |
|
688 |
|
689 [true] whileTrue:[ |
|
690 peekChar notNil ifTrue:[ |
|
691 "/ kludge - should be called peekSym. |
|
692 "/ used when xlating Foo.Bar into Foo::Bar |
|
693 peekChar isSymbol ifTrue:[ |
|
694 tokenValue := nil. |
|
695 tokenType := peekChar. |
|
696 peekChar := nil. |
|
697 ^ tokenType |
|
698 ]. |
|
699 |
|
700 peekChar isSeparator ifTrue:[ |
|
701 peekChar == (Character cr) ifTrue:[ |
|
702 lineNr := lineNr + 1. |
|
703 ]. |
|
704 peekChar := peekChar2. |
|
705 peekChar2 := nil. |
|
706 ]. |
|
707 ]. |
|
708 peekChar notNil ifTrue:[ |
|
709 ch := peekChar. |
|
710 peekChar := peekChar2. |
|
711 peekChar2 := nil. |
|
712 hereChar := nil. |
|
713 tokenStartPosition := source position - 1. |
|
714 ] ifFalse:[ |
|
715 skipping := true. |
|
716 [skipping] whileTrue:[ |
|
717 hereChar := source skipSeparatorsExceptCR. |
|
718 hereChar == (Character cr) ifTrue:[ |
|
719 lineNr := lineNr + 1. |
|
720 source next. |
|
721 ] ifFalse:[ |
|
722 hereChar == (Character return) ifTrue:[ |
|
723 source next. |
|
724 ] ifFalse:[ |
|
725 (self isCommentCharacter:hereChar) ifTrue:[ |
|
726 "start of a comment" |
|
727 |
|
728 self skipComment. |
|
729 hereChar := source peekOrNil. |
|
730 ] ifFalse:[ |
|
731 skipping := false |
|
732 ] |
|
733 ] |
|
734 ] |
|
735 ]. |
|
736 hereChar isNil ifTrue:[ |
|
737 tokenValue := nil. |
|
738 tokenType := #EOF. |
|
739 ^ tokenType |
|
740 ]. |
|
741 ch := hereChar. |
|
742 tokenStartPosition := source position. |
|
743 ]. |
|
744 tokenLineNr := lineNr. |
|
745 |
|
746 (v := ch asciiValue) == 0 ifTrue:[ |
|
747 v := Character space codePoint |
|
748 ]. |
|
749 actionBlock := actionArray at:v. |
|
750 actionBlock notNil ifTrue:[ |
|
751 tok := actionBlock value:self value:ch. |
|
752 tok notNil ifTrue:[ |
|
753 ^ tok |
|
754 ]. |
|
755 ] ifFalse:[ |
|
756 self syntaxError:('invalid character: ''' , ch asString , ''' ', |
|
757 '(' , v printString , ')') |
|
758 position:tokenStartPosition to:tokenStartPosition. |
|
759 source next. |
|
760 tokenValue := nil. |
|
761 tokenType := #Error. |
|
762 ^ #Error |
|
763 ] |
|
764 ]. |
|
765 ! |
|
766 |
|
767 skipComment |
|
768 self subclassResponsibility |
|
769 ! ! |
|
770 |
|
771 !ScannerBase methodsFor:'reading next token - private'! |
|
772 |
|
773 nextMantissa:radix |
|
774 "read the mantissa of a radix number. |
|
775 Return post-decimal value (i.e. 0.xxxx); leave number of post-decimal |
|
776 digits in numberScale; scaled post-decimal value in scaledMantissaValue (xxx)." |
|
777 |
|
778 |nextChar value factor| |
|
779 |
|
780 value := scaledMantissaValue := 0. |
|
781 factor := 1.0 / radix. |
|
782 nextChar := source peekOrNil. |
|
783 numberScale := 0. |
|
784 |
|
785 [(nextChar notNil and:[nextChar isDigitRadix:radix])] whileTrue:[ |
|
786 scaledMantissaValue := (scaledMantissaValue * radix) + (nextChar digitValue). |
|
787 value := value + (nextChar digitValue * factor). |
|
788 factor := factor / radix. |
|
789 numberScale := numberScale + 1. |
|
790 nextChar := source nextPeek |
|
791 ]. |
|
792 ^ value |
|
793 |
|
794 "Modified: / 5.3.1998 / 02:54:11 / cg" |
|
795 ! ! |
|
796 |
|
797 !ScannerBase class methodsFor:'documentation'! |
|
798 |
|
799 version_HG |
|
800 |
|
801 ^ '$Changeset: <not expanded> $' |
|
802 ! ! |
|
803 |
|
804 ScannerBase initialize! |