44 ! |
44 ! |
45 |
45 |
46 documentation |
46 documentation |
47 " |
47 " |
48 a first version of a tokenStream. |
48 a first version of a tokenStream. |
49 This is still being constructed and will probably be enhanced |
49 This is still being constructed and will probably be enhanced |
50 and may finally help a simplified Scanner class. |
50 and may finally help a simplified Scanner class. |
51 |
51 |
52 For now, it may be useful when textual input files are to be read and |
52 For now, it may be useful when textual input files are to be read and |
53 parsed. For example, ascii data files are often in a simple free form format |
53 parsed. For example, ascii data files are often in a simple free form format |
54 which requires some little processing. |
54 which requires some little processing. |
55 Dont blame me, if its not powerful enough for your needs. |
55 Dont blame me, if its not powerful enough for your needs. |
56 |
56 |
57 operation: |
57 operation: |
58 |
58 |
59 characters are read from a real input stream |
59 characters are read from a real input stream |
60 and the tokenizer dispatches to a token reading method by the help |
60 and the tokenizer dispatches to a token reading method by the help |
61 of an actionTable, which is indexed by the tokenType. |
61 of an actionTable, which is indexed by the tokenType. |
62 The tokenType itself is aquired via another table via |
62 The tokenType itself is aquired via another table via |
63 the characters ascii code. |
63 the characters ascii code. |
64 |
64 |
65 By default, the table is setup to only read numbers |
65 By default, the table is setup to only read numbers |
66 and identifiers. |
66 and identifiers. |
67 Whitespace is ignored, and all other characters return themself. |
67 Whitespace is ignored, and all other characters return themself. |
68 |
68 |
69 The returned tokens are either symbols (#Identifier, #Integer ..) or |
69 The returned tokens are either symbols (#Identifier, #Integer ..) or |
70 characters ($+ $, etc.) |
70 characters ($+ $, etc.) |
71 If its #Identifier, the name is found in tokenName (there is an access method for that). |
71 If its #Identifier, the name is found in tokenName (there is an access method for that). |
134 s next == #Integer ifTrue:[ |
134 s next == #Integer ifTrue:[ |
135 num1 := s tokenValue. |
135 num1 := s tokenValue. |
136 s next == $+ ifTrue:[ |
136 s next == $+ ifTrue:[ |
137 s next == #Integer ifTrue:[ |
137 s next == #Integer ifTrue:[ |
138 num2 := s tokenValue. |
138 num2 := s tokenValue. |
139 Transcript showCR:num1 printString |
139 Transcript showCR:num1 printString |
140 , ' + ' |
140 , ' + ' |
141 , num2 printString |
141 , num2 printString |
142 , ' => ' |
142 , ' => ' |
143 , (num1 + num2) printString. |
143 , (num1 + num2) printString. |
144 ] |
144 ] |
145 ] |
145 ] |
146 ] |
146 ] |
147 ]. |
147 ]. |
164 s next == #Integer ifTrue:[ |
164 s next == #Integer ifTrue:[ |
165 num1 := s tokenValue. |
165 num1 := s tokenValue. |
166 s next == $+ ifTrue:[ |
166 s next == $+ ifTrue:[ |
167 s next == #Integer ifTrue:[ |
167 s next == #Integer ifTrue:[ |
168 num2 := s tokenValue. |
168 num2 := s tokenValue. |
169 Transcript showCR:num1 printString |
169 Transcript showCR:num1 printString |
170 , ' + ' |
170 , ' + ' |
171 , num2 printString |
171 , num2 printString |
172 , ' => ' |
172 , ' => ' |
173 , (num1 + num2) printString. |
173 , (num1 + num2) printString. |
174 ] |
174 ] |
175 ] |
175 ] |
176 ] |
176 ] |
177 ]. |
177 ]. |
526 "Created: 1.2.1996 / 18:26:27 / cg" |
526 "Created: 1.2.1996 / 18:26:27 / cg" |
527 "Modified: 11.1.1997 / 19:24:19 / cg" |
527 "Modified: 11.1.1997 / 19:24:19 / cg" |
528 ! |
528 ! |
529 |
529 |
530 nextIdentifier |
530 nextIdentifier |
531 |nextChar string oldString |
531 |nextChar string oldString |
532 index "{ Class: SmallInteger }" |
532 index "{ Class: SmallInteger }" |
533 max "{ Class: SmallInteger }" |
533 max "{ Class: SmallInteger }" |
534 t done| |
534 t done| |
535 |
535 |
536 nextChar := inputStream peek. |
536 nextChar := inputStream peek. |
537 string := String basicNew:20. |
537 string := String basicNew:20. |
538 index := 0. |
538 index := 0. |
780 skipping := true. |
780 skipping := true. |
781 [skipping] whileTrue:[ |
781 [skipping] whileTrue:[ |
782 outStream notNil ifTrue:[ |
782 outStream notNil ifTrue:[ |
783 [(hereChar := inputStream peek) == Character space] whileTrue:[ |
783 [(hereChar := inputStream peek) == Character space] whileTrue:[ |
784 inputStream next. |
784 inputStream next. |
785 outStream space. |
785 outStream space. |
786 outCol := outCol + 1. |
786 outCol := outCol + 1. |
787 ] |
787 ] |
788 ] ifFalse:[ |
788 ] ifFalse:[ |
789 hereChar := inputStream skipSeparatorsExceptCR. |
789 hereChar := inputStream skipSeparatorsExceptCR. |
790 ]. |
790 ]. |