allow for action to skip something and continue in nextToken
(needed for eol comments using nextMulti)
--- a/Scanner.st Wed Oct 21 22:24:12 1998 +0200
+++ b/Scanner.st Thu Oct 22 22:22:46 1998 +0200
@@ -1638,94 +1638,97 @@
nextToken
"return the next token from the source-stream"
- |skipping actionBlock v ch|
+ |skipping actionBlock v ch tok|
- peekChar notNil ifTrue:[
- peekChar isSeparator ifTrue:[
- peekChar == (Character cr) ifTrue:[
- lineNr := lineNr + 1.
+ [true] whileTrue:[
+ peekChar notNil ifTrue:[
+ peekChar isSeparator ifTrue:[
+ peekChar == (Character cr) ifTrue:[
+ lineNr := lineNr + 1.
+ ].
+ peekChar := peekChar2.
+ peekChar2 := nil.
].
+ ].
+
+ peekChar notNil ifTrue:[
+ ch := peekChar.
peekChar := peekChar2.
peekChar2 := nil.
+ hereChar := nil.
+ ] ifFalse:[
+ skipping := true.
+ [skipping] whileTrue:[
+
+ outStream notNil ifTrue:[
+ hereChar := source peekOrNil.
+ [(hereChar == Character space)
+ or:[hereChar isSeparator]
+ ] whileTrue:[
+ source next.
+ outStream space.
+ outCol := outCol + 1.
+ ]
+ ] ifFalse:[
+ hereChar := source skipSeparatorsExceptCR.
+ ].
+
+ hereChar == (Character cr) ifTrue:[
+ lineNr := lineNr + 1.
+ source next.
+ outStream notNil ifTrue:[
+ outStream cr.
+ outCol := 1
+ ]
+ ] ifFalse:[
+ hereChar == (Character return) ifTrue:[
+ outStream notNil ifTrue:[
+ outStream nextPut:hereChar.
+ outCol := 1
+ ].
+ source next.
+ ] ifFalse:[
+ (self isCommentCharacter:hereChar) ifTrue:[
+ "start of a comment"
+
+ self skipComment.
+ hereChar := source peekOrNil.
+ ] ifFalse:[
+ skipping := false
+ ]
+ ]
+ ]
+ ].
+ hereChar isNil ifTrue:[
+ token := nil.
+ tokenType := #EOF.
+ ^ tokenType
+ ].
+ ch := hereChar
].
+ tokenPosition := source position.
+ tokenLineNr := lineNr.
+
+ (v := ch asciiValue) == 0 ifTrue:[
+ v := Character space asciiValue
+ ].
+ actionBlock := actionArray at:v.
+ actionBlock notNil ifTrue:[
+ tok := actionBlock value:self value:ch.
+ tok notNil ifTrue:[^ tok].
+ ] ifFalse:[
+ self syntaxError:('invalid character: ''' , ch asString , ''' ',
+ '(' , v printString , ')')
+ position:tokenPosition to:tokenPosition.
+ source next.
+ tokenName := nil.
+ tokenType := token := #Error.
+ ^ #Error
+ ]
].
- peekChar notNil ifTrue:[
- ch := peekChar.
- peekChar := peekChar2.
- peekChar2 := nil.
- hereChar := nil.
- ] ifFalse:[
- skipping := true.
- [skipping] whileTrue:[
-
- outStream notNil ifTrue:[
- hereChar := source peekOrNil.
- [(hereChar == Character space)
- or:[hereChar isSeparator]
- ] whileTrue:[
- source next.
- outStream space.
- outCol := outCol + 1.
- ]
- ] ifFalse:[
- hereChar := source skipSeparatorsExceptCR.
- ].
-
- hereChar == (Character cr) ifTrue:[
- lineNr := lineNr + 1.
- source next.
- outStream notNil ifTrue:[
- outStream cr.
- outCol := 1
- ]
- ] ifFalse:[
- hereChar == (Character return) ifTrue:[
- outStream notNil ifTrue:[
- outStream nextPut:hereChar.
- outCol := 1
- ].
- source next.
- ] ifFalse:[
- (self isCommentCharacter:hereChar) ifTrue:[
- "start of a comment"
-
- self skipComment.
- hereChar := source peekOrNil.
- ] ifFalse:[
- skipping := false
- ]
- ]
- ]
- ].
- hereChar isNil ifTrue:[
- token := nil.
- tokenType := #EOF.
- ^ tokenType
- ].
- ch := hereChar
- ].
- tokenPosition := source position.
- tokenLineNr := lineNr.
-
- (v := ch asciiValue) == 0 ifTrue:[
- v := Character space asciiValue
- ].
- actionBlock := actionArray at:v.
- actionBlock notNil ifTrue:[
- ^ actionBlock value:self value:ch
- ].
-
- self syntaxError:('invalid character: ''' , ch asString , ''' ',
- '(' , v printString , ')')
- position:tokenPosition to:tokenPosition.
- source next.
- tokenName := nil.
- tokenType := token := #Error.
- ^ #Error
-
"Modified: / 13.9.1995 / 12:56:14 / claus"
- "Modified: / 18.6.1998 / 23:11:24 / cg"
+ "Modified: / 22.10.1998 / 22:15:27 / cg"
!
nextToken:aCharacter
@@ -1881,6 +1884,6 @@
!Scanner class methodsFor:'documentation'!
version
- ^ '$Header: /cvs/stx/stx/libcomp/Scanner.st,v 1.87 1998-10-17 14:50:17 cg Exp $'
+ ^ '$Header: /cvs/stx/stx/libcomp/Scanner.st,v 1.88 1998-10-22 20:22:46 cg Exp $'
! !
Scanner initialize!