--- a/Scanner.st Sat Jun 08 15:03:07 2019 +0200
+++ b/Scanner.st Sat Jun 08 16:16:19 2019 +0200
@@ -1,3 +1,5 @@
+"{ Encoding: utf8 }"
+
"
COPYRIGHT (c) 1989 by Claus Gittinger
All Rights Reserved
@@ -143,7 +145,7 @@
extendedBinarySelectorCharacters
"return a collection of characters which are optionally allowed in binary selectors"
- "/ ^ '±×·÷«»'.
+ "/ ^ '±×·÷«»'.
^ String
with:(Character value:16rB1) "/ plus-minus
with:(Character value:16rD7) "/ times
@@ -1856,13 +1858,13 @@
!
warnParagraphAt:position
- "warn about §-character in an identifier"
+ "warn about §-character in an identifier"
ignoreWarnings ifFalse:[
"/ didWarnAboutParagraphInIdentifier ifFalse:[
parserFlags warnParagraphInIdentifier ifTrue:[
self
- warning:'§-characters in identifiers/symbols are nonportable'
+ warning:'§-characters in identifiers/symbols are nonportable'
doNotShowAgainAction:[ ParserFlags warnParagraphInIdentifier:false ]
position:position to:position.
"
@@ -3006,7 +3008,7 @@
(((nextChar == $_) and:[allowUnderscoreInIdentifier])
or:[ (allowDollarInIdentifier and:[nextChar == $$ ])
- or:[ (nextChar == $§ and:[ parserFlags allowParagraphInIdentifier])
+ or:[ (nextChar == $§ and:[ parserFlags allowParagraphInIdentifier])
or:[ allowNationalCharactersInIdentifier and:[ nextChar notNil and:[nextChar isNationalLetter]]]]]) ifTrue:[
pos := source position + 1.
nextChar == $_ ifTrue:[
@@ -3015,7 +3017,7 @@
nextChar == $$ ifTrue:[
self warnDollarAt:pos.
] ifFalse:[
- nextChar == $§ ifTrue:[
+ nextChar == $§ ifTrue:[
self warnParagraphAt:pos.
] ifFalse:[
"/ self warnNationalCharacterAt:pos.
@@ -3035,7 +3037,7 @@
].
ok := ((nextChar == $_) and:[allowUnderscoreInIdentifier])
or:[((nextChar == $$ ) and:[allowDollarInIdentifier])
- or:[((nextChar == $§ ) and:[parserFlags allowParagraphInIdentifier])
+ or:[((nextChar == $§ ) and:[parserFlags allowParagraphInIdentifier])
or:[(nextChar notNil and:[nextChar isNationalLetter]) and:[allowNationalCharactersInIdentifier]]]].
]
].
@@ -3600,161 +3602,157 @@
|skipping actionBlock v ch tok|
source isPositionable ifTrue:[
- tokenLastEndPosition := source position+1.
+ tokenLastEndPosition := source position+1.
].
[
- peekChar notNil ifTrue:[
- "/ kludge - should be called peekSym.
- "/ used when xlating Foo.Bar into Foo::Bar
- peekChar isSymbol ifTrue:[
- token := nil.
- tokenType := peekChar.
- peekChar := nil.
- ^ tokenType
- ].
-
- (peekChar isSeparator or:[ peekChar codePoint == 16rFEFF ]) ifTrue:[
- peekChar == (Character cr) ifTrue:[
- lineNr := lineNr + 1.
- ].
- hereChar := peekChar.
- peekChar := peekChar2.
- peekChar2 := nil.
- (hereChar == Character cr) ifTrue:[
- (self eolIsWhiteSpace) ifFalse:[
- token := nil.
- tokenType := #EOL.
- ^ tokenType
- ].
- ].
- ].
- ].
-
- peekChar notNil ifTrue:[
- ch := peekChar.
- peekChar := peekChar2.
- peekChar2 := nil.
- hereChar := nil.
- ] ifFalse:[
- skipping := true.
- [skipping] whileTrue:[
-
- outStream notNil ifTrue:[
- [
- hereChar := source peekOrNil.
- (hereChar notNil
- and:[(hereChar == Character space) or:[hereChar isSeparator]])
- ] whileTrue:[
- source next.
- outStream space.
- outCol := outCol + 1.
- hereChar == (Character cr) ifTrue:[
- self eolIsWhiteSpace ifFalse:[
- source isPositionable ifTrue:[
- tokenPosition := source position.
- ].
- token := nil.
- tokenType := #EOL.
- ^ tokenType
- ].
- ]
- ]
- ] ifFalse:[
- hereChar := source skipSeparatorsExceptCR.
- ].
-
- hereChar isNil ifTrue:[
- skipping := false
- ] ifFalse:[
- hereChar == (Character cr) ifTrue:[
- lineNr := lineNr + 1.
- source next.
- outStream notNil ifTrue:[
- outStream cr.
- outCol := 1
- ].
- self eolIsWhiteSpace ifFalse:[
- source isPositionable ifTrue:[
- tokenPosition := source position.
- ].
- token := nil.
- tokenType := #EOL.
- ^ tokenType
- ].
- ] ifFalse:[
- hereChar == (Character return) ifTrue:[
- outStream notNil ifTrue:[
- outStream nextPut:hereChar.
- outCol := 1
- ].
- source next.
- ] ifFalse:[
- (self isCommentCharacter:hereChar) ifTrue:[
- "start of a comment"
-
- self skipComment.
- hereChar := source peekOrNil.
- ] ifFalse:[
- skipping := false
- ]
- ]
- ]
- ].
- ].
- hereChar isNil ifTrue:[
- source isPositionable ifTrue:[
- tokenPosition := source position + 1.
- ].
- token := nil.
- tokenType := #EOF.
- ^ tokenType
- ].
- ch := hereChar
- ].
- source isPositionable ifTrue:[
- tokenPosition := source position + 1.
- ].
- tokenLineNr := lineNr.
-
- (v := ch codePoint) == 0 ifTrue:[
- v := Character space codePoint
- ].
- v <= 16rFF ifTrue:[
- actionBlock := actionArray at:v.
- ] ifFalse:[
- actionBlock := unicodeActions at:v ifAbsent:nil
- ].
- actionBlock notNil ifTrue:[
- tok := actionBlock value:self value:ch.
- tok notNil ifTrue:[
- ^ tok
- ].
- "/ a nil token means: continue reading
- ] ifFalse:[
- (ch isNationalLetter and:[parserFlags allowNationalCharactersInIdentifier]) ifTrue:[
- tok := self nextIdentifier.
- tok notNil ifTrue:[
- ^ tok
- ].
- "/ a nil token means: continue reading
- ] ifFalse:[
- (ch == $§ and:[parserFlags allowParagraphInIdentifier]) ifTrue:[
- tok := self nextIdentifier.
- tok notNil ifTrue:[
- ^ tok
- ].
- "/ a nil token means: continue reading
- ] ifFalse:[
- ^ self invalidCharacter:ch.
- ].
- ].
- ]
+ peekChar notNil ifTrue:[
+ "/ kludge - should be called peekSym.
+ "/ used when xlating Foo.Bar into Foo::Bar
+ peekChar isSymbol ifTrue:[
+ token := nil.
+ tokenType := peekChar.
+ peekChar := nil.
+ ^ tokenType
+ ].
+
+ (peekChar isSeparator or:[ peekChar codePoint == 16rFEFF ]) ifTrue:[
+ peekChar == (Character cr) ifTrue:[
+ lineNr := lineNr + 1.
+ ].
+ hereChar := peekChar.
+ peekChar := peekChar2.
+ peekChar2 := nil.
+ (hereChar == Character cr) ifTrue:[
+ (self eolIsWhiteSpace) ifFalse:[
+ token := nil.
+ tokenType := #EOL.
+ ^ tokenType
+ ].
+ ].
+ ].
+ ].
+
+ peekChar notNil ifTrue:[
+ ch := peekChar.
+ peekChar := peekChar2.
+ peekChar2 := nil.
+ hereChar := nil.
+ ] ifFalse:[
+ skipping := true.
+ [skipping] whileTrue:[
+
+ outStream notNil ifTrue:[
+ [
+ hereChar := source peekOrNil.
+ (hereChar notNil
+ and:[(hereChar == Character space) or:[hereChar isSeparator]])
+ ] whileTrue:[
+ source next.
+ outStream space.
+ outCol := outCol + 1.
+ hereChar == (Character cr) ifTrue:[
+ self eolIsWhiteSpace ifFalse:[
+ source isPositionable ifTrue:[
+ tokenPosition := source position.
+ ].
+ token := nil.
+ tokenType := #EOL.
+ ^ tokenType
+ ].
+ ]
+ ]
+ ] ifFalse:[
+ hereChar := source skipSeparatorsExceptCR.
+ ].
+
+ hereChar isNil ifTrue:[
+ skipping := false
+ ] ifFalse:[
+ hereChar == (Character cr) ifTrue:[
+ lineNr := lineNr + 1.
+ source next.
+ outStream notNil ifTrue:[
+ outStream cr.
+ outCol := 1
+ ].
+ self eolIsWhiteSpace ifFalse:[
+ source isPositionable ifTrue:[
+ tokenPosition := source position.
+ ].
+ token := nil.
+ tokenType := #EOL.
+ ^ tokenType
+ ].
+ ] ifFalse:[
+ hereChar == (Character return) ifTrue:[
+ outStream notNil ifTrue:[
+ outStream nextPut:hereChar.
+ outCol := 1
+ ].
+ source next.
+ ] ifFalse:[
+ (self isCommentCharacter:hereChar) ifTrue:[
+ "start of a comment"
+
+ self skipComment.
+ hereChar := source peekOrNil.
+ ] ifFalse:[
+ skipping := false
+ ]
+ ]
+ ]
+ ].
+ ].
+ hereChar isNil ifTrue:[
+ source isPositionable ifTrue:[
+ tokenPosition := source position + 1.
+ ].
+ token := nil.
+ tokenType := #EOF.
+ ^ tokenType
+ ].
+ ch := hereChar
+ ].
+ source isPositionable ifTrue:[
+ tokenPosition := source position + 1.
+ ].
+ tokenLineNr := lineNr.
+
+ (v := ch codePoint) == 0 ifTrue:[
+ v := Character space codePoint
+ ].
+ v <= 16rFF ifTrue:[
+ actionBlock := actionArray at:v.
+ ] ifFalse:[
+ actionBlock := unicodeActions at:v ifAbsent:nil
+ ].
+ actionBlock notNil ifTrue:[
+ tok := actionBlock value:self value:ch.
+ tok notNil ifTrue:[
+ ^ tok
+ ].
+ "/ a nil token means: continue reading
+ ] ifFalse:[
+ ((ch isNationalLetter and:[parserFlags allowNationalCharactersInIdentifier])
+ or:[ (ch == $§ and:[parserFlags allowParagraphInIdentifier])
+ or:[ (ch isGreekLetter and:[parserFlags allowGreekCharactersInIdentifier])
+ ]]) ifTrue:[
+ tok := self nextIdentifier.
+ tok notNil ifTrue:[
+ ^ tok
+ ].
+ "/ a nil token means: continue reading
+ ] ifFalse:[
+ ^ self invalidCharacter:ch.
+ ].
+ ]
] loop.
"Modified: / 13-09-1995 / 12:56:14 / claus"
"Modified: / 27-07-2011 / 15:36:53 / Jan Vrany <jan.vrany@fit.cvut.cz>"
"Modified: / 12-02-2017 / 11:27:59 / cg"
+ "Modified: / 08-06-2019 / 14:59:14 / Claus Gittinger"
!
nextToken:aCharacter