--- a/Scanner.st Wed Aug 05 15:42:21 2020 +0200
+++ b/Scanner.st Wed Aug 05 15:42:29 2020 +0200
@@ -1,5 +1,3 @@
-"{ Encoding: utf8 }"
-
"
COPYRIGHT (c) 1989 by Claus Gittinger
All Rights Reserved
@@ -152,7 +150,7 @@
extendedBinarySelectorCharacters
"return a collection of characters which are optionally allowed in binary selectors"
- "/ ^ '±×·÷«»'.
+ "/ ^ '±×·÷«»'.
^ String
with:(Character value:16rB1) "/ plus-minus
with:(Character value:16rD7) "/ times
@@ -1904,13 +1902,13 @@
!
warnParagraphAt:position
- "warn about §-character in an identifier"
+ "warn about §-character in an identifier"
ignoreWarnings ifFalse:[
"/ didWarnAboutParagraphInIdentifier ifFalse:[
parserFlags warnParagraphInIdentifier ifTrue:[
self
- warning:'§-characters in identifiers/symbols are nonportable'
+ warning:'§-characters in identifiers/symbols are nonportable'
doNotShowAgainAction:(self actionToDisableWarning:#warnParagraphInIdentifier)
position:position to:position.
"
@@ -3062,7 +3060,7 @@
(((nextChar == $_) and:[allowUnderscoreInIdentifier])
or:[ (allowDollarInIdentifier and:[nextChar == $$ ])
- or:[ (nextChar == $§ and:[ allowParagraphInIdentifier])
+ or:[ (nextChar == $§ and:[ allowParagraphInIdentifier])
or:[ (allowNationalCharactersInIdentifier and:[ nextChar notNil and:[nextChar isNationalLetter]])
or:[ (allowGreekCharactersInIdentifier and:[ nextChar notNil and:[nextChar isNationalLetter]])
]]]]) ifTrue:[
@@ -3073,7 +3071,7 @@
nextChar == $$ ifTrue:[
self warnDollarAt:pos.
] ifFalse:[
- nextChar == $§ ifTrue:[
+ nextChar == $§ ifTrue:[
self warnParagraphAt:pos.
] ifFalse:[
"/ self warnNationalCharacterAt:pos.
@@ -3093,7 +3091,7 @@
].
ok := ((nextChar == $_) and:[allowUnderscoreInIdentifier])
or:[((nextChar == $$ ) and:[allowDollarInIdentifier])
- or:[((nextChar == $§ ) and:[allowParagraphInIdentifier])
+ or:[((nextChar == $§ ) and:[allowParagraphInIdentifier])
or:[(nextChar notNil and:[allowNationalCharactersInIdentifier and:[nextChar isNationalLetter]])
or:[(nextChar notNil and:[allowGreekCharactersInIdentifier and:[nextChar isGreekLetter]])
]]]].
@@ -3906,7 +3904,7 @@
"/ a nil token means: continue reading
] ifFalse:[
((ch isNationalLetter and:[parserFlags allowNationalCharactersInIdentifier])
- or:[ (ch == $§ and:[parserFlags allowParagraphInIdentifier])
+ or:[ (ch == $§ and:[parserFlags allowParagraphInIdentifier])
or:[ (ch isGreekLetter and:[parserFlags allowGreekCharactersInIdentifier])
]]) ifTrue:[
tok := self nextIdentifier.
@@ -4140,7 +4138,7 @@
lineNr := lineNr + 1.
!
-xnextString:delimiter escapeStyle:escapeStyle
+xnextString:delimiter escapeStyle:escapeStyleOrNil
"a quote has been scanned; scan the string (caring for doubled quotes).
escapeStyle may be:
nil - old style ST80 strings (no character escapes)
@@ -4148,10 +4146,14 @@
'c' - C-style escapes:
\n,\t,\r,\b,\xXX,\uXXXX,\UXXXXXX,
- 'e' - C-style plus embedded escapes:
+ 'e' - C-style with embedded expressions:
e'...{ expr1 } ... { exprN }' will generate:
'...%1 ... %N' bindWithArguments:{ expr1 . ... . exprN }
+ 'i' - C-style plus embedded expressions internationalized:
+ i'...{ expr1 } ... { exprN }' will generate:
+ resources string:'...%1 ... %N' withArguments:{ expr1 . ... . exprN }
+
'r' - regex
'x' - extended-style escapes:
@@ -4162,12 +4164,32 @@
index "{ Class: SmallInteger }"
len "{ Class: SmallInteger }"
inString peekChar
- isCString isEString isRString isIString|
-
- isEString := (escapeStyle = 'e').
- isCString := (escapeStyle = 'c').
- isRString := (escapeStyle = 'r').
- isIString := (escapeStyle = 'i').
+ isCString isEString isRString isIString typeOfToken typeOfTokenIfNoEmbeddedExpression|
+
+ isEString := isCString := isRString := isIString := false.
+
+ (isEString := (escapeStyleOrNil = 'e')) ifTrue:[
+ typeOfToken := #EStringFragment.
+ typeOfTokenIfNoEmbeddedExpression := #String.
+ ] ifFalse:[
+ (isCString := (escapeStyleOrNil = 'c')) ifTrue:[
+ typeOfToken := #String
+ ] ifFalse:[
+ (isRString := (escapeStyleOrNil = 'r')) ifTrue:[
+ typeOfToken := #RegexString
+ ] ifFalse:[
+ (isIString := (escapeStyleOrNil = 'i')) ifTrue:[
+ typeOfToken := #IStringFragment.
+ typeOfTokenIfNoEmbeddedExpression := #IString.
+ ] ifFalse:[
+ escapeStyleOrNil notNil ifTrue:[
+ self parseError:'unhandled string style'.
+ ].
+ typeOfToken := #String
+ ]
+ ]
+ ]
+ ].
string := String uninitializedNew:20.
len := 20.
@@ -4195,18 +4217,18 @@
inString := false
]
] ifFalse:[
- (escapeStyle notNil and:[isRString not]) ifTrue:[
+ (escapeStyleOrNil notNil and:[isRString not]) ifTrue:[
((nextChar == ${) and:[isEString or:[isIString]]) ifTrue:[
"/ bail out, to read one expression
tokenValue := token := string copyTo:(index - 1).
- tokenType := isIString ifTrue:[#IStringFragment] ifFalse:[#EStringFragment].
+ tokenType := typeOfToken.
^ tokenType
].
(nextChar == $\) ifTrue:[
peekChar := source peekOrNil.
peekChar notNil ifTrue:[
source next.
- nextChar := self escapeCharacterFor:peekChar escapeStyle:escapeStyle.
+ nextChar := self escapeCharacterFor:peekChar escapeStyle:escapeStyleOrNil.
]
]
]
@@ -4235,7 +4257,7 @@
].
tokenValue := token := string copyTo:(index - 1).
- tokenType := isRString ifTrue:[#RegexString] ifFalse:[#String].
+ tokenType := typeOfTokenIfNoEmbeddedExpression ? typeOfToken.
^ tokenType
"Created: / 22-05-2019 / 20:31:36 / Claus Gittinger"