Scanner.st
changeset 4705 7abcbbf55231
parent 4702 03ceb59050b4
child 4715 84c2d6467ff1
--- a/Scanner.st	Wed Aug 05 15:42:21 2020 +0200
+++ b/Scanner.st	Wed Aug 05 15:42:29 2020 +0200
@@ -1,5 +1,3 @@
-"{ Encoding: utf8 }"
-
 "
  COPYRIGHT (c) 1989 by Claus Gittinger
 	      All Rights Reserved
@@ -152,7 +150,7 @@
 extendedBinarySelectorCharacters
     "return a collection of characters which are optionally allowed in binary selectors"
 
-    "/ ^ '±×·÷«»'.
+    "/ ^ '±×·÷«»'.
     ^ String
 	with:(Character value:16rB1)  "/ plus-minus
 	with:(Character value:16rD7)  "/ times
@@ -1904,13 +1902,13 @@
 !
 
 warnParagraphAt:position
-    "warn about §-character in an identifier"
+    "warn about §-character in an identifier"
 
     ignoreWarnings ifFalse:[
         "/ didWarnAboutParagraphInIdentifier ifFalse:[
             parserFlags warnParagraphInIdentifier ifTrue:[
                 self
-                    warning:'§-characters in identifiers/symbols are nonportable'
+                    warning:'§-characters in identifiers/symbols are nonportable'
                     doNotShowAgainAction:(self actionToDisableWarning:#warnParagraphInIdentifier)
                     position:position to:position.
                 "
@@ -3062,7 +3060,7 @@
 
     (((nextChar == $_) and:[allowUnderscoreInIdentifier])
       or:[ (allowDollarInIdentifier and:[nextChar == $$ ])
-      or:[ (nextChar == $§ and:[ allowParagraphInIdentifier])
+      or:[ (nextChar == $§ and:[ allowParagraphInIdentifier])
       or:[ (allowNationalCharactersInIdentifier and:[ nextChar notNil and:[nextChar isNationalLetter]])
       or:[ (allowGreekCharactersInIdentifier and:[ nextChar notNil and:[nextChar isNationalLetter]])
     ]]]]) ifTrue:[
@@ -3073,7 +3071,7 @@
             nextChar == $$ ifTrue:[
                 self warnDollarAt:pos.
             ] ifFalse:[
-                nextChar == $§ ifTrue:[
+                nextChar == $§ ifTrue:[
                     self warnParagraphAt:pos.
                 ] ifFalse:[
                     "/ self warnNationalCharacterAt:pos.
@@ -3093,7 +3091,7 @@
                 ].
                 ok := ((nextChar == $_) and:[allowUnderscoreInIdentifier])
                         or:[((nextChar == $$ ) and:[allowDollarInIdentifier])
-                        or:[((nextChar == $§ ) and:[allowParagraphInIdentifier])
+                        or:[((nextChar == $§ ) and:[allowParagraphInIdentifier])
                         or:[(nextChar notNil and:[allowNationalCharactersInIdentifier and:[nextChar isNationalLetter]])
                         or:[(nextChar notNil and:[allowGreekCharactersInIdentifier and:[nextChar isGreekLetter]])
                       ]]]].
@@ -3906,7 +3904,7 @@
             "/ a nil token means: continue reading
         ] ifFalse:[
             ((ch isNationalLetter and:[parserFlags allowNationalCharactersInIdentifier])
-              or:[ (ch == $§ and:[parserFlags allowParagraphInIdentifier])
+              or:[ (ch == $§ and:[parserFlags allowParagraphInIdentifier])
               or:[ (ch isGreekLetter and:[parserFlags allowGreekCharactersInIdentifier])
             ]]) ifTrue:[
                 tok := self nextIdentifier.
@@ -4140,7 +4138,7 @@
     lineNr := lineNr + 1.
 !
 
-xnextString:delimiter escapeStyle:escapeStyle
+xnextString:delimiter escapeStyle:escapeStyleOrNil
     "a quote has been scanned; scan the string (caring for doubled quotes).
      escapeStyle may be:
         nil - old style ST80 strings (no character escapes)
@@ -4148,10 +4146,14 @@
         'c' - C-style escapes:
                         \n,\t,\r,\b,\xXX,\uXXXX,\UXXXXXX,
 
-        'e' - C-style plus embedded escapes:
+        'e' - C-style with embedded expressions:
                         e'...{ expr1 } ... { exprN }' will generate:
                         '...%1 ... %N' bindWithArguments:{ expr1 . ... . exprN }
 
+        'i' - C-style plus embedded expressions internationalized:
+                        i'...{ expr1 } ... { exprN }' will generate:
+                        resources string:'...%1 ... %N' withArguments:{ expr1 . ... . exprN }
+
         'r' - regex
 
         'x' - extended-style escapes:
@@ -4162,12 +4164,32 @@
      index "{ Class: SmallInteger }"
      len   "{ Class: SmallInteger }"
      inString peekChar
-     isCString isEString isRString isIString|
-
-    isEString := (escapeStyle = 'e').
-    isCString := (escapeStyle = 'c').
-    isRString := (escapeStyle = 'r').
-    isIString := (escapeStyle = 'i').
+     isCString isEString isRString isIString typeOfToken typeOfTokenIfNoEmbeddedExpression|
+
+    isEString := isCString := isRString := isIString := false.
+
+    (isEString := (escapeStyleOrNil = 'e')) ifTrue:[
+        typeOfToken := #EStringFragment.
+        typeOfTokenIfNoEmbeddedExpression := #String.
+    ] ifFalse:[
+        (isCString := (escapeStyleOrNil = 'c')) ifTrue:[
+            typeOfToken := #String
+        ] ifFalse:[
+            (isRString := (escapeStyleOrNil = 'r')) ifTrue:[   
+                typeOfToken := #RegexString
+            ] ifFalse:[
+                (isIString := (escapeStyleOrNil = 'i')) ifTrue:[
+                    typeOfToken := #IStringFragment.
+                    typeOfTokenIfNoEmbeddedExpression := #IString.
+                ] ifFalse:[
+                    escapeStyleOrNil notNil ifTrue:[
+                        self parseError:'unhandled string style'.
+                    ].
+                    typeOfToken := #String
+                ]
+            ]
+        ]
+    ].
 
     string := String uninitializedNew:20.
     len := 20.
@@ -4195,18 +4217,18 @@
                     inString := false
                 ]
             ] ifFalse:[
-                (escapeStyle notNil and:[isRString not]) ifTrue:[
+                (escapeStyleOrNil notNil and:[isRString not]) ifTrue:[
                     ((nextChar == ${) and:[isEString or:[isIString]]) ifTrue:[
                         "/ bail out, to read one expression
                         tokenValue := token := string copyTo:(index - 1).
-                        tokenType := isIString ifTrue:[#IStringFragment] ifFalse:[#EStringFragment].
+                        tokenType := typeOfToken.
                         ^ tokenType
                     ].
                     (nextChar == $\) ifTrue:[
                         peekChar := source peekOrNil.
                         peekChar notNil ifTrue:[
                             source next.
-                            nextChar := self escapeCharacterFor:peekChar escapeStyle:escapeStyle.
+                            nextChar := self escapeCharacterFor:peekChar escapeStyle:escapeStyleOrNil.
                         ]
                     ]
                 ]
@@ -4235,7 +4257,7 @@
     ].
 
     tokenValue := token := string copyTo:(index - 1).
-    tokenType := isRString ifTrue:[#RegexString] ifFalse:[#String].
+    tokenType := typeOfTokenIfNoEmbeddedExpression ? typeOfToken.
     ^ tokenType
 
     "Created: / 22-05-2019 / 20:31:36 / Claus Gittinger"