--- a/Scanner.st Wed May 22 21:17:27 2019 +0200
+++ b/Scanner.st Wed May 22 21:17:42 2019 +0200
@@ -2566,6 +2566,16 @@
!Scanner methodsFor:'reading next token'!
+continueEscapedString
+ "after escaping out of a string with \{,
+ an expression was read by the parser and it found the terminating $}.
+ It calls this to continue reading the string..."
+
+ ^ self xnextString:$' escapeStyle:'e'
+
+ "Created: / 22-05-2019 / 20:24:51 / Claus Gittinger"
+!
+
nextAssignmentArrow
"return a left-arrow"
@@ -2945,7 +2955,8 @@
((nextChar == $')
and:[ (string size == 1)
and:[ parserFlags allowCStrings ]]) ifTrue:[
- ^ self nextString:$' escapeStyle:string
+ source next.
+ ^ self xnextString:$' escapeStyle:string
].
(((nextChar == $_) and:[allowUnderscoreInIdentifier])
@@ -3044,7 +3055,7 @@
"Created: / 13-09-1995 / 12:56:42 / claus"
"Modified: / 17-11-2016 / 09:19:46 / cg"
- "Modified: / 11-02-2019 / 12:25:36 / Claus Gittinger"
+ "Modified: / 22-05-2019 / 20:32:04 / Claus Gittinger"
!
nextMantissa:radix
@@ -3365,7 +3376,8 @@
nextString:delimiter
"a quote has been scanned; scan the string (caring for doubled quotes)"
- ^ self nextString:delimiter escapeStyle:nil
+ source next.
+ ^ self xnextString:delimiter escapeStyle:nil
"
old style ST80 string (no escapes):
@@ -3386,15 +3398,21 @@
"Created: / 01-08-2006 / 14:56:07 / cg"
"Modified: / 22-08-2006 / 14:10:26 / cg"
- "Modified (comment): / 08-02-2019 / 22:29:30 / Claus Gittinger"
+ "Modified: / 22-05-2019 / 20:32:14 / Claus Gittinger"
!
nextString:delimiter escapeStyle:escapeStyle
"a quote has been scanned; scan the string (caring for doubled quotes).
escapeStyle may be:
nil - old style ST80 strings (no character escapes)
+ ot a single char string:
'c' - C-style escapes:
\n,\t,\r,\b,\xXX,\uXXXX,\UXXXXXX,
+
+ 'e' - C-style plus embedded escapes:
+ e'...\{ expr1 } ... \{ exprN }' will generate:
+ '...%1 ... %N' bindWithArguments:{ expr1 . ... . exprN }
+
'x' - extended-style escapes:
as yet unsupported
'r' - regex
@@ -3402,77 +3420,11 @@
"
- |nextChar string pos
- index "{ Class: SmallInteger }"
- len "{ Class: SmallInteger }"
- inString peekChar|
-
- string := String uninitializedNew:20.
- len := 20.
- index := 1.
- pos := source position + 1.
source next.
- nextChar := source next.
- inString := true.
-
- [inString] whileTrue:[
- nextChar isNil ifTrue:[
- self syntaxError:'unexpected end-of-input in String'
- position:pos to:(source position).
- self markStringFrom:pos to:source position.
- token := nil.
- tokenType := #EOF.
- ^ tokenType
- ].
- (nextChar == Character cr) ifTrue:[
- lineNr := lineNr + 1
- ] ifFalse:[
- (nextChar == delimiter) ifTrue:[
- (source peekOrNil == delimiter) ifTrue:[
- source next
- ] ifFalse:[
- inString := false
- ]
- ] ifFalse:[
- (parserFlags allowExtendedSTXSyntax == true or:[escapeStyle notNil]) ifTrue:[
- (nextChar == $\) ifTrue:[
- peekChar := source peekOrNil.
- peekChar notNil ifTrue:[
- source next.
- nextChar := self escapeCharacterFor:peekChar escapeStyle:escapeStyle.
- ]
- ]
- ]
- ].
- ].
- inString ifTrue:[
- nextChar notNil ifTrue:[
- nextChar codePoint > 255 ifTrue:[
- string bitsPerCharacter < nextChar bitsPerCharacter ifTrue:[
- nextChar codePoint > 16rFFFF ifTrue:[
- string := string asUnicode32String
- ] ifFalse:[
- string := string asUnicode16String.
- ].
- ].
- ].
- string at:index put:nextChar.
- (index == len) ifTrue:[
- string := string , (string species new:len).
- len := len * 2
- ].
- index := index + 1.
- ].
- nextChar := source next
- ]
- ].
-
- tokenValue := token := string copyTo:(index - 1).
- tokenType := #String.
- ^ tokenType
+ ^ self xnextString:delimiter escapeStyle:escapeStyle
"Created: / 08-02-2019 / 19:07:57 / Claus Gittinger"
- "Modified (comment): / 11-02-2019 / 12:28:24 / Claus Gittinger"
+ "Modified: / 22-05-2019 / 20:33:04 / Claus Gittinger"
!
nextSymbolAfterHash
@@ -3960,6 +3912,103 @@
hereChar := source nextPeek.
].
lineNr := lineNr + 1.
+!
+
+xnextString:delimiter escapeStyle:escapeStyle
+ "a quote has been scanned; scan the string (caring for doubled quotes).
+ escapeStyle may be:
+ nil - old style ST80 strings (no character escapes)
+ ot a single char string:
+ 'c' - C-style escapes:
+ \n,\t,\r,\b,\xXX,\uXXXX,\UXXXXXX,
+
+ 'e' - C-style plus embedded escapes:
+ e'...\{ expr1 } ... \{ exprN }' will generate:
+ '...%1 ... %N' bindWithArguments:{ expr1 . ... . exprN }
+
+ 'x' - extended-style escapes:
+ as yet unsupported
+ 'r' - regex
+ as yet unsupported
+
+ "
+
+ |nextChar string pos
+ index "{ Class: SmallInteger }"
+ len "{ Class: SmallInteger }"
+ inString peekChar|
+
+ string := String uninitializedNew:20.
+ len := 20.
+ index := 1.
+ pos := source position.
+ nextChar := source next.
+ inString := true.
+
+ [inString] whileTrue:[
+ nextChar isNil ifTrue:[
+ self syntaxError:'unexpected end-of-input in String'
+ position:pos to:(source position).
+ self markStringFrom:pos to:source position.
+ token := nil.
+ tokenType := #EOF.
+ ^ tokenType
+ ].
+ (nextChar == Character cr) ifTrue:[
+ lineNr := lineNr + 1
+ ] ifFalse:[
+ (nextChar == delimiter) ifTrue:[
+ (source peekOrNil == delimiter) ifTrue:[
+ source next
+ ] ifFalse:[
+ inString := false
+ ]
+ ] ifFalse:[
+ (parserFlags allowExtendedSTXSyntax == true or:[escapeStyle notNil]) ifTrue:[
+ (nextChar == $\) ifTrue:[
+ peekChar := source peekOrNil.
+ peekChar notNil ifTrue:[
+ source next.
+ (peekChar == ${ and:[escapeStyle = 'e']) ifTrue:[
+ "/ bail out, to read one expression
+ tokenValue := token := string copyTo:(index - 1).
+ tokenType := #StringFragment.
+ ^ tokenType
+ ] ifFalse:[
+ nextChar := self escapeCharacterFor:peekChar escapeStyle:escapeStyle.
+ ]
+ ]
+ ]
+ ]
+ ].
+ ].
+ inString ifTrue:[
+ nextChar notNil ifTrue:[
+ nextChar codePoint > 255 ifTrue:[
+ string bitsPerCharacter < nextChar bitsPerCharacter ifTrue:[
+ nextChar codePoint > 16rFFFF ifTrue:[
+ string := string asUnicode32String
+ ] ifFalse:[
+ string := string asUnicode16String.
+ ].
+ ].
+ ].
+ string at:index put:nextChar.
+ (index == len) ifTrue:[
+ string := string , (string species new:len).
+ len := len * 2
+ ].
+ index := index + 1.
+ ].
+ nextChar := source next
+ ]
+ ].
+
+ tokenValue := token := string copyTo:(index - 1).
+ tokenType := #String.
+ ^ tokenType
+
+ "Created: / 22-05-2019 / 20:31:36 / Claus Gittinger"
! !
!Scanner::Comment methodsFor:'accessing'!