# HG changeset patch
# User Claus Gittinger <cg@exept.de>
# Date 1559553916 -7200
# Node ID fef0c840421ecfbff0281b7325ca816ba48024ab
# Parent  5cbe584d6f60df4ebc4e9c7cfb2b7b1987132220
#DOCUMENTATION by cg
class: Scanner
comment/format in: #nextString:
changed:
	#nextIdentifier
	#xnextString:escapeStyle:

diff -r 5cbe584d6f60 -r fef0c840421e Scanner.st
--- a/Scanner.st	Mon Jun 03 11:23:03 2019 +0200
+++ b/Scanner.st	Mon Jun 03 11:25:16 2019 +0200
@@ -1,3 +1,5 @@
+"{ Encoding: utf8 }"
+
 "
  COPYRIGHT (c) 1989 by Claus Gittinger
 	      All Rights Reserved
@@ -143,7 +145,7 @@
 extendedBinarySelectorCharacters
     "return a collection of characters which are optionally allowed in binary selectors"
 
-    "/ ^ '±×·÷«»'.
+    "/ ^ '±×·÷«»'.
     ^ String
 	with:(Character value:16rB1)  "/ plus-minus
 	with:(Character value:16rD7)  "/ times
@@ -1856,13 +1858,13 @@
 !
 
 warnParagraphAt:position
-    "warn about §-character in an identifier"
+    "warn about §-character in an identifier"
 
     ignoreWarnings ifFalse:[
 	"/ didWarnAboutParagraphInIdentifier ifFalse:[
 	    parserFlags warnParagraphInIdentifier ifTrue:[
 		self
-		    warning:'§-characters in identifiers/symbols are nonportable'
+		    warning:'§-characters in identifiers/symbols are nonportable'
 		    doNotShowAgainAction:[ ParserFlags warnParagraphInIdentifier:false ]
 		    position:position to:position.
 		"
@@ -2995,9 +2997,10 @@
 
     ((nextChar == $') 
       and:[ (string size == 1)
-      and:[ (parserFlags allowCStrings 
-            or:[parserFlags allowEStrings
-            or:[parserFlags allowExtendedSTXSyntax ]]) ]]
+      and:[ ((parserFlags allowCStrings and:[string = 'c'])
+            or:[(parserFlags allowEStrings and:[string = 'e'])
+            or:[(parserFlags allowRStrings and:[string = 'r'])
+            or:[parserFlags allowExtendedSTXSyntax ]]]) ]]
     ) ifTrue:[
         source next.
         ^ self xnextString:$' escapeStyle:string
@@ -3005,7 +3008,7 @@
 
     (((nextChar == $_) and:[allowUnderscoreInIdentifier])
     or:[ (allowDollarInIdentifier and:[nextChar == $$ ])
-    or:[ (nextChar == $§ and:[ parserFlags allowParagraphInIdentifier])
+    or:[ (nextChar == $§ and:[ parserFlags allowParagraphInIdentifier])
     or:[ allowNationalCharactersInIdentifier and:[ nextChar notNil and:[nextChar isNationalLetter]]]]]) ifTrue:[
         pos := source position + 1.
         nextChar == $_ ifTrue:[
@@ -3014,7 +3017,7 @@
             nextChar == $$ ifTrue:[
                 self warnDollarAt:pos.
             ] ifFalse:[
-                nextChar == $§ ifTrue:[
+                nextChar == $§ ifTrue:[
                     self warnParagraphAt:pos.
                 ] ifFalse:[
                     "/ self warnNationalCharacterAt:pos.
@@ -3034,7 +3037,7 @@
                 ].
                 ok := ((nextChar == $_) and:[allowUnderscoreInIdentifier])
                       or:[((nextChar == $$ ) and:[allowDollarInIdentifier])
-                      or:[((nextChar == $§ ) and:[parserFlags allowParagraphInIdentifier])
+                      or:[((nextChar == $§ ) and:[parserFlags allowParagraphInIdentifier])
                       or:[(nextChar notNil and:[nextChar isNationalLetter]) and:[allowNationalCharactersInIdentifier]]]].
             ]
         ].
@@ -3099,7 +3102,7 @@
 
     "Created: / 13-09-1995 / 12:56:42 / claus"
     "Modified: / 17-11-2016 / 09:19:46 / cg"
-    "Modified: / 23-05-2019 / 10:44:00 / Claus Gittinger"
+    "Modified: / 03-06-2019 / 11:14:55 / Claus Gittinger"
 !
 
 nextMantissa:radix
@@ -3440,11 +3443,20 @@
         c'a\r\\\nb'
      
      ParserFlags allowCStrings:false.
+
+
+     ParserFlags allowRStrings:true.
+
+     STX regex:
+        r'a+b+'
+
+     ParserFlags allowRStrings:false.
     "
 
     "Created: / 01-08-2006 / 14:56:07 / cg"
     "Modified: / 22-08-2006 / 14:10:26 / cg"
     "Modified: / 22-05-2019 / 20:32:14 / Claus Gittinger"
+    "Modified (comment): / 03-06-2019 / 11:16:00 / Claus Gittinger"
 !
 
 nextString:delimiter escapeStyle:escapeStyle
@@ -3729,7 +3741,7 @@
 		].
 		"/ a nil token means: continue reading
 	    ] ifFalse:[
-		(ch == $§ and:[parserFlags allowParagraphInIdentifier]) ifTrue:[
+		(ch == $§ and:[parserFlags allowParagraphInIdentifier]) ifTrue:[
 		    tok := self nextIdentifier.
 		    tok notNil ifTrue:[
 			^ tok
@@ -3973,21 +3985,21 @@
                         e'...{ expr1 } ... { exprN }' will generate:
                         '...%1 ... %N' bindWithArguments:{ expr1 . ... . exprN }
                         
+        'r' - regex
+
         'x' - extended-style escapes:
                         as yet unsupported
-        'r' - regex
-                        as yet unsupported
-
     "
 
     |nextChar string pos
      index "{ Class: SmallInteger }"
      len   "{ Class: SmallInteger }"
      inString peekChar
-     isCString isEString|
+     isCString isEString isRString|
 
     isEString := (escapeStyle = 'e').
     isCString := (escapeStyle = 'c').
+    isRString := (escapeStyle = 'r').
     
     string := String uninitializedNew:20.
     len := 20.
@@ -4015,7 +4027,7 @@
                     inString := false
                 ]
             ] ifFalse:[
-                escapeStyle notNil ifTrue:[
+                (escapeStyle notNil and:[isRString not]) ifTrue:[
                     ((nextChar == ${) and:[isEString]) ifTrue:[
                         "/ bail out, to read one expression
                         tokenValue := token := string copyTo:(index - 1).
@@ -4055,11 +4067,11 @@
     ].
 
     tokenValue := token := string copyTo:(index - 1).
-    tokenType := #String.
+    tokenType := isRString ifTrue:[#RegexString] ifFalse:[#String].
     ^ tokenType
 
     "Created: / 22-05-2019 / 20:31:36 / Claus Gittinger"
-    "Modified: / 23-05-2019 / 10:45:19 / Claus Gittinger"
+    "Modified: / 03-06-2019 / 11:10:24 / Claus Gittinger"
 ! !
 
 !Scanner::Comment methodsFor:'accessing'!