#DOCUMENTATION by cg
class: Scanner
comment/format in: #nextString:
changed:
#nextIdentifier
#xnextString:escapeStyle:
--- a/Scanner.st Mon Jun 03 11:23:03 2019 +0200
+++ b/Scanner.st Mon Jun 03 11:25:16 2019 +0200
@@ -1,3 +1,5 @@
+"{ Encoding: utf8 }"
+
"
COPYRIGHT (c) 1989 by Claus Gittinger
All Rights Reserved
@@ -143,7 +145,7 @@
extendedBinarySelectorCharacters
"return a collection of characters which are optionally allowed in binary selectors"
- "/ ^ '±×·÷«»'.
+ "/ ^ '±×·÷«»'.
^ String
with:(Character value:16rB1) "/ plus-minus
with:(Character value:16rD7) "/ times
@@ -1856,13 +1858,13 @@
!
warnParagraphAt:position
- "warn about §-character in an identifier"
+ "warn about §-character in an identifier"
ignoreWarnings ifFalse:[
"/ didWarnAboutParagraphInIdentifier ifFalse:[
parserFlags warnParagraphInIdentifier ifTrue:[
self
- warning:'§-characters in identifiers/symbols are nonportable'
+ warning:'§-characters in identifiers/symbols are nonportable'
doNotShowAgainAction:[ ParserFlags warnParagraphInIdentifier:false ]
position:position to:position.
"
@@ -2995,9 +2997,10 @@
((nextChar == $')
and:[ (string size == 1)
- and:[ (parserFlags allowCStrings
- or:[parserFlags allowEStrings
- or:[parserFlags allowExtendedSTXSyntax ]]) ]]
+ and:[ ((parserFlags allowCStrings and:[string = 'c'])
+ or:[(parserFlags allowEStrings and:[string = 'e'])
+ or:[(parserFlags allowRStrings and:[string = 'r'])
+ or:[parserFlags allowExtendedSTXSyntax ]]]) ]]
) ifTrue:[
source next.
^ self xnextString:$' escapeStyle:string
@@ -3005,7 +3008,7 @@
(((nextChar == $_) and:[allowUnderscoreInIdentifier])
or:[ (allowDollarInIdentifier and:[nextChar == $$ ])
- or:[ (nextChar == $§ and:[ parserFlags allowParagraphInIdentifier])
+ or:[ (nextChar == $§ and:[ parserFlags allowParagraphInIdentifier])
or:[ allowNationalCharactersInIdentifier and:[ nextChar notNil and:[nextChar isNationalLetter]]]]]) ifTrue:[
pos := source position + 1.
nextChar == $_ ifTrue:[
@@ -3014,7 +3017,7 @@
nextChar == $$ ifTrue:[
self warnDollarAt:pos.
] ifFalse:[
- nextChar == $§ ifTrue:[
+ nextChar == $§ ifTrue:[
self warnParagraphAt:pos.
] ifFalse:[
"/ self warnNationalCharacterAt:pos.
@@ -3034,7 +3037,7 @@
].
ok := ((nextChar == $_) and:[allowUnderscoreInIdentifier])
or:[((nextChar == $$ ) and:[allowDollarInIdentifier])
- or:[((nextChar == $§ ) and:[parserFlags allowParagraphInIdentifier])
+ or:[((nextChar == $§ ) and:[parserFlags allowParagraphInIdentifier])
or:[(nextChar notNil and:[nextChar isNationalLetter]) and:[allowNationalCharactersInIdentifier]]]].
]
].
@@ -3099,7 +3102,7 @@
"Created: / 13-09-1995 / 12:56:42 / claus"
"Modified: / 17-11-2016 / 09:19:46 / cg"
- "Modified: / 23-05-2019 / 10:44:00 / Claus Gittinger"
+ "Modified: / 03-06-2019 / 11:14:55 / Claus Gittinger"
!
nextMantissa:radix
@@ -3440,11 +3443,20 @@
c'a\r\\\nb'
ParserFlags allowCStrings:false.
+
+
+ ParserFlags allowRStrings:true.
+
+ STX regex:
+ r'a+b+'
+
+ ParserFlags allowRStrings:false.
"
"Created: / 01-08-2006 / 14:56:07 / cg"
"Modified: / 22-08-2006 / 14:10:26 / cg"
"Modified: / 22-05-2019 / 20:32:14 / Claus Gittinger"
+ "Modified (comment): / 03-06-2019 / 11:16:00 / Claus Gittinger"
!
nextString:delimiter escapeStyle:escapeStyle
@@ -3729,7 +3741,7 @@
].
"/ a nil token means: continue reading
] ifFalse:[
- (ch == $§ and:[parserFlags allowParagraphInIdentifier]) ifTrue:[
+ (ch == $§ and:[parserFlags allowParagraphInIdentifier]) ifTrue:[
tok := self nextIdentifier.
tok notNil ifTrue:[
^ tok
@@ -3973,21 +3985,21 @@
e'...{ expr1 } ... { exprN }' will generate:
'...%1 ... %N' bindWithArguments:{ expr1 . ... . exprN }
+ 'r' - regex
+
'x' - extended-style escapes:
as yet unsupported
- 'r' - regex
- as yet unsupported
-
"
|nextChar string pos
index "{ Class: SmallInteger }"
len "{ Class: SmallInteger }"
inString peekChar
- isCString isEString|
+ isCString isEString isRString|
isEString := (escapeStyle = 'e').
isCString := (escapeStyle = 'c').
+ isRString := (escapeStyle = 'r').
string := String uninitializedNew:20.
len := 20.
@@ -4015,7 +4027,7 @@
inString := false
]
] ifFalse:[
- escapeStyle notNil ifTrue:[
+ (escapeStyle notNil and:[isRString not]) ifTrue:[
((nextChar == ${) and:[isEString]) ifTrue:[
"/ bail out, to read one expression
tokenValue := token := string copyTo:(index - 1).
@@ -4055,11 +4067,11 @@
].
tokenValue := token := string copyTo:(index - 1).
- tokenType := #String.
+ tokenType := isRString ifTrue:[#RegexString] ifFalse:[#String].
^ tokenType
"Created: / 22-05-2019 / 20:31:36 / Claus Gittinger"
- "Modified: / 23-05-2019 / 10:45:19 / Claus Gittinger"
+ "Modified: / 03-06-2019 / 11:10:24 / Claus Gittinger"
! !
!Scanner::Comment methodsFor:'accessing'!