hg/stx-libcomp: comparison Scanner.st

equal deleted inserted replaced

-:87133d993838
+:49123df91ffa
-"{ Encoding: utf8 }"
 "
 COPYRIGHT (c) 1989 by Claus Gittinger
 	      All Rights Reserved
 This software is furnished under a license and may be used
 !
 extendedBinarySelectorCharacters
 "return a collection of characters which are optionally allowed in binary selectors"
-"/ ^ '±×·÷«»'.
+"/ ^ '�׷���'.
 ^ String
 	with:(Character value:16rB1)  "/ plus-minus
 	with:(Character value:16rD7)  "/ times
 	with:(Character value:16rB7)  "/ centered dot
 	with:(Character value:16rF7)  "/ divide
 invalidCharacter:ch
 |errMsg v|
 v := ch codePoint.
 ch isPrintable ifTrue:[
-	errMsg := 'Invalid character: ''' , ch asString , ''' ', '(' , (v radixPrintStringRadix:16) , ').'.
+errMsg := 'Invalid character: ''' , ch asString , ''' ', '(' , (v radixPrintStringRadix:16) , ').'.
 ] ifFalse:[
-	errMsg := 'Invalid character: ' , (v radixPrintStringRadix:16) , '.'.
+errMsg := 'Invalid character: ' , (v radixPrintStringRadix:16) , '.'.
 ].
 v > 16r7F ifTrue:[
-	errMsg := errMsg , '
+errMsg := errMsg , '
 Notice:
 Only 7-bit ascii allowed (for compatibility with other Smalltalk dialects).
 You can enable some of the special characters via the compiler-settings dialog.'.
 ].
 "Modified: 23.5.1997 / 12:16:48 / cg"
 !
 warnParagraphAt:position
-"warn about §-character in an identifier"
+"warn about �-character in an identifier"
 ignoreWarnings ifFalse:[
 	"/ didWarnAboutParagraphInIdentifier ifFalse:[
 	    parserFlags warnParagraphInIdentifier ifTrue:[
 		self
-		    warning:'§-characters in identifiers/symbols are nonportable'
+		    warning:'�-characters in identifiers/symbols are nonportable'
 		    doNotShowAgainAction:[ ParserFlags warnParagraphInIdentifier:false ]
 		    position:position to:position.
 		"
 		 only warn once (per method)
 		"
 ^ self invalidCharacter:source peek.
 !
 nextHash
 "a # has been read - return either
-	a symbol,
+a symbol,
-	HashLeftParen     (for '#('),
+HashLeftParen     (for '#('),
-	HashLeftBrack     (for '#['),
+HashLeftBrack     (for '#['),
-	HashLeftBrace     (for '#{'  and AllowQualifiedNames)
+HashLeftBrace     (for '#{'  and AllowQualifiedNames)
-	HashHashLeftParen (for '##(' and AllowDolphinExtensions)
+HashHashLeftParen (for '##(' and AllowDolphinExtensions)
-	HashHashLeftBrack (for '##[' )
+HashHashLeftBrack (for '##[' )
-	HashHash          (for '##' )
+HashHash          (for '##' )
 extended syntax (scheme-style literal arrays):
-	HashTypedArrayParen   (for '#u8(', '#s8(' , '#u16(' ...)
+(requires ParserFlags allowSTXExtendedArrayLiterals:true)
-	type in tokenValue: u1, u8, u16, u32, u64, s8, s16, s32, s64,
+HashTypedArrayParen   (for '#u8(', '#s8(' , '#u16(' ...)
-			    f16, f32, f64, f, d, b, B
+type in tokenValue: u1, u8, u16, u32, u64, s8, s16, s32, s64,
+f16, f32, f64, f, d, b, B
 "
 |nextChar string allowUnderscoreInIdentifier|
 allowUnderscoreInIdentifier := parserFlags allowUnderscoreInIdentifier.
 nextChar := source nextPeek.
 nextChar notNil ifTrue:[
-	(nextChar == $( ) ifTrue:[
+(nextChar == $( ) ifTrue:[
-	    source next.
+source next.
-	    token := '#('.
+token := '#('.
-	    tokenType := #HashLeftParen.
+tokenType := #HashLeftParen.
-	    ^ tokenType
+^ tokenType
-	].
+].
-	(nextChar == $[ ) ifTrue:[
+(nextChar == $[ ) ifTrue:[
-	    "ST-80 & ST/X support Constant ByteArrays as #[...]"
+"ST-80 & ST/X support Constant ByteArrays as #[...]
-	    source next.
+now all Smalltalk dialects do."
-	    token := '#['.
+source next.
-	    tokenType := #HashLeftBrack.
+token := '#['.
-	    ^ tokenType
+tokenType := #HashLeftBrack.
-	].
+^ tokenType
+].
-	(nextChar == ${ ) ifTrue:[
-	    " #{ ... } is one of:
+(nextChar == ${ ) ifTrue:[
-		#{ Foo.Bar.Baz }            VW3 and later qualified name
+" #{ ... } is one of:
-		#{ xx-xx-xx-xx-...-xx }     StAgents UUID
+#{ Foo.Bar.Baz }            VW3 and later qualified name
-		#{ URL }                    url object qualifier
+#{ xx-xx-xx-xx-...-xx }     StAgents UUID
-	    "
+#{ URL }                    url object qualifier
-	    source next.
+#{ key: value ... }         inline literal object
-	    token := '#{'.
+"
-	    tokenType := #HashLeftBrace.
+source next.
-	    ^ tokenType
+token := '#{'.
-	].
+tokenType := #HashLeftBrace.
+^ tokenType
-	(nextChar == $' ) ifTrue:[
+].
-	    "ST-80 and ST/X support arbitrary symbols as #'...'"
-	    self nextString:nextChar.
+(nextChar == $' ) ifTrue:[
-	    self markSymbolFrom:tokenPosition to:(source position).
+"ST-80 and ST/X support arbitrary symbols as #'...'
-	    tokenType == #EOF ifFalse:[
+now all dialects do"
-		tokenValue isWideString ifTrue:[
+self nextString:nextChar.
-		    self syntaxError:'symbols which require 2-byte characters are not (yet) allowed'
+self markSymbolFrom:tokenPosition to:(source position).
-			    position:tokenPosition to:(source position).
+tokenType == #EOF ifFalse:[
-		].
+tokenValue isWideString ifTrue:[
-		tokenValue := token := tokenValue asSymbol.
+self syntaxError:'symbols which require 2-byte characters are not (yet) allowed'
-		tokenType := #Symbol.
+position:tokenPosition to:(source position).
-	    ].
+].
-	    ^ tokenType
+tokenValue := token := tokenValue asSymbol.
-	].
+tokenType := #Symbol.
+].
-	(nextChar == $#) ifTrue:[
+^ tokenType
-	    nextChar := source nextPeek.
+].
-	    nextChar == $( ifTrue:[
-		parserFlags allowDolphinExtensions == true ifTrue:[
+(nextChar == $#) ifTrue:[
-		    "dolphin does computed literals as ##( ... )"
+nextChar := source nextPeek.
-		    source next.
+nextChar == $( ifTrue:[
-		    token := '##('.
+parserFlags allowDolphinExtensions == true ifTrue:[
-		    tokenType := #HashHashLeftParen.
+"dolphin does computed literals as ##( expression )"
-		    ^ tokenType
+source next.
-		].
+token := '##('.
-	    ].
+tokenType := #HashHashLeftParen.
+^ tokenType
-	    nextChar == $[ ifTrue:[
+].
-		source next.
+].
-		token := '##['.
-		tokenType := #HashHashLeftBrack.
+nextChar == $[ ifTrue:[
-		^ tokenType
+source next.
-	    ].
+token := '##['.
+tokenType := #HashHashLeftBrack.
-	    parserFlags allowVisualAgeESSymbolLiterals == true ifTrue:[
+^ tokenType
-		(self nextSymbolAfterHash) notNil ifTrue:[
+].
-		    tokenType := #ESSymbol.
-		    ^ #ESSymbol
+parserFlags allowVisualAgeESSymbolLiterals == true ifTrue:[
-		].
+"V'age has special ESsymbols as ##name or ##'name'"
-		(nextChar == $') ifTrue:[
+(self nextSymbolAfterHash) notNil ifTrue:[
-		    source next.
+tokenType := #ESSymbol.
-		    self nextString:nextChar.
+^ #ESSymbol
-		    tokenType := #ESSymbol.
+].
-		    ^ #ESSymbol
+(nextChar == $') ifTrue:[
-		].
+source next.
-	    ].
+self nextString:nextChar.
+tokenType := #ESSymbol.
-	    token := '##'.
+^ #ESSymbol
-	    tokenType := #HashHash.
+].
-	    ^ tokenType
+].
-	].
+token := '##'.
-	parserFlags allowSTXExtendedArrayLiterals ifTrue:[
+tokenType := #HashHash.
-	    "/ scheme-style typed literal array extension
+^ tokenType
-	    ('usfdbB' includes:nextChar) ifTrue:[
+].
-		|prefix|
+"/ scheme-style typed literal arrays:
-		prefix := String with:nextChar.
+"/    #uXX( ... )  XX = { 1, 8, 16, 32, 64 } - bit, uint8, uint16, uint32 or uint64 array
-		nextChar := source nextPeek.
+"/    #iXX( ... )  XX = { 8, 16, 32, 64 }    - int8, int16, int32 or int64 array
-		[nextChar notNil and:[nextChar isDigit]] whileTrue:[
+"/    #fXX( ... )  XX = { 16, 32, 64 }       - IEEE half, single or double array
-		    prefix := prefix copyWith:nextChar.
+"/    #f( ... ) - IEEE single float array
-		    nextChar := source nextPeek.
+"/    #d( ... ) - IEEE double array
-		].
+"/    #b( ... ) - bit array
-		nextChar == $( ifTrue:[
+"/    #B( ... ) - boolean array
-		    source next.
+('usfdbB' includes:nextChar) ifTrue:[
-		    (
+|prefix|
-			#( 'f' 'd' 'b' 'B'
-			   'u1' 'u8' 'u16' 'u32' 'u64'
+"/ collec tuntil we know what we get...
-			   's8' 's16' 's32' 's64'
+prefix := String with:nextChar.
-			   'f16' 'f32' 'f64'
+nextChar := source nextPeek.
-			) includes:prefix
+[nextChar notNil and:[nextChar isDigit]] whileTrue:[
-		    ) ifTrue:[
+prefix := prefix copyWith:nextChar.
-			tokenType := #HashTypedArrayParen.
+nextChar := source nextPeek.
-			tokenValue := prefix asSymbol.
+].
-			^ tokenType
+nextChar == $( ifTrue:[
-		    ].
+parserFlags allowSTXExtendedArrayLiterals ifFalse:[
-		    self parseError:'unsupported literal array type: ',prefix.
+self parseError:c'Non-Standard ST/X extension used: #XXX( .. ) unboxed array literal.\nPlease enable "allowSTXExtendedArrayLiterals" in the ParserFlags\n\nNotice: this is currently not supported by stc'
-		    tokenType := #HashLeftParen.
+position:tokenPosition to:source position
-		    ^ #HashLeftParen
+].
-		].
+source next.
-		^ self nextSymbolAfterHash:prefix.
+(
-	    ].
+#( 'f' 'd' 'b' 'B'
-	].
+'u1' 'u8' 'u16' 'u32' 'u64'
+'s8' 's16' 's32' 's64'
-	(self nextSymbolAfterHash) notNil ifTrue:[
+'f16' 'f32' 'f64'
-	    ^ #Symbol
+) includes:prefix
-	].
+) ifTrue:[
+tokenType := #HashTypedArrayParen.
-	(self isSpecialOrExtendedSpecialCharacter:nextChar) ifTrue:[
+tokenValue := prefix asSymbol.
-	    string := source next asString.
+^ tokenType
-	    nextChar := source peek.
+].
-	    nextChar notNil ifTrue:[
+self parseError:'unsupported literal array type: ',prefix.
-		(self isSpecialOrExtendedSpecialCharacter:nextChar) ifTrue:[
+tokenType := #HashLeftParen.
-		    source next.
+^ #HashLeftParen
-		    string := string copyWith:nextChar
+].
-		]
+^ self nextSymbolAfterHash:prefix.
-	    ].
+].
-	    self markSymbolFrom:tokenPosition to:(source position).
-	    tokenValue := token := string asSymbol.
+(self nextSymbolAfterHash) notNil ifTrue:[
-	    tokenType := #Symbol.
+^ #Symbol
-	    ^ tokenType
+].
-	]
+(self isSpecialOrExtendedSpecialCharacter:nextChar) ifTrue:[
+string := source next asString.
+nextChar := source peek.
+nextChar notNil ifTrue:[
+(self isSpecialOrExtendedSpecialCharacter:nextChar) ifTrue:[
+source next.
+string := string copyWith:nextChar
+]
+].
+self markSymbolFrom:tokenPosition to:(source position).
+tokenValue := token := string asSymbol.
+tokenType := #Symbol.
+^ tokenType
+]
 ].
 "this allows hash to be used as binop -
 I don't know, if this is correct ..."
 tokenName := token := '#'.
 tokenType := #BinaryOperator.
 ^ tokenType
 "Modified: / 01-08-2006 / 14:57:19 / cg"
 "Modified (format): / 30-09-2011 / 12:23:04 / cg"
+"Modified: / 30-05-2019 / 19:06:36 / Claus Gittinger"
 !
 nextId
 "no longer used here - remains for backwardCompatibility for
 subclass users ... (sigh)"
 ^ self xnextString:$' escapeStyle:string
 ].
 (((nextChar == $_) and:[allowUnderscoreInIdentifier])
 or:[ (allowDollarInIdentifier and:[nextChar == $$ ])
-or:[ (nextChar == $§ and:[ parserFlags allowParagraphInIdentifier])
+or:[ (nextChar == $� and:[ parserFlags allowParagraphInIdentifier])
 or:[ allowNationalCharactersInIdentifier and:[ nextChar notNil and:[nextChar isNationalLetter]]]]]) ifTrue:[
 pos := source position + 1.
 nextChar == $_ ifTrue:[
 self warnUnderscoreAt:pos.
 ] ifFalse:[
 nextChar == $$ ifTrue:[
 self warnDollarAt:pos.
 ] ifFalse:[
-nextChar == $§ ifTrue:[
+nextChar == $� ifTrue:[
 self warnParagraphAt:pos.
 ] ifFalse:[
 "/ self warnNationalCharacterAt:pos.
 ]
 ]
 string := string , source nextAlphaNumericWord.
 nextChar := source peekOrNil.
 ].
 ok := ((nextChar == $_) and:[allowUnderscoreInIdentifier])
 or:[((nextChar == $$ ) and:[allowDollarInIdentifier])
-or:[((nextChar == $§ ) and:[parserFlags allowParagraphInIdentifier])
+or:[((nextChar == $� ) and:[parserFlags allowParagraphInIdentifier])
 or:[(nextChar notNil and:[nextChar isNationalLetter]) and:[allowNationalCharactersInIdentifier]]]].
 ]
 ].
 ].
 		tok notNil ifTrue:[
 		    ^ tok
 		].
 		"/ a nil token means: continue reading
 	    ] ifFalse:[
-		(ch == $§ and:[parserFlags allowParagraphInIdentifier]) ifTrue:[
+		(ch == $� and:[parserFlags allowParagraphInIdentifier]) ifTrue:[
 		    tok := self nextIdentifier.
 		    tok notNil ifTrue:[
 			^ tok
 		    ].
 		    "/ a nil token means: continue reading

changeset 4426	49123df91ffa
parent 4422	a06688034985
child 4432	fef0c840421e