hg/stx-libcomp: comparison Scanner.st

equal deleted inserted replaced

-:5fdfb29a67e2
+:a9137b4a88bc
 !
 extendedBinarySelectorCharacters
 "return a collection of characters which are optionally allowed in binary selectors"
-"/    ^ '�׷���'.
+"/ ^ '�׷���'.
 ^ String
 with:(Character value:16rB1)  "/ plus-minus
 with:(Character value:16rD7)  "/ times
 with:(Character value:16rB7)  "/ centered dot
 with:(Character value:16rF7)  "/ divide
 with:(Character value:16rAB)  "/ <<
 with:(Character value:16rBB). "/ >>
+"Modified (comment): / 17-11-2016 / 09:22:42 / cg"
 !
 setupActions
-"initialize the scanners actionTables - these are used to dispatch
+"initialize the scanner's actionTables - these are used to dispatch
 into scanner methods as characters are read.
 Compatibility note: in previous versions, these tables used to be kept
 in classVariables, which made reuse hard as subclasses had no easy way of
 defining their own tables. These are now class instance variables."
 "
 Scanner setupActions
 Scanner withAllSubclassesDo:[:cls | cls setupActions ]
 "
-"Modified: / 25-03-2011 / 13:58:59 / cg"
+"Modified: / 17-11-2016 / 09:31:38 / cg"
 ! !
 !Scanner class methodsFor:'instance creation'!
 for:aStringOrStream
 errMsg := 'Invalid character: ''' , ch asString , ''' ', '(' , (v radixPrintStringRadix:16) , ').'.
 ] ifFalse:[
 errMsg := 'Invalid character: ' , (v radixPrintStringRadix:16) , '.'.
 ].
 v > 16r7F ifTrue:[
-errMsg := errMsg , '\\Notice:\  Only 7-bit ascii allowed (for compatibility with other Smalltalk dialects).' withCRs.
+errMsg := errMsg , '
+Notice:
+Only 7-bit ascii allowed (for compatibility with other Smalltalk dialects).
+You can enable some of the special characters via the compiler-settings dialog.'.
 ].
 self syntaxError:errMsg position:tokenPosition to:tokenPosition.
 source next.
 tokenName := token := nil.
 tokenType := #Error.
 ^ #Error
-"Modified: / 22-08-2006 / 14:26:21 / cg"
+"Modified: / 17-11-2016 / 09:37:52 / cg"
 !
 lastTokenLineNumber
 "return the line number of the token which was just read."
 didWarnAboutOldStyleAssignment := true
 ]
 ]
 "Modified: 23.5.1997 / 12:16:48 / cg"
+!
+warnParagraphAt:position
+"warn about �-character in an identifier"
+ignoreWarnings ifFalse:[
+"/ didWarnAboutParagraphInIdentifier ifFalse:[
+parserFlags warnParagraphInIdentifier ifTrue:[
+self
+warning:'�-characters in identifiers/symbols are nonportable'
+doNotShowAgainAction:[ ParserFlags warnParagraphInIdentifier:false ]
+position:position to:position.
+"
+only warn once (per method)
+"
+parserFlags := parserFlags copy.
+parserFlags warnParagraphInIdentifier:false.
+"/ didWarnAboutParagraphInIdentifier := true
+]
+"/ ]
+]
+"Created: / 17-11-2016 / 09:16:22 / cg"
 !
 warnPeriodAt:position
 "warn about a period in an identifier"
 ]
 ].
 nextChar := source peekOrNil.
 (((nextChar == $_) and:[allowUnderscoreInIdentifier])
-or:[(allowDollarInIdentifier and:[nextChar == $$ ])
+or:[ (allowDollarInIdentifier and:[nextChar == $$ ])
-or:[allowNationalCharactersInIdentifier and:[ nextChar notNil and:[nextChar isNationalLetter]]]]) ifTrue:[
+or:[ (nextChar == $� and:[ parserFlags allowParagraphInIdentifier])
+or:[ allowNationalCharactersInIdentifier and:[ nextChar notNil and:[nextChar isNationalLetter]]]]]) ifTrue:[
 pos := source position + 1.
 nextChar == $_ ifTrue:[
 self warnUnderscoreAt:pos.
 ] ifFalse:[
 nextChar == $$ ifTrue:[
 self warnDollarAt:pos.
 ] ifFalse:[
-"/ self warnNationalCharacterAt:pos.
+nextChar == $� ifTrue:[
+self warnParagraphAt:pos.
+] ifFalse:[
+"/ self warnNationalCharacterAt:pos.
+]
 ]
 ].
 ok := true.
 [ok] whileTrue:[
 string := string copyWith:nextChar.
 string := string , source nextAlphaNumericWord.
 nextChar := source peekOrNil.
 ].
 ok := ((nextChar == $_) and:[allowUnderscoreInIdentifier])
 or:[((nextChar == $$ ) and:[allowDollarInIdentifier])
-or:[(nextChar notNil and:[nextChar isNationalLetter]) and:[allowNationalCharactersInIdentifier]]].
+or:[((nextChar == $� ) and:[parserFlags allowParagraphInIdentifier])
+or:[(nextChar notNil and:[nextChar isNationalLetter]) and:[allowNationalCharactersInIdentifier]]]].
 ]
 ].
 ].
 (nextChar == $: and:[scanColonAsKeyword]) ifTrue:[
 tokenType := #Identifier.
 ].
 ^ tokenType
 "Created: / 13-09-1995 / 12:56:42 / claus"
-"Modified: / 24-11-2006 / 10:38:47 / cg"
+"Modified: / 17-11-2016 / 09:19:46 / cg"
 !
 nextMantissa:radix
 "read the mantissa of a radix number"
 (ch isNationalLetter and:[parserFlags allowNationalCharactersInIdentifier]) ifTrue:[
 tok := self nextIdentifier.
 tok notNil ifTrue:[
 ^ tok
 ].
+"/ a nil token means: continue reading
 ] ifFalse:[
-^ self invalidCharacter:ch.
+(ch == $� and:[parserFlags allowParagraphInIdentifier]) ifTrue:[
-]
+tok := self nextIdentifier.
+tok notNil ifTrue:[
+^ tok
+].
+"/ a nil token means: continue reading
+] ifFalse:[
+^ self invalidCharacter:ch.
+].
+].
 ]
 ] loop.
 "Modified: / 13-09-1995 / 12:56:14 / claus"
-"Modified: / 25-03-2011 / 13:56:28 / cg"
 "Modified: / 27-07-2011 / 15:36:53 / Jan Vrany <jan.vrany@fit.cvut.cz>"
+"Modified: / 17-11-2016 / 09:35:39 / cg"
 !
 nextToken:aCharacter
 "return a character token"

changeset 4042	a9137b4a88bc
parent 4037	9be91cda4b4d
child 4083	314f412f15c1