"

 COPYRIGHT (c) 1989 by Claus Gittinger

 This software is furnished under a license and may be used

 only in accordance with the terms of that license and with the

 inclusion of the above copyright notice.   This software may not

 be provided or otherwise made available to, or used by, any

 other person.  No title to or ownership of the software is

 hereby transferred.

"

"{ Package: 'stx:libcomp' }"

"{ NameSpace: Smalltalk }"

Object subclass:#Scanner

	instanceVariableNames:'typeArray actionArray source lineNr token tokenType tokenPosition

		tokenValue tokenName tokenLineNr tokenLastEndPosition hereChar

		peekChar peekChar2 requestor exitBlock errorFlag ignoreErrors

		ignoreWarnings saveComments currentComments collectedSource

		scanColonAsKeyword outStream outCol inArrayLiteral lastDirective

		parserFlags didWarnAboutSTXSpecialComment

		didWarnAboutUnderscoreInIdentifier didWarnAboutOldStyleAssignment

		didWarnAboutDollarInIdentifier didWarnAboutPeriodInSymbol

		unicodeActions'

	classVariableNames:'DefaultTypeArray DefaultActionArray EmptySourceNotificationSignal

		DefaultUnicodeActions'

	poolDictionaries:''

	category:'System-Compiler'

!

Scanner class instanceVariableNames:'TypeArray ActionArray UnicodeActions'

"

 No other class instance variables are inherited by this class.

"

!

Object subclass:#Comment

	instanceVariableNames:'commentType commentString startPosition endPosition'

	classVariableNames:''

	poolDictionaries:''

	privateIn:Scanner

!

Object subclass:#Directive

	instanceVariableNames:''

	classVariableNames:''

	poolDictionaries:''

	privateIn:Scanner

!

Scanner::Directive subclass:#ClassDirective

	instanceVariableNames:'className'

	classVariableNames:''

	poolDictionaries:''

	privateIn:Scanner::Directive

!

Scanner::Directive::ClassDirective subclass:#ClassHintDirective

	instanceVariableNames:''

	classVariableNames:''

	poolDictionaries:''

	privateIn:Scanner::Directive

!

!Scanner class methodsFor:'documentation'!

bugs

"

   array constant containing keywords as in:

   is scanned as 4-element array containing ( #foo: #bar: #fee: #baz: )

   this MUST be fixed.

   workaround:

"

!

copyright

"

 COPYRIGHT (c) 1989 by Claus Gittinger

 This software is furnished under a license and may be used

 only in accordance with the terms of that license and with the

 inclusion of the above copyright notice.   This software may not

 be provided or otherwise made available to, or used by, any

 other person.  No title to or ownership of the software is

 hereby transferred.

"

!

documentation

"

    Scanner reads from a stream and returns individual smalltalk tokens

    Its main method is #nextToken, which reads and returns the next token

    instance variables as a side effect.

    Extensions:

    [author:]

    [see also:]

"

! !

!Scanner class methodsFor:'initialization'!

binarySelectorCharacters

    "return a collection of characters which are allowed in binary selectors"

    ^ Method binarySelectorCharacters.

!

extendedBinarySelectorCharacters

    "return a collection of characters which are optionally allowed in binary selectors"

    "/ ^ '±×·÷«»'.

    ^ String

    "Modified (comment): / 17-11-2016 / 09:22:42 / cg"

!

setupActions

    "initialize the scanner's actionTables - these are used to dispatch

     into scanner methods as characters are read.

     Compatibility note: in previous versions, these tables used to be kept

     in classVariables, which made reuse hard as subclasses had no easy way of

     defining their own tables. These are now class instance variables."

    |block actionArray typeArray unicodeActions unicodeTypes|

    actionArray := Array new:256.

    typeArray := Array new:256.

    unicodeActions := Dictionary new.

    unicodeTypes := Dictionary new.

    "/ TODO: later versions should be configurable w.r.t separators.

    "/ #(9 10 12 13 26 32) do: [:i | TypeArray at:(i+1) put: #separator].

    block := [:s :char | s nextNumber].

    actionArray from:($0 codePoint) to:($9 codePoint) put:block.

    block := [:s :char | s nextSpecial].

    self binarySelectorCharacters do:[:binop |

    ].

    block := [:s :char | s nextExtendedSpecial:char].

    self extendedBinarySelectorCharacters do:[:binop |

    ].

    "/ that one is a special case (both binarySelector AND syntax).

    typeArray at:($| codePoint) put:nil.

    block := [:s :char | s nextToken:char].

    ';.^|()[]{}' do:[:ch |

    ].

    block := [:s :char | s nextIdentifier].

    actionArray from:($a codePoint) to:($z codePoint) put:block.

    actionArray from:($A codePoint) to:($Z codePoint) put:block.

    "kludge: action is characterToken, but type is special"

    typeArray at:($| codePoint) put:#special.

    "kludge: action is nextColonOrAssign, but type is special"

    typeArray at:($: codePoint) put:#special.

    actionArray at:($' codePoint) put:[:s :char | s nextString:char].

    actionArray at:($$ codePoint) put:[:s :char | s nextCharacter].

    actionArray at:($# codePoint) put:[:s :char | s nextHash].

    actionArray at:($!! codePoint) put:[:s :char | s nextExcla].

    actionArray at:($% codePoint) put:[:s :char | s nextPrimitive].

    actionArray at:($: codePoint) put:[:s :char | s nextColonOrAssign].

    actionArray at:($_ codePoint) put:[:s :char | s nextUnderline].

    unicodeActions at:2190 "<- left arrow" put:[:s :char | s nextAssignmentArrow].

    ActionArray := DefaultActionArray := actionArray.

    TypeArray := DefaultTypeArray := typeArray.

    UnicodeActions := DefaultUnicodeActions := unicodeActions.

    "

     Scanner setupActions

     Scanner withAllSubclassesDo:[:cls | cls setupActions ]

    "

    "Modified: / 02-07-2017 / 01:11:27 / cg"

! !

!Scanner class methodsFor:'instance creation'!

for:aStringOrStream

    "create & return a new scanner reading from aStringOrStream"

    ^ self basicNew initializeFor:aStringOrStream

    "Modified: 23.5.1997 / 12:08:42 / cg"

!

    "create & return a new scanner"

    ^ self basicNew initialize.

    "Modified: / 23.5.1997 / 12:08:42 / cg"

    "Created: / 26.5.1999 / 12:02:16 / stefan"

! !

!Scanner class methodsFor:'Signal constants'!

doNotShowCompilerWarningAgainActionQuery

    ^ DoNotShowCompilerWarningAgainActionQuery

!

emptySourceNotificationSignal

    ^ EmptySourceNotificationSignal

    "Created: / 16.5.1998 / 15:55:14 / cg"

! !

!Scanner class methodsFor:'accessing'!

actionArray

    ActionArray isNil ifTrue:[

    ].

    ^ ActionArray ? DefaultActionArray

!

flushActionArray

    ActionArray := DefaultActionArray := nil.

    TypeArray := DefaultTypeArray := nil.

    "

     Scanner flushActionArray

     Parser flushActionArray

     ByteCodeCompiler flushActionArray

     Explainer flushActionArray

    "

!

typeArray

    TypeArray isNil ifTrue:[

    ].

    ^ TypeArray ? DefaultTypeArray

!

unicodeActions

    UnicodeActions isNil ifTrue:[

    ].

    ^ UnicodeActions ? DefaultUnicodeActions

    "Created: / 25-03-2011 / 13:57:34 / cg"

! !

!Scanner class methodsFor:'class initialization'!

initialize

    "initialize the classes defaults. Typically, these are changed

     later in the 'private.rc' file."

    EmptySourceNotificationSignal isNil ifTrue:[

    ].

    "

     self initialize

    "

    "Modified: / 16.5.1998 / 15:55:41 / cg"

! !

!Scanner class methodsFor:'defaults'!

allowDollarInIdentifier

    "return true, if $-characters are allowed in identifiers.

     Notice, that dollars are NEVER allowed as the first character in an identifier."

    ^ ParserFlags allowDollarInIdentifier

    "Created: 7.9.1997 / 01:32:18 / cg"

    "Modified: 7.9.1997 / 01:39:44 / cg"

!

allowDollarInIdentifier:aBoolean

    "this allows turning on/off $-characters in identifiers.

     Notice, that dollars are NEVER allowed as the first character in an identifier.

     If turned off (the default), dollars are not allowed in identifiers,

     but instead are scanned as character-constant prefix.

     If turned on, dollars are in identifiers are allowed, while extra

     dollars are still scanned as constant character prefix.

     If you have to fileIn old VW-Vsn2.x classes, turn this off

     before filing them in; i.e.:

    ParserFlags allowDollarInIdentifier:aBoolean.

    "Created: 7.9.1997 / 01:34:49 / cg"

    "Modified: 7.9.1997 / 01:39:30 / cg"

!

allowDolphinExtensions

    "return true, if ##(..) computed literal Arrays are allowed"

    ^ ParserFlags allowDolphinExtensions

!

allowDolphinExtensions:aBoolean

    "this allows turning on/off support for computed literal Arrays ##(..) as in dolphin.

     If you want to fileIn Dolphin classes, enable this with:

    ParserFlags allowDolphinExtensions:aBoolean.

    "

     self allowDolphinExtensions:true

     self allowDolphinExtensions:false

    "

!

allowLiteralNameSpaceSymbols

    "return true, if literal nameSpace symbols are allowed (#foo::bar) are allowed"

    ^ ParserFlags allowLiteralNameSpaceSymbols

!

allowOldStyleAssignment

    "return true, if underscore-assignment (pre ST-80v4 syntax) are to be allowed"

    ^ ParserFlags allowOldStyleAssignment

!

allowOldStyleAssignment:aBoolean

    "this allows turning on/off recognition of underscore-assignment (pre ST-80v4 syntax).

     You must turn this off, if code with variables named '_' is to be filedIn"

    ParserFlags allowOldStyleAssignment:aBoolean

!

allowQualifiedNames

    "return true, if #{..} qualified names are allowed"

    ^ ParserFlags allowQualifiedNames

!

allowQualifiedNames:aBoolean

    "this allows turning on/off support for qualifiedNames #{ .., } as in vw3.

     If you want to fileIn vw3 or later classes, enable this with:

     Notice, that qualified names are not really supported semantically

     (they are parsed, but treated like regular globals)

    "

    ParserFlags allowQualifiedNames:aBoolean.

    "

     self allowQualifiedNames:true

     self allowQualifiedNames:false

    "

!

allowSqueakExtensions

    "return true, if support for squeak extensions

     is enabled."

    ^ ParserFlags allowSqueakExtensions

!

allowSqueakExtensions:aBoolean

    "this allows turning on/off support for squeak extensions:

     If you want to fileIn Squeak classes, enable this with:

    ParserFlags allowSqueakExtensions:aBoolean.

    "

     self allowSqueakExtensions:true

     self allowSqueakExtensions:false

    "

!

allowUnderscoreInIdentifier

    "return true, if underscores are allowed in identifiers"

    ^ ParserFlags allowUnderscoreInIdentifier

!

allowUnderscoreInIdentifier:aBoolean

    "this allows turning on/off underscores in identifiers.

     If turned off (the default), underscores are not allowed in identifiers,

     but instead scanned as assignment character (old ST/80 syntax).

     If turned on, underscores are in identifiers are allowed, while extra

     underscores are still scanned as assignment.

     turn them off with:

    ParserFlags allowUnderscoreInIdentifier:aBoolean.

    "Modified: 7.9.1997 / 01:35:19 / cg"

!

maxBinarySelectorSize

    ^ Method maxBinarySelectorSize

!

warnCommonMistakes

    "return true, if common beginners mistakes are to be warned about"

    ^ ParserFlags warnCommonMistakes

!

warnCommonMistakes:aBoolean

    "this allows turning on/off warnings about common beginners mistakes.

     Those are not really errors in the strict sense, but often lead to

     run time errors later.

     Examples are: expr or:expr2, where expr2 is not a block.

     If you get bored by those warnings, turn them off by adding

     a line as:

     in your 'private.rc' file"