"

 COPYRIGHT (c) 1989 by Claus Gittinger

	      All Rights Reserved

 This software is furnished under a license and may be used

 only in accordance with the terms of that license and with the

 inclusion of the above copyright notice.   This software may not

 be provided or otherwise made available to, or used by, any

 other person.  No title to or ownership of the software is

 hereby transferred.

"

"{ Package: 'stx:libcomp' }"

Object subclass:#Scanner

	instanceVariableNames:'typeArray actionArray source lineNr token tokenType tokenPosition

		tokenValue tokenName tokenLineNr hereChar peekChar peekChar2

		requestor exitBlock errorFlag ignoreErrors ignoreWarnings

		saveComments currentComments collectedSource

		allowUnderscoreInIdentifier allowDollarInIdentifier

		allowOldStyleAssignment scanColonAsKeyword allowSqueakExtensions

		warnSTXSpecialComment warnUnderscoreInIdentifier

		warnOldStyleAssignment warnCommonMistakes outStream outCol

		warnSTXNameSpaceUse warnPossibleIncompatibilities

		warnDollarInIdentifier inArrayLiteral'

	classVariableNames:'TypeArray ActionArray Warnings EmptySourceNotificationSignal

		WarnSTXSpecials WarnOldStyleAssignment WarnUnderscoreInIdentifier

		WarnCommonMistakes WarnPossibleIncompatibilities

		WarnDollarInIdentifier AllowUnderscoreInIdentifier

		AllowDollarInIdentifier AllowSqueakExtensions AllowQualifiedNames

		AllowDolphinExtensions AllowOldStyleAssignment

		AllowExtendedBinarySelectors AllowExtendedSTXSyntax

		AllowFixedPointLiterals'

	poolDictionaries:''

	category:'System-Compiler'

!

Object subclass:#Comment

	instanceVariableNames:'commentType commentString'

	classVariableNames:''

	poolDictionaries:''

	privateIn:Scanner

!

!Scanner class methodsFor:'documentation'!

bugs

"

   array constant containing keywords as in:

        #(

                foo:bar:

                fee:baz:

         )

   is scanned as 4-element array containing ( #foo: #bar: #fee: #baz: )

   this MUST be fixed.

   workaround:

        #(

                #'foo:bar:'

                #'fee:baz:'

         )

"

!

copyright

"

 COPYRIGHT (c) 1989 by Claus Gittinger

	      All Rights Reserved

 This software is furnished under a license and may be used

 only in accordance with the terms of that license and with the

 inclusion of the above copyright notice.   This software may not

 be provided or otherwise made available to, or used by, any

 other person.  No title to or ownership of the software is

 hereby transferred.

"

!

documentation

"

    Scanner reads from a stream and returns individual smalltalk tokens

    Its main method is #nextToken, which reads and returns the next token

    Possibly placing additional information (such as tokenValue) into 

    instance variables as a side effect.

    TODO: 

        some testers noticed, that ST-80's scanner methods are called

        xLetter, xDigit etc. For code using these (internals), the nextNumber,

        nextIdentifier etc. methods should be renamed.

        (to me, these seem to be internal private methods; 

         their public use is not a good idea ..)

        Scanner is typically subclassed for parsing and #nextToken

        invoked via #self-sends.

        This should be changed and scanner ought to be an instance variable

        of Parser - this allows more flexible use of the scanner/parser

        framework (i.e. changing the scanner without affecting the parser).

    Extensions:

        this scanner allows for 3-character binary selectors.

        also, # is a valid selector. (however, ## is currently scanned as a symbol literal).

    [author:]

        Claus Gittinger

    [see also:]

        Parser

"

! !

!Scanner class methodsFor:'initialization'!

binarySelectorCharacters

    "return a collection of characters which are allowed in

     binary selectors"

    |characters|

    characters := #( $& $- $+ $= $* $/ $\ $< $> $~ $@ $, $? $!! $| $% $#).

    AllowExtendedBinarySelectors ifTrue:[

        characters := characters asOrderedCollection.

        characters add:(Character value:16rB1).  "/ plus-minus

        characters add:(Character value:16rD7).  "/ times

        characters add:(Character value:16rB7).  "/ centered dot

        characters add:(Character value:16rF7).  "/ divide

    ].

    ^ characters

    "Created: / 4.1.1997 / 14:13:24 / cg"

    "Modified: / 27.2.1998 / 02:01:28 / cg"

!

setupActions

    "initialize the scanners actionTables - these are used to dispatch

     into scanner methods as characters are read"

    |block|

    ActionArray := Array new:256.

    TypeArray := Array new:256.

    "/ TODO: later versions should be configurable w.r.t separators.

    "/ #(9 10 12 13 26 32) do: [:i | TypeArray at:(i+1) put: #separator].

    block := [:s :char | s nextNumber].

    ($0 asciiValue) to:($9 asciiValue) do:[:index |

        ActionArray at:index put:block

    ].

    block := [:s :char | s nextSpecial].

    self binarySelectorCharacters do:[:binop |

        TypeArray at:(binop asciiValue) put:#special.

        ActionArray at:(binop asciiValue) put:block

    ].

    "/ that one is a special case (both binarySelector AND syntax).

    TypeArray at:($| asciiValue) put:nil.

    block := [:s :char | s nextToken:char].

    #( $; $. $( $) $[ $] $^ $| $_ ${ $} ) do:[:ch |

        ActionArray at:(ch asciiValue) put:block

    ].

    block := [:s :char | s nextIdentifier].

    ($a asciiValue) to:($z asciiValue) do:[:index |

        ActionArray at:index put:block

    ].

    ($A asciiValue) to:($Z asciiValue) do:[:index |

        ActionArray at:index put:block

    ].

    AllowUnderscoreInIdentifier ifTrue:[

        ActionArray at:$_ asciiValue put:block

    ].

    "kludge: action is characterToken, but type is special"

    TypeArray at:($| asciiValue) put:#special.

    "kludge: action is nextColonOrAssign, but type is special"

    TypeArray at:($: asciiValue) put:#special.

    ActionArray at:($' asciiValue) put:[:s :char | s nextString].

    ActionArray at:($$ asciiValue) put:[:s :char | s nextCharacter].

    ActionArray at:($# asciiValue) put:[:s :char | s nextHash].

    ActionArray at:($!! asciiValue) put:[:s :char | s nextExcla].

    ActionArray at:($% asciiValue) put:[:s :char | s nextPrimitive].

    ActionArray at:($: asciiValue) put:[:s :char | s nextColonOrAssign]

    "

     Scanner setupActions

    "

    "Modified: 23.5.1997 / 12:07:55 / cg"

! !

!Scanner class methodsFor:'instance creation'!

for:aStringOrStream

    "create & return a new scanner reading from aStringOrStream"

    ^ self basicNew initializeFor:aStringOrStream

    "Modified: 23.5.1997 / 12:08:42 / cg"

!

new    

    "create & return a new scanner"

    ^ self basicNew initialize.

    "Modified: / 23.5.1997 / 12:08:42 / cg"

    "Created: / 26.5.1999 / 12:02:16 / stefan"

! !

!Scanner class methodsFor:'Signal constants'!

emptySourceNotificationSignal

    ^ EmptySourceNotificationSignal

    "Created: / 16.5.1998 / 15:55:14 / cg"

! !

!Scanner class methodsFor:'class initialization'!

initialize

    "initialize the classes defaults. Typically, these are changed

     later in the 'private.rc' file."

    EmptySourceNotificationSignal isNil ifTrue:[

        EmptySourceNotificationSignal := QuerySignal new mayProceed:true.

        EmptySourceNotificationSignal notifierString:'empty source given to evaluate'.

        EmptySourceNotificationSignal nameClass:self message:#emptySourceNotificationSignal.

    ].

    Warnings := true.

    WarnSTXSpecials := false.

    WarnUnderscoreInIdentifier := false.

    WarnDollarInIdentifier := true.

    WarnOldStyleAssignment := true.

    WarnCommonMistakes := true.

    WarnPossibleIncompatibilities := false.

    AllowUnderscoreInIdentifier := true.        "/ underscores in identifiers

    AllowDollarInIdentifier := false.           "/ st80-vms dollars in identifiers

    AllowOldStyleAssignment := true.            "/ st80 underscore as assignment

    AllowSqueakExtensions := false.             "/ squeak computed array

    AllowQualifiedNames := false.               "/ vw3 qualified names

    AllowExtendedBinarySelectors := false.      "/ vw5.4 extended binary selectors

    "

     self initialize

    "

    "Modified: / 16.5.1998 / 15:55:41 / cg"

! !

!Scanner class methodsFor:'defaults'!

allowDollarInIdentifier

    "return true, if $-characters are allowed in identifiers.

     Notice, that dollars are NEVER allowed as the first character in an identifier."

    ^ AllowDollarInIdentifier

    "Created: 7.9.1997 / 01:32:18 / cg"

    "Modified: 7.9.1997 / 01:39:44 / cg"

!

allowDollarInIdentifier:aBoolean

    "this allows turning on/off $-characters in identifiers.

     Notice, that dollars are NEVER allowed as the first character in an identifier.

     If turned off (the default), dollars are not allowed in identifiers,

     but instead are scanned as character-constant prefix.

     If turned on, dollars are in identifiers are allowed, while extra

     dollars are still scanned as constant character prefix.

     If you have to fileIn old VW-Vsn2.x classes, turn this off

     before filing them in; i.e.:

        Compiler allowDollarInIdentifiers:false"

    AllowDollarInIdentifier := aBoolean.

    "Created: 7.9.1997 / 01:34:49 / cg"

    "Modified: 7.9.1997 / 01:39:30 / cg"

!

allowDolphinExtensions

    "return true, if ##(..) computed literals are allowed"

    ^ AllowDolphinExtensions

!

allowDolphinExtensions:aBoolean

    "this allows turning on/off support for computed arrays ##(..) as in dolphin.

     If you want to fileIn Dolphin classes, enable this with:

        Compiler allowDolphinComputedArrays:true"

    AllowDolphinExtensions := aBoolean.

    "

     self allowDolphinExtensions:true

     self allowDolphinExtensions:false

    "

!

allowOldStyleAssignment

    "return true, if underscore-assignment (pre ST-80v4 syntax) are to be allowed"

    ^ AllowOldStyleAssignment

!

allowOldStyleAssignment:aBoolean

    "this allows turning on/off recognition of underscore-assignment (pre ST-80v4 syntax).

     You must turn this off, if code with variables named '_' is to be filedIn"

    AllowOldStyleAssignment := aBoolean

!

allowQualifiedNames

    "return true, if #{..} qualified names are allowed"

    ^ AllowQualifiedNames

!

allowQualifiedNames:aBoolean

    "this allows turning on/off support for qualifiedNames #{ .., } as in vw3.

     If you want to fileIn vw3 or later classes, enable this with:

        Compiler allowQualifiedNames:true

     Notice, that qualified names are not really supported semantically

     (they are parsed, but treated like regular globals)

    "

    AllowQualifiedNames := aBoolean.

    "

     self allowQualifiedNames:true

     self allowQualifiedNames:false

    "

!

allowSqueakExtensions

    "return true, if {..} computed arrays are allowed"

    ^ AllowSqueakExtensions

!

allowSqueakExtensions:aBoolean

    "this allows turning on/off support for computed arrays { .., } as in squeak.

     If you want to fileIn Squeak classes, enable this with:

        Compiler allowSqueakComputedArrays:true"

    AllowSqueakExtensions := aBoolean.

    "

     self allowSqueakExtensions:true

     self allowSqueakExtensions:false

    "

!

allowUnderscoreInIdentifier

    "return true, if underscores are allowed in identifiers"

    ^ AllowUnderscoreInIdentifier

!

allowUnderscoreInIdentifier:aBoolean

    "this allows turning on/off underscores in identifiers.

     If turned off (the default), underscores are not allowed in identifiers,

     but instead scanned as assignment character (old ST/80 syntax).

     If turned on, underscores are in identifiers are allowed, while extra

     underscores are still scanned as assignment.

     If you have to fileIn old VW-Vsn2.x classes, 

     turn them off with:

        Compiler allowUnderscoreInIdentifiers:false"

    AllowUnderscoreInIdentifier := aBoolean.

    self setupActions

    "Modified: 7.9.1997 / 01:35:19 / cg"

!

warnCommonMistakes

    "return true, if common beginners mistakes are to be warned about"

    ^ Warnings and:[WarnCommonMistakes]

!

warnCommonMistakes:aBoolean

    "this allows turning on/off warnings about common beginners mistakes.

     Those are not really errors in the strict sense, but often lead to

     run time errors later.

     Examples are: expr or:expr2, where expr2 is not a block.

     If you get bored by those warnings, turn them off by adding

     a line as:

	Compiler warnCommonMistakes:false

     in your 'private.rc' file"

    WarnCommonMistakes := aBoolean

!

warnDollarInIdentifier

    "return true, if $-characters in identifiers are to be warned about"

    ^ Warnings and:[WarnDollarInIdentifier]

    "Created: 7.9.1997 / 01:36:17 / cg"

!

warnDollarInIdentifier:aBoolean

    "this allows turning on/off warnings about $-characters in identifiers.

     You may find those warnings useful, to make certain that your code

     is portable to other smalltalk versions, which do not allow this

     (i.e. VW releases 2.x and maybe others).

     Notice, that dollars are NEVER allowed as the first character in an identifier.

     If you get bored by those warnings, turn them off by adding

     a line as:

        Compiler warnDollarInIdentifier:false

     in your 'private.rc' file"

    WarnDollarInIdentifier := aBoolean

    "Created: 7.9.1997 / 01:37:42 / cg"

    "Modified: 7.9.1997 / 01:40:02 / cg"

!

warnOldStyleAssignment

    "return true, if underscore-assignment (pre ST-80v4 syntax) are to be warned about"

    ^ Warnings and:[WarnOldStyleAssignment]

!

warnOldStyleAssignment:aBoolean

    "this allows turning on/off warnings about underscore-assignment (pre ST-80v4 syntax).

     If you get bored by those warnings, turn them off by adding

     a line as:

	Compiler warnOldStyleAssignment:false

     in your 'private.rc' file"

    WarnOldStyleAssignment := aBoolean

!

warnPossibleIncompatibilities

    "return true, if possible incompatibilities (with other ST systems)

     are to be warned about"

    ^ Warnings and:[WarnPossibleIncompatibilities]

    "Modified: 23.5.1997 / 12:02:02 / cg"

!

warnPossibleIncompatibilities:aBoolean

    "this turns warnings about possible incompatibilities (with other ST systems)

     on or off.

     If you get bored by those warnings, turn them off by adding

     a line as:

        Compiler warnPossibleIncompatibilities:false

     in your 'private.rc' file."

    WarnPossibleIncompatibilities := aBoolean

    "Created: 23.5.1997 / 12:02:45 / cg"

!

warnSTXSpecials

    "return true, if ST/X specials are to be warned about"

    ^ Warnings and:[WarnSTXSpecials]

!

warnSTXSpecials:aBoolean

    "this allows turning on/off warnings about stx specials.

     If you get bored by those warnings, turn them off by adding

     a line as:

	Compiler warnSTXSpecials:false

     in your 'private.rc' file"

    WarnSTXSpecials := aBoolean

!

warnUnderscoreInIdentifier

    "return true, if underscores in identifiers are to be warned about"

    ^ Warnings and:[WarnUnderscoreInIdentifier]

!

warnUnderscoreInIdentifier:aBoolean

    "this allows turning on/off warnings about underscores in identifiers.

     You may find those warnings useful, to make certain that your code