"

 COPYRIGHT (c) 2003 by Claus Gittinger

	      All Rights Reserved

 This software is furnished under a license and may be used

 only in accordance with the terms of that license and with the

 inclusion of the above copyright notice.   This software may not

 be provided or otherwise made available to, or used by, any

 other person.  No title to or ownership of the software is

 hereby transferred.

"

"{ Package: 'stx:libjava/tools' }"

JavaScannerBase subclass:#JavaScanner

	instanceVariableNames:'allowDegeneratedMantissa allowRunawayString keywordTable'

	classVariableNames:'Verbose'

	poolDictionaries:''

	category:'Languages-Java-Parser'

!

Object subclass:#Token

	instanceVariableNames:'type value startPosition endPosition'

	classVariableNames:''

	poolDictionaries:''

	privateIn:JavaScanner

!

!JavaScanner class methodsFor:'documentation'!

copyright

"

 COPYRIGHT (c) 2003 by Claus Gittinger

	      All Rights Reserved

 This software is furnished under a license and may be used

 only in accordance with the terms of that license and with the

 inclusion of the above copyright notice.   This software may not

 be provided or otherwise made available to, or used by, any

 other person.  No title to or ownership of the software is

 hereby transferred.

"

!

examples

"

    |s in|

    in := '

function scalefactor(value) {

    scalevector[0]=value;

    scalevector[1]=1.;

    scalevector[2]=1.;

}

'.

    s := JavaScanner for:in readStream.

    s nextToken

    |s in|

    in := '

function scalefactor(value) {

    scalevector[0]=value;

    scalevector[1]=1.;

    scalevector[2]=1.;

}

'.

    s := JavaScanner new.

    s scanTokens:(in readStream).

    |s in|

    in := '

function scalefactor(value) {

    scalevector[0]=value;

    scalevector[1]=1.;

    scalevector[2]=1.;

}

'.

    in := in readStream.

    s := JavaScanner for:in.

    [in atEnd] whileFalse:[

	Transcript showCR:s nextToken

    ]

"

    "Created: / 13.5.1998 / 14:54:06 / cg"

! !

!JavaScanner class methodsFor:'initialization'!

setupActions

    "initialize the scanners actionTables - these are used to dispatch

     into scanner methods as characters are read"

    |block|

    self setupKeywordTable.

    ActionArray := Array new:256.

    TypeArray := Array new:256.

    block := [:s :char | s nextNumber].

    ($0 asciiValue) to:($9 asciiValue) do:[:index |

        ActionArray at:index put:block

    ].

    block := [:s :char | s nextSingleCharacterToken:char].

    #( $: $; $, ${ $} $( $) $[ $] $_ $? $@) do:[:ch |

        ActionArray at:(ch asciiValue) put:block

    ].

    block := [:s :char | s nextIdentifier].

    ($a asciiValue) to:($z asciiValue) do:[:index |

        ActionArray at:index put:block

    ].

    ($A asciiValue) to:($Z asciiValue) do:[:index |

        ActionArray at:index put:block

    ].

    ActionArray at:$_ asciiValue put:block.

    ActionArray at:$$ asciiValue put:block.

    ActionArray at:($. asciiValue) put:[:s :char | s nextDotOrFloatOrEllipsis].

    ActionArray at:($' asciiValue) put:[:s :char | s nextString:$' character:true].

    ActionArray at:($" asciiValue) put:[:s :char | s nextString:$" character:false].

    ActionArray at:($!! asciiValue) put:[:s :char | s nextMulti:#(($= #'!!=')) after:char].

    ActionArray at:($= asciiValue) put:[:s :char | s nextMulti:#(($= #'==')) after:char].

    ActionArray at:($< asciiValue) put:[:s :char | s nextMulti:#(($= #'<=') ($< #'<<')) after:char].

    ActionArray at:($> asciiValue) put:[:s :char | s nextMulti:#(($= #'>=') ($> #'>>' $> #'>>>' $= #'>>>=')) after:char].

    ActionArray at:($- asciiValue) put:[:s :char | s nextMulti:#(($- #'--') ($= #'-=')) after:char].

    ActionArray at:($+ asciiValue) put:[:s :char | s nextMulti:#(($+ #'++') ($= #'+=')) after:char].

    ActionArray at:($* asciiValue) put:[:s :char | s nextMulti:#(($= #'*=')) after:char].

    ActionArray at:($/ asciiValue) put:[:s :char | s nextMulti:#(($= #'/=') ($/ nil #skipEOLComment) ($* nil #skipComment)) after:char].

    ActionArray at:($% asciiValue) put:[:s :char | s nextMulti:#(($= #'%=')) after:char].

    ActionArray at:($& asciiValue) put:[:s :char | s nextMulti:#(($= #'&=') ($& #'&&')) after:char].

    ActionArray at:($^ asciiValue) put:[:s :char | s nextMulti:#(($= #'^=')) after:char].

    ActionArray at:($~ asciiValue) put:[:s :char | s nextMulti:#(($= #'~=')) after:char].

    ActionArray at:($| asciiValue) put:[:s :char | s nextMulti:#(($= #'|=') ($| #'||')) after:char].

    "

     self setupActions

    "

    "Created: / 14-05-1998 / 15:48:03 / cg"

    "Modified: / 17-05-1998 / 21:03:37 / cg"

    "Modified: / 16-03-2012 / 23:49:40 / Jan Vrany <jan.vrany@fit.cvut.cz>"

!

setupKeywordTable

    "initialize the scanners actionTables - these are used to dispatch

     into scanner methods as characters are read"

    KeywordTable := Dictionary new.

    #(

        'abstract'     abstract

        'boolean'      boolean

        'break'        break

        'byte'         byte

        'case'         case

        'catch'        catch

        'char'         char

        'class'        class

        'continue'     continue

        'default'      default

        'do'           do

        'double'       double

        'enum'         enum

        'else'         else

        'extends'      extends

        'final'        final

        'finally'      finally

        'float'        float

        'for'          for

        'if'           if

        'implements'   implements

        'import'       import

        'instanceof'   instanceof

        'int'          int

        'interface'    interface

        'long'         long

        'native'       native

        'new'          new

        'package'      package

        'private'      private

        'protected'    protected

        'public'       public

        'return'       return

        'short'        short

        'static'       static

        'super'        super

        'strictfp'     strictfp

        'switch'       switch

        'synchronized' synchronized

        'this'         this

        'threadsafe'   threadsafe

        'throw'        throw

        'throws'       throws

        'transient'    transient

        'try'          try

        'void'         void

        'volatile'     volatile

        'while'        while

        "/ Reserved, but Unused KeyWords

        'const'        const

        'byvalue'      byvalue

        'goto'         goto

        "/ literals

        'null'         null

        'true'         boolean

        'false'        boolean

    ) pairWiseDo:[:s :kw |

        KeywordTable at:s put:kw

    ].

    "

     NewJavaScanner setupKeywordTable

    "

    "Created: / 14-05-1998 / 15:48:03 / cg"

    "Modified: / 17-05-1998 / 21:03:37 / cg"

    "Modified: / 15-03-2012 / 22:47:26 / Jan Vrany <jan.vrany@fit.cvut.cz>"

! !

!JavaScanner methodsFor:'accessing'!

allowRunawayString:aBoolean

    allowRunawayString := aBoolean.

!

token

    "the previously scanned token"

    ^Token new

        type: tokenType;

        value: tokenValue;

        startPosition: tokenStartPosition + 1;

        endPosition: tokenEndPosition + 1;

        yourself

    "Created: / 17-03-2012 / 13:32:09 / Jan Vrany <jan.vrany@fit.cvut.cz>"

    "Modified: / 03-08-2013 / 20:23:44 / Jan Vrany <jan.vrany@fit.cvut.cz>"

! !

!JavaScanner methodsFor:'converting'!

asPetitStream

    ^self

    "Created: / 14-03-2012 / 22:51:43 / Jan Vrany <jan.vrany@fit.cvut.cz>"

! !

!JavaScanner methodsFor:'error handling'!

errorMessagePrefix

    ^ 'Java Error:'

!

warningMessagePrefix

    ^ 'Java Warning:'

! !

!JavaScanner methodsFor:'initialization'!

initialize

    "initialize the scanner"

    super initialize.

    allowDegeneratedMantissa := true.     "/ something like 123.

    allowRunawayString := false.

    keywordTable := self class keywordTable.

    saveComments := true.

    "Modified: / 11-08-2014 / 21:14:21 / Jan Vrany <jan.vrany@fit.cvut.cz>"

! !

!JavaScanner methodsFor:'private'!

checkForKeyword:string

    "check if string is a keyword (as opposed to an identifier)."

    |tok|

    (tok := keywordTable at:string ifAbsent:nil) notNil ifTrue:[

	tokenType := tok.

	^ true

    ].

    ^ false

!

isCommentCharacter:ch

    "return true, if ch is the comment-start character"

    ^ false

    "Created: / 14.5.1998 / 20:53:33 / cg"

!

rememberTokenStartPosition

    self rememberTokenStartPosition:0

    "Created: / 17-03-2012 / 00:19:57 / Jan Vrany <jan.vrany@fit.cvut.cz>"

!

rememberTokenStartPosition: offset

    tokenStartPosition := source position - offset

    "Created: / 17-03-2012 / 17:39:25 / Jan Vrany <jan.vrany@fit.cvut.cz>"

! !

!JavaScanner methodsFor:'reading next token'!

characterEscape:char

    |ascii c |

    char == $" ifTrue:[

        ^ $".

    ].

    char == $b ifTrue:[

        ^ Character backspace

    ].

    char == $t ifTrue:[

        ^ Character tab

    ].

    char == $n ifTrue:[

        ^ Character cr

    ].

    char == $r ifTrue:[

        ^ Character return

    ].

    char == $f ifTrue:[

        ^ Character newPage

    ].

    char == $u ifTrue:[

        ascii := 0.

        c := source peekOrNil.

        4 timesRepeat:[

            (c isDigitRadix:16) ifFalse:[

                self syntaxError:'invalid hex character constant'

                        position:source position-2 to:(source position - 1).

                ^ Character value:ascii

            ].

            ascii := (ascii bitShift:4).

            ascii := ascii + c digitValue.

            source next. c := source peekOrNil.

        ].

        ^ Character value:ascii

    ].

"/    char == $x ifTrue:[

"/        ascii := 0.

"/        c := source peekOrNil.

"/        2 timesRepeat:[

"/            (c isDigitRadix:16) ifFalse:[

"/                self syntaxError:'invalid hex character constant'

"/                        position:source position-2 to:(source position - 1).

"/                ^ Character value:ascii

"/            ].

"/            ascii := (ascii bitShift:4).

"/            ascii := ascii + c digitValue.

"/            source next. c := source peekOrNil.

"/        ].

"/        ^ Character value:ascii

"/    ].

    "OctalEscape ::= \ OctalDigit |

                     \ OctalDigit OctalDigit

                     \ ZeroToThree OctalDigit OctalDigit"

    (char between:$0 and:$3) ifTrue:[

        ascii := char digitValue.

        c := source peekOrNil.

        (c between: $0 and: $7) ifTrue:[

            source next.

            ascii := (ascii bitShift:3).

            ascii := ascii + c digitValue.

        ].

        c := source peekOrNil.

        (c between: $0 and: $7) ifTrue:[

            source next.

            ascii := (ascii bitShift:3).

            ascii := ascii + c digitValue.

        ].

        ^ Character value:ascii

    ].

    (char between:$4 and: $7) ifTrue:[

        ascii := char digitValue.            

        c := source peekOrNil.

        (c between: $0 and: $7) ifTrue:[

            source next.

            ascii := (ascii bitShift:3).

            ascii := ascii + c digitValue.

        ].

        ^ Character value:ascii

    ].

    ^ char

    "Modified: / 16-03-2012 / 10:07:10 / Jan Vrany <jan.vrany@fit.cvut.cz>"

!

exponentPart:initialValue

    |nextChar value s|

    value := initialValue.

    nextChar := source peekOrNil.

    ((nextChar == $e) or:[nextChar == $E]) ifTrue:[

	nextChar := source nextPeek.

	(nextChar notNil and:[(nextChar isDigitRadix:10) or:['+-' includes:nextChar]]) ifTrue:[

	    s := 1.

	    (nextChar == $+) ifTrue:[

		nextChar := source nextPeek

	    ] ifFalse:[

		(nextChar == $-) ifTrue:[

		    nextChar := source nextPeek.

		    s := s negated

		]

	    ].

	    value := value asFloat

		     * (10.0 raisedToInteger:((Integer readFrom:source radix:10) * s))

	]

    ].

    ^ value

!

hexponentPart:initialValue

    |nextChar value s|

    value := initialValue.

    nextChar := source peekOrNil.

    ((nextChar == $p) or:[nextChar == $P]) ifTrue:[

        nextChar := source nextPeek.

        (nextChar notNil and:[(nextChar isDigitRadix:16) or:['+-' includes:nextChar]]) ifTrue:[

            s := 1.

            (nextChar == $+) ifTrue:[

                nextChar := source nextPeek

            ] ifFalse:[

                (nextChar == $-) ifTrue:[

                    nextChar := source nextPeek.

                    s := s negated

                ]

            ].

            value := value asFloat

                     * (16.0 raisedToInteger:((Integer readFrom:source radix:16) * s))

        ]

    ].

    ^ value

    "Created: / 16-03-2012 / 00:00:09 / Jan Vrany <jan.vrany@fit.cvut.cz>"

!

nextDotOrFloat

    |nextChar|

    nextChar := source nextPeek.

    nextChar isDigit ifTrue:[

	^ self nextFractionalPart:0.

    ].

    tokenType := tokenValue := $. .

    ^ tokenType

!

nextDotOrFloatOrEllipsis

    |nextChar nextChar2|

    nextChar := source nextPeek.

    (nextChar isNil and:[ source atEnd ]) ifFalse:[

        nextChar isDigit ifTrue:[

            ^ self nextFractionalPart:0.

        ].

        nextChar == $. ifTrue:[

            nextChar2 := source nextPeek.

            nextChar2 == $. ifTrue:[

                source next.

                tokenType := #Ellipsis.

                tokenValue := '...'.

                ^tokenType.

            ] ifFalse:[

                source skip: -1.

            ].

        ].

    ].

    tokenType := tokenValue := $. .

    ^ tokenType

    "Created: / 15-03-2012 / 10:08:13 / Jan Vrany <jan.vrany@fit.cvut.cz>"

    "Modified: / 14-05-2014 / 16:12:03 / Jan Vrany <jan.vrany@fit.cvut.cz>"

!

nextFractionalPart:intValue

    |nextChar value|

    value := intValue.

    nextChar := source peekOrNil.

    (nextChar notNil and:[nextChar isDigitRadix:10]) ifTrue:[

        value := value asFloat + (self nextMantissa:10).

        nextChar := source peekOrNil

    ] ifFalse:[

        allowDegeneratedMantissa == true ifTrue:[

            self warning:'degenerated float constant: ' , value printString , '.' .

            tokenValue := value asFloat.

            tokenType := #Float.

            ^ tokenType

        ].

        nextChar := peekChar := $..

    ].

    ((nextChar == $e) or:[nextChar == $E]) ifTrue:[

        value := self exponentPart:value.

        nextChar := source peekOrNil

    ] ifFalse:[

        ((nextChar == $p) or:[nextChar == $P]) ifTrue:[