"

 COPYRIGHT (c) 2003 by Claus Gittinger

	      All Rights Reserved

 This software is furnished under a license and may be used

 only in accordance with the terms of that license and with the

 inclusion of the above copyright notice.   This software may not

 be provided or otherwise made available to, or used by, any

 other person.  No title to or ownership of the software is

 hereby transferred.

"

"{ Package: 'jv:dart/compiler' }"

"{ NameSpace: Dart }"

ScannerBase subclass:#Scanner

	instanceVariableNames:'allowDegeneratedMantissa keywordTable'

	classVariableNames:'Verbose'

	poolDictionaries:''

	category:'Languages-Dart-Parser'

!

Object subclass:#Token

	instanceVariableNames:'type value startPosition endPosition'

	classVariableNames:''

	poolDictionaries:''

	privateIn:Scanner

!

!Scanner class methodsFor:'documentation'!

copyright

"

 COPYRIGHT (c) 2003 by Claus Gittinger

	      All Rights Reserved

 This software is furnished under a license and may be used

 only in accordance with the terms of that license and with the

 inclusion of the above copyright notice.   This software may not

 be provided or otherwise made available to, or used by, any

 other person.  No title to or ownership of the software is

 hereby transferred.

"

!

examples

"

    |s in|

    in := '

function scalefactor(value) {

    scalevector[0]=value;

    scalevector[1]=1.;

    scalevector[2]=1.;

}

'.

    s := JavaScanner for:in readStream.

    s nextToken

    |s in|

    in := '

function scalefactor(value) {

    scalevector[0]=value;

    scalevector[1]=1.;

    scalevector[2]=1.;

}

'.

    s := JavaScanner new.

    s scanTokens:(in readStream).

    |s in|

    in := '

function scalefactor(value) {

    scalevector[0]=value;

    scalevector[1]=1.;

    scalevector[2]=1.;

}

'.

    in := in readStream.

    s := JavaScanner for:in.

    [in atEnd] whileFalse:[

	Transcript showCR:s nextToken

    ]

"

    "Created: / 13.5.1998 / 14:54:06 / cg"

! !

!Scanner class methodsFor:'initialization'!

setupActions

    "initialize the scanners actionTables - these are used to dispatch

     into scanner methods as characters are read"

    |block|

    self setupKeywordTable.

    ActionArray := Array new:256.

    TypeArray := Array new:256.

    block := [:s :char | s nextNumber].

    ($0 codePoint) to:($9 codePoint) do:[:index |

        ActionArray at:index put:block

    ].

    block := [:s :char | s nextSingleCharacterToken:char].

    #( $: $; $, ${ $} $( $) $[ $] $_ $? $@) do:[:ch |

        ActionArray at:(ch asciiValue) put:block

    ].

    block := [:s :char | s nextIdentifier].

    ($a codePoint) to:($z codePoint) do:[:index |

        ActionArray at:index put:block

    ].

    ($A codePoint) to:($Z codePoint) do:[:index |

        ActionArray at:index put:block

    ].

    ActionArray at:$_ codePoint put:block.

    ActionArray at:$$ codePoint put:block.

    ActionArray at:($. codePoint) put:[:s :char | s nextDotOrFloatOrEllipsis].

    ActionArray at:($' codePoint) put:[:s :char | s nextString:$' character:false].

    ActionArray at:($" codePoint) put:[:s :char | s nextString:$" character:false].

    ActionArray at:($!! codePoint) put:[:s :char | s nextMulti:#(($= #'!!=')) after:char].

    ActionArray at:($= codePoint) put:[:s :char | s nextMulti:#(($= #'==')) after:char].

    ActionArray at:($< codePoint) put:[:s :char | s nextMulti:#(($= #'<=') ($< #'<<')) after:char].

    ActionArray at:($> codePoint) put:[:s :char | s nextMulti:#(($= #'>=') ($> #'>>' $> #'>>>' $= #'>>>=')) after:char].

    ActionArray at:($- codePoint) put:[:s :char | s nextMulti:#(($- #'--') ($= #'-=')) after:char].

    ActionArray at:($+ codePoint) put:[:s :char | s nextMulti:#(($+ #'++') ($= #'+=')) after:char].

    ActionArray at:($* codePoint) put:[:s :char | s nextMulti:#(($= #'*=')) after:char].

    ActionArray at:($/ codePoint) put:[:s :char | s nextMulti:#(($= #'/=') ($/ nil #skipEOLComment) ($* nil #skipComment)) after:char].

    ActionArray at:($% codePoint) put:[:s :char | s nextMulti:#(($= #'%=')) after:char].

    ActionArray at:($& codePoint) put:[:s :char | s nextMulti:#(($= #'&=') ($& #'&&')) after:char].

    ActionArray at:($^ codePoint) put:[:s :char | s nextMulti:#(($= #'^=')) after:char].

    ActionArray at:($~ codePoint) put:[:s :char | s nextMulti:#(($= #'~=')) after:char].

    ActionArray at:($| codePoint) put:[:s :char | s nextMulti:#(($= #'|=') ($| #'||')) after:char].

    "

     self setupActions

    "

    "Created: / 14-05-1998 / 15:48:03 / cg"

    "Modified: / 17-05-1998 / 21:03:37 / cg"

    "Modified: / 11-01-2013 / 13:04:40 / Jan Vrany <jan.vrany@fit.cvut.cz>"

!

setupKeywordTable

    "initialize the scanners actionTables - these are used to dispatch

     into scanner methods as characters are read"

    KeywordTable := Dictionary new.

    #(

        'abstract'              abstract

        'assert'                assert

        'class'                 class

        'extends'               extends

        'factory'               factory

        'get'                   get

        'implements'            implements

        'import'                import

        'interface'             interface

        'is'                    is

        'library'               library

        'native'                native

        'negate'                negate

        'operator'              operator

        'set'                   set

        'source'                source

        'static'                static

        'typedef'               typedef

        'this'                  this

        'super'                 super

        'null'                  null

        'true'                  #true

        'false'                 #false

        'const'                 const

        'new'                   new

        'void'                  void

        'final'                 final

        'var'                   var

        'while'                 while

        'do'                    do

        'for'                   for

        'in'                    in

        'if'                    if

        'else'                  else

        'switch'                switch

        'case'                  case

        'default'               default

        'try'                   try

        'catch'                 catch

        'finally'               finally

        'break'                 break

        'continue'              continue

        'return'                return

        'throw'                 throw

    ) pairWiseDo:[:s :kw |

        KeywordTable at:s put:kw

    ].

    "

     NewJavaScanner setupKeywordTable

    "

    "Created: / 14-05-1998 / 15:48:03 / cg"

    "Modified: / 17-05-1998 / 21:03:37 / cg"

    "Modified: / 10-01-2013 / 10:51:58 / Jan Vrany <jan.vrany@fit.cvut.cz>"

! !

!Scanner class methodsFor:'utility scanning'!

scan: aString

    | tokens scanner |

    tokens := OrderedCollection new.

    scanner := (self for: aString).

    [ scanner atEnd ] whileFalse:[

        tokens add: scanner nextToken

    ].

    ^tokens

    "Created: / 11-01-2013 / 12:57:02 / Jan Vrany <jan.vrany@fit.cvut.cz>"

! !

!Scanner methodsFor:'accessing'!

token

    "the previously scanned token"

    ^Token new

        type: tokenType;

        value: tokenValue;

        startPosition: tokenStartPosition;

        endPosition: tokenEndPosition;

        yourself

    "Created: / 17-03-2012 / 13:32:09 / Jan Vrany <jan.vrany@fit.cvut.cz>"

    "Modified: / 10-01-2013 / 11:20:19 / Jan Vrany <jan.vrany@fit.cvut.cz>"

! !

!Scanner methodsFor:'converting'!

asPetitStream

    ^self

    "Created: / 14-03-2012 / 22:51:43 / Jan Vrany <jan.vrany@fit.cvut.cz>"

! !

!Scanner methodsFor:'error handling'!

errorMessagePrefix

    ^ 'Dart Error:'

    "Modified: / 10-01-2013 / 11:21:14 / Jan Vrany <jan.vrany@fit.cvut.cz>"

!

warningMessagePrefix

    ^ 'Dart Warning:'

    "Modified: / 10-01-2013 / 11:21:18 / Jan Vrany <jan.vrany@fit.cvut.cz>"

! !

!Scanner methodsFor:'initialization'!

initialize

    "initialize the scanner"

    super initialize.

    allowDegeneratedMantissa := true.     "/ something like 123.

    keywordTable := self class keywordTable.

! !

!Scanner methodsFor:'private'!

checkForKeyword:string

    "check if string is a keyword (as opposed to an identifier)."

    |tok|

    (tok := keywordTable at:string ifAbsent:nil) notNil ifTrue:[

	tokenType := tok.

	^ true

    ].

    ^ false

!

isCommentCharacter:ch

    "return true, if ch is the comment-start character"

    ^ false

    "Created: / 14.5.1998 / 20:53:33 / cg"

!

rememberTokenStartPosition

    self rememberTokenStartPosition:0

    "Created: / 17-03-2012 / 00:19:57 / Jan Vrany <jan.vrany@fit.cvut.cz>"

!

rememberTokenStartPosition: offset

    tokenStartPosition := source position - offset

    "Created: / 17-03-2012 / 17:39:25 / Jan Vrany <jan.vrany@fit.cvut.cz>"

! !

!Scanner methodsFor:'reading next token'!

characterEscape:char

    |ascii c |

    char == $" ifTrue:[

        ^ $".

    ].

    char == $b ifTrue:[

        ^ Character backspace

    ].

    char == $t ifTrue:[

        ^ Character tab

    ].

    char == $n ifTrue:[

        ^ Character cr

    ].

    char == $r ifTrue:[

        ^ Character return

    ].

    char == $f ifTrue:[

        ^ Character newPage

    ].

    char == $u ifTrue:[

        ascii := 0.

        c := source peekOrNil.

        4 timesRepeat:[

            (c isDigitRadix:16) ifFalse:[

                self syntaxError:'invalid hex character constant'

                        position:source position-2 to:(source position - 1).

                ^ Character value:ascii

            ].

            ascii := (ascii bitShift:4).

            ascii := ascii + c digitValue.

            source next. c := source peekOrNil.

        ].

        ^ Character value:ascii

    ].

"/    char == $x ifTrue:[

"/        ascii := 0.

"/        c := source peekOrNil.

"/        2 timesRepeat:[

"/            (c isDigitRadix:16) ifFalse:[

"/                self syntaxError:'invalid hex character constant'

"/                        position:source position-2 to:(source position - 1).

"/                ^ Character value:ascii

"/            ].

"/            ascii := (ascii bitShift:4).

"/            ascii := ascii + c digitValue.

"/            source next. c := source peekOrNil.

"/        ].

"/        ^ Character value:ascii

"/    ].

    "OctalEscape ::= \ OctalDigit |

                     \ OctalDigit OctalDigit

                     \ ZeroToThree OctalDigit OctalDigit"

    (char between:$0 and:$3) ifTrue:[

        ascii := char digitValue.

        c := source peekOrNil.

        (c between: $0 and: $7) ifTrue:[

            source next.

            ascii := (ascii bitShift:3).

            ascii := ascii + c digitValue.

        ].

        c := source peekOrNil.

        (c between: $0 and: $7) ifTrue:[

            source next.

            ascii := (ascii bitShift:3).

            ascii := ascii + c digitValue.

        ].

        ^ Character value:ascii

    ].

    (char between:$4 and: $7) ifTrue:[

        ascii := char digitValue.            

        c := source peekOrNil.

        (c between: $0 and: $7) ifTrue:[

            source next.

            ascii := (ascii bitShift:3).

            ascii := ascii + c digitValue.

        ].

        ^ Character value:ascii

    ].

    ^ char

    "Modified: / 16-03-2012 / 10:07:10 / Jan Vrany <jan.vrany@fit.cvut.cz>"

!

exponentPart:initialValue

    |nextChar value s|

    value := initialValue.

    nextChar := source peekOrNil.

    ((nextChar == $e) or:[nextChar == $E]) ifTrue:[

	nextChar := source nextPeek.

	(nextChar notNil and:[(nextChar isDigitRadix:10) or:['+-' includes:nextChar]]) ifTrue:[

	    s := 1.

	    (nextChar == $+) ifTrue:[

		nextChar := source nextPeek

	    ] ifFalse:[

		(nextChar == $-) ifTrue:[

		    nextChar := source nextPeek.

		    s := s negated

		]

	    ].

	    value := value asFloat

		     * (10.0 raisedToInteger:((Integer readFrom:source radix:10) * s))

	]

    ].

    ^ value

!

hexponentPart:initialValue

    |nextChar value s|

    value := initialValue.

    nextChar := source peekOrNil.

    ((nextChar == $p) or:[nextChar == $P]) ifTrue:[

        nextChar := source nextPeek.

        (nextChar notNil and:[(nextChar isDigitRadix:16) or:['+-' includes:nextChar]]) ifTrue:[

            s := 1.

            (nextChar == $+) ifTrue:[

                nextChar := source nextPeek

            ] ifFalse:[

                (nextChar == $-) ifTrue:[

                    nextChar := source nextPeek.

                    s := s negated

                ]

            ].

            value := value asFloat

                     * (16.0 raisedToInteger:((Integer readFrom:source radix:16) * s))

        ]

    ].

    ^ value

    "Created: / 16-03-2012 / 00:00:09 / Jan Vrany <jan.vrany@fit.cvut.cz>"

!

nextDotOrFloat

    |nextChar|

    nextChar := source nextPeek.

    nextChar isDigit ifTrue:[

	^ self nextFractionalPart:0.

    ].

    tokenType := tokenValue := $. .

    ^ tokenType

!

nextDotOrFloatOrEllipsis

    |nextChar nextChar2|

    nextChar := source nextPeek.

    nextChar isDigit ifTrue:[

        ^ self nextFractionalPart:0.

    ].

    nextChar == $. ifTrue:[

        nextChar2 := source nextPeek.

        nextChar2 == $. ifTrue:[

            source next.

            tokenType := #Ellipsis.

            tokenValue := '...'.

            ^tokenType.

        ] ifFalse:[

            source skip: -1.

        ].

    ].

    tokenType := tokenValue := $. .

    ^ tokenType

    "Created: / 15-03-2012 / 10:08:13 / Jan Vrany <jan.vrany@fit.cvut.cz>"

!

nextFractionalPart:intValue

    |nextChar value|

    value := intValue.

    nextChar := source peekOrNil.

    (nextChar notNil and:[nextChar isDigitRadix:10]) ifTrue:[

        value := value asFloat + (self nextMantissa:10).

        nextChar := source peekOrNil

    ] ifFalse:[

        allowDegeneratedMantissa == true ifTrue:[

            self warning:'degenerated float constant: ' , value printString , '.' .

            tokenValue := value asFloat.

            tokenType := #Float.

            ^ tokenType

        ].

        nextChar := peekChar := $..

    ].

    ((nextChar == $e) or:[nextChar == $E]) ifTrue:[

        value := self exponentPart:value.

        nextChar := source peekOrNil

    ] ifFalse:[

        ((nextChar == $p) or:[nextChar == $P]) ifTrue:[

            value := self hexponentPart:value.

            nextChar := source peekOrNil

        ].

    ].