"

 COPYRIGHT (c) 1998 by eXept Software AG

	      All Rights Reserved

 This software is furnished under a license and may be used

 only in accordance with the terms of that license and with the

 inclusion of the above copyright notice.   This software may not

 be provided or otherwise made available to, or used by, any

 other person.  No title to or ownership of the software is

 hereby transferred.

"

"{ Package: 'stx:libjavascript' }"

"{ NameSpace: Smalltalk }"

Scanner subclass:#JavaScriptScanner

	instanceVariableNames:'allowDegeneratedMantissa acceptRegexString previousToken'

	classVariableNames:'Verbose SupportRegex'

	poolDictionaries:''

	category:'Languages-JavaScript-Compiling & Parsing'

!

!JavaScriptScanner class methodsFor:'documentation'!

copyright

"

 COPYRIGHT (c) 1998 by eXept Software AG

	      All Rights Reserved

 This software is furnished under a license and may be used

 only in accordance with the terms of that license and with the

 inclusion of the above copyright notice.   This software may not

 be provided or otherwise made available to, or used by, any

 other person.  No title to or ownership of the software is

 hereby transferred.

"

!

documentation

"

    tokenizer for JavaScript

    news:

        does regex '/pattern/' vs. division operator 

        determination via a rule described in

            https://www-archive.mozilla.org/js/language/js20-2002-04/rationale/syntax.html#regular-expressions

        if that makes problems, set the variable:

            SupportRegex

        to false, to get the previous behavior (which is: not supporting regex)

"

!

examples

"<<END

[exBegin]

    |s in|

    in := '

function scalefactor(value) {

    scalevector[0]=value;

    scalevector[1]=1.;

    scalevector[2]=1.;

}

'.

    s := JavaScriptScanner for:in readStream.

    s nextToken  

[exEnd]

[exBegin]

    |s in|

    in := '

function scalefactor(value) {

    scalevector[0]=value;

    scalevector[1]=1.;

    scalevector[2]=1.;

}

'.

    s := JavaScriptScanner new.

    s scanTokens:(in readStream).

[exEnd]

[exBegin]

    |s in|

    in := '

function scalefactor(value) {

    return "hello";

}

'.

    s := JavaScriptScanner new.

    s scanTokens:(in readStream).

[exEnd]

[exBegin]

    |s in|

    in := '

function scalefactor(value) {

    return /hello/;

}

'.

    in := in readStream.

    s := JavaScriptScanner for:in.

    [in atEnd] whileFalse:[

        Transcript showCR:s nextToken

    ]

[exEnd]

[exBegin]

    |s in|

    in := '

function scalefactor(value) {

    return a /hello/;

}

'.

    in := in readStream.

    s := JavaScriptScanner for:in.

    [in atEnd] whileFalse:[

        Transcript showCR:s nextToken

    ]

[exEnd]

END"

    "Created: / 13.5.1998 / 14:54:06 / cg"

! !

!JavaScriptScanner class methodsFor:'initialization'!

setupActions

    "initialize the scanners actionTables - these are used to dispatch

     into scanner methods as characters are read

     Notice, this is only called if ActionArray is nil.

         ActionArray := nil

    "

    |block actionArray typeArray|

    actionArray := Array new:256.

    typeArray := Array new:256.

    block := [:s :char | s nextNumber].

    actionArray from:($0 codePoint) to:($9 codePoint) put:block.

    block := [:s :char | s nextToken:char].

    #( $: $; $. $, ${ $} $( $) $[ $] $_ $? $~) do:[:ch |

        actionArray at:(ch codePoint) put:block

    ].

    block := [:s :char | s nextIdentifier].

    actionArray from:($a codePoint) to:($z codePoint) put:block.

    actionArray from:($A codePoint) to:($Z codePoint) put:block.

    actionArray at:$_ codePoint put:block.

    actionArray at:($' codePoint) put:[:s :char | s nextString ].

    actionArray at:($" codePoint) put:[:s :char | s nextString ].

    actionArray at:($` codePoint) put:[:s :char | s nextString ].

    actionArray at:($!! codePoint) put:[:s :char | s nextMulti:#(($= #'!!=' (($= #'!!==')))) after:char].

    actionArray at:($= codePoint) put:[:s :char | s nextMulti:#(($> #'=>') ($= #'==' (($= #'===')))) after:char].

    actionArray at:($< codePoint) put:[:s :char | s nextMulti:#(($= #'<=') ($< #'<<' (($= #'<<=')))) after:char].

    actionArray at:($> codePoint) put:[:s :char | s nextMulti:#(($= #'>=') ($> #'>>' (($> #'>>>') ($= #'>>=') ))) after:char].

    actionArray at:($- codePoint) put:[:s :char | s nextMulti:#(($- #'--') ($= #'-=')) after:char].

    actionArray at:($+ codePoint) put:[:s :char | s nextMulti:#(($+ #'++') ($= #'+=')) after:char].

    actionArray at:($: codePoint) put:[:s :char | s nextMulti:#(($: #'::')) after:char].

    actionArray at:($* codePoint) put:[:s :char | s nextMulti:#(($= #'*=') ($* #'**' (($= #'**=')))) after:char].

    actionArray at:($/ codePoint) put:[:s :char | s nextSlash].

    actionArray at:($% codePoint) put:[:s :char | s nextMulti:#(($= #'%=')) after:char].

    actionArray at:($& codePoint) put:[:s :char | s nextMulti:#(($= #'&=') ($& #'&&')) after:char].

    actionArray at:($^ codePoint) put:[:s :char | s nextMulti:#(($= #'^=')) after:char].

    actionArray at:($| codePoint) put:[:s :char | s nextMulti:#(($= #'|=') ($| #'||')) after:char].

    actionArray at:($$ codePoint) put:[:s :char | s nextCharacter ].

    actionArray at:($# codePoint) put:[:s :char | s nextSymbol ].

    ActionArray := actionArray.

    TypeArray := typeArray.

    UnicodeActions := Dictionary new.

    "

     JavaScriptScanner setupActions

     JavaScriptParser setupActions

     JavaScriptCompiler setupActions

     JavaScriptSyntaxHighlighter setupActions

     self withAllSubclassesDo:#setupActions

    "

    "Created: / 14-05-1998 / 15:48:03 / cg"

    "Modified: / 17-05-1998 / 21:03:37 / cg"

    "Modified (comment): / 09-06-2019 / 17:04:10 / Claus Gittinger"

! !

!JavaScriptScanner class methodsFor:'class initialization'!

initialize

    SupportRegex := true.

    "/ super initialize. -- no need for that

! !

!JavaScriptScanner methodsFor:'accessing'!

tokenType

    ^ tokenType

! !

!JavaScriptScanner methodsFor:'initialization'!

initialize

    "initialize the scanner"

    super initialize.

    "/ have my own tables ...

    self allowUnderscoreInIdentifier:true.

    self allowDollarInIdentifier:false.

    self allowOldStyleAssignment:false.

    parserFlags allowNationalCharactersInIdentifier:false.

    parserFlags allowQualifiedNames:false.

    allowDegeneratedMantissa := true.     "/ something like 123.

    scanColonAsKeyword := false.

    actionArray := self class actionArray.

    typeArray := self class typeArray.

    "Created: / 14.5.1998 / 15:48:04 / cg"

    "Modified: / 26.10.1998 / 14:50:26 / cg"

! !

!JavaScriptScanner methodsFor:'private'!

checkForKeyword:string

    "check if string is a keyword (as opposed to an identifier)."

    |firstChar|

    firstChar := string at:1.

    (firstChar == $b) ifTrue:[

        (string = 'break') ifTrue:[tokenType := #break. ^true].

        ^ false

    ].

    (firstChar == $c) ifTrue:[

        (string = 'continue') ifTrue:[tokenType := #continue. ^true].

        (string = 'case') ifTrue:[tokenType := #case. ^true].

        (string = 'catch') ifTrue:[tokenType := #catch. ^true].

        (string = 'const') ifTrue:[

            parserFlags allowJavaScriptConst ifTrue:[

                tokenType := #const. ^true

            ].

        ].

        ^ false

    ].

    (firstChar == $d) ifTrue:[

        (string = 'default') ifTrue:[tokenType := #default. ^true].

        "/ do is not a keyword - it is also allowed as identifier

        ^ false

    ].

    (firstChar == $e) ifTrue:[

        (string = 'else') ifTrue:[tokenType := #else. ^true].

        ^ false

    ].

    (firstChar == $f) ifTrue:[

        (string = 'function') ifTrue:[tokenType := #function. ^true].

        (string = 'for') ifTrue:[tokenType := #for. ^true].

"/        (string = 'foreach') ifTrue:[tokenType := #foreach. ^true].

        (string = 'false') ifTrue:[tokenType := #false. tokenValue := false. ^true].

        (string = 'finally') ifTrue:[tokenType := #finally. tokenValue := false. ^true].

        ^ false

    ].

    (firstChar == $i) ifTrue:[

        (string = 'if') ifTrue:[tokenType := #if. ^true].

        "/ import is done as Identifier - not supported by 

        "/ (string = 'import') ifTrue:[tokenType := #import. ^true].

        ^ false

    ].

    (firstChar == $l) ifTrue:[

        (string = 'let') ifTrue:[tokenType := #let. ^true].

        ^ false

    ].

    (firstChar == $n) ifTrue:[

        (string = 'null') ifTrue:[tokenType := #null. tokenValue := nil. ^true].

        (string = 'nil') ifTrue:[tokenType := #null. tokenValue := nil. ^true].

        ^ false

    ].

    (firstChar == $r) ifTrue:[

        (string = 'return') ifTrue:[tokenType := #return. ^true].

        ^ false

    ].

    (firstChar == $s) ifTrue:[

        "/ (string = 'static') ifTrue:[tokenType := #static. ^true]. - handled as identifier

        "/ (string = 'synchronized') ifTrue:[tokenType := #synchronized. ^true].

        "/ (string = 'short') ifTrue:[tokenType := #short. ^true].

        (string = 'super') ifTrue:[tokenType := #super. ^true].

        (string = 'switch') ifTrue:[tokenType := #switch. ^true].

        ^ false

    ].

    (firstChar == $t) ifTrue:[

        (string = 'this') ifTrue:[tokenType := #this. ^true].

        (string = 'try') ifTrue:[tokenType := #try. ^true].

        (string = 'true') ifTrue:[tokenType := #true. tokenValue := true. ^true].

        (string = 'throw') ifTrue:[tokenType := #throw. ^true].

        "/ (string = 'transient') ifTrue:[tokenType := #transient. ^true].

        "/ (string = 'throws') ifTrue:[tokenType := #throws. ^true].

        (string = 'typeof') ifTrue:[tokenType := #typeof. ^true].

        ^ false

    ].

    (firstChar == $u) ifTrue:[

        (string = 'undefined') ifTrue:[tokenType := #null. tokenValue := nil. ^true].

        ^ false

    ].

    (firstChar == $v) ifTrue:[

        (string = 'var') ifTrue:[tokenType := #var. ^true].

        (string = 'void') ifTrue:[tokenType := #void. tokenValue := Void. ^true]. "/ - handled as identifier

        "/ (string = 'volatile') ifTrue:[tokenType := #volatile. ^true].

        ^ false

    ].

    (firstChar == $w) ifTrue:[

        (string = 'with') ifTrue:[tokenType := #with. ^true].

        (string = 'while') ifTrue:[tokenType := #while. ^true].

        ^ false

    ].

    ^ false

    "Created: / 14-05-1998 / 15:48:04 / cg"

    "Modified: / 09-08-2017 / 00:03:56 / cg"

    "Modified: / 31-08-2018 / 12:55:31 / Claus Gittinger"

!

handleCategoryDirective:categoryString

    "called when encountering a /** category: xxxx **/ comment;

     categoryString will be xxxx.

     Can be redefined in subclasses"

    "Created: / 26-10-2011 / 17:47:28 / cg"

!

handleCommentDirectivesIn:commentText

    "/ called for the text after the initial "/*"

    |plainText|

    (commentText endsWith:'**/') ifFalse:[^ self].

    (commentText startsWith:'* ') ifFalse:[^ self].

    plainText := commentText copyButLast:3.

    plainText := plainText copyFrom:3.

    (plainText startsWith:'category: ') ifTrue:[

        self handleCategoryDirective:(plainText copyFrom:'category: ' size+1) withoutSeparators.

        ^ self.

    ].

    "Created: / 26-10-2011 / 17:39:35 / cg"

!

isCommentCharacter:ch

    "return true, if ch is the comment-start character"

    ^ false

    "Created: / 14.5.1998 / 20:53:33 / cg"

!

previousTokenInitiatesRegexForSlash

    "read 

        https://www-archive.mozilla.org/js/language/js20-2002-04/rationale/syntax.html#regular-expressions

     for an explanation"

    ^ (#(

        nil

        '!!' '!!=' '!!=='

        '#'   '%'   '%='   

        '&'   '&&'   '&&='   '&='

        '('   

        '*'   '*='   

        '+'   '+='   

        ','   

        '-'   '-='   '->'   

        '.'   '..'   '...'   

        '/'   '/='   

        ':'   '::'   

        ';'   

        '<'   '<<'   '<<='   '<='   '='   '=='   '==='

        '>'   '>='   '>>'   '>>='   '>>>'   '>>>='   

        '?'   '@'   '['   '^'   '^='   '^^'   '^^='   

        '{'   '|'   '|='   '||'   '||='   '~'

        'abstract'   'break'   'case'   'catch'   'const'   'continue'

        'debugger'   'default'   'delete'   'do'   

        'else'   'enum'   'export'   'extends'   

        'final'   'finally'   'for'   'function'   

        'goto'   

        'if'   'implements'  'import'   'in'   'instanceof'   'interface'   'is'   

        'namespace'   'native'   'new'   'package'  

        'return'   'static'   'switch'   'synchronized'   

        'throw'   'throws'   'transient'   'try'   'typeof'   

        'use'   'var'   'volatile'   'while'   'with'

    ) includes:previousToken)

! !

!JavaScriptScanner methodsFor:'reading next token'!

hex2CharacterEscape

    |x1 x2|

    x1 := source next.

    x2 := source next.

    (x1 isNil or:[x2 isNil]) ifTrue:[

        self syntaxError:'unexpected end-of-input in String'

                position:tokenPosition to:(source position - 1).

        self markStringFrom:tokenPosition to:source position-1.

        ^ nil.

    ].

    ((x1 isDigitRadix:16) and:[(x2 isDigitRadix:16)]) ifFalse:[

        self syntaxError:'invalid hex-character escape'

                position:tokenPosition to:(source position - 1).

        ^ Character value:0

    ].

    ^ Character value:((x1 digitValue * 16) + (x2 digitValue))

    "Modified: / 26-10-2011 / 17:15:49 / cg"

!

hex4CharacterEscape

    |x1 x2 x3 x4|

    x1 := source next.

    x2 := source next.

    x3 := source next.

    x4 := source next.

    (x1 isNil or:[x2 isNil or:[x3 isNil or:[x4 isNil]]]) ifTrue:[

        self syntaxError:'unexpected end-of-input in String'

                position:tokenPosition to:(source position - 1).

        self markStringFrom:tokenPosition to:source position-1.

        ^ nil.

    ].

    ((x1 isDigitRadix:16) 

    and:[(x2 isDigitRadix:16)

    and:[(x3 isDigitRadix:16)

    and:[(x4 isDigitRadix:16)]]]) ifFalse:[

        self syntaxError:'invalid hex-character escape'

                position:tokenPosition to:(source position - 1).

        ^ Character value:0

    ].

    ^ Character value:(((((x1 digitValue * 16) + (x2 digitValue)) * 16) + (x3 digitValue)) * 16) + (x4 digitValue).

    "Modified: / 26-10-2011 / 17:15:45 / cg"

!

nextCharacter

    "this is a special (non-JavaScript) extension to support $'x' for character literals"

    |pos|

    pos := tokenPosition.

    source next.

    (hereChar := source peek) == $' ifTrue:[

        ^ self nextStringOrCharacter:true

    ].

    self syntaxError:'incorrect character constant (expected $''x'')'

            position:pos to:(source position).

    "Modified: / 15-06-2019 / 09:56:53 / Claus Gittinger"

!

nextMulti:list after:firstChar

    "a char has been read - peek ahead in list"

    |pc|

    peekChar isNil ifTrue:[

        source next.

    ] ifFalse:[

        peekChar := nil.

    ].

    pc := source peek.

    list do:[:spec |

        |ch tok idx chOrSelectorOrSpec|

        ch := spec at:1.

        pc == ch ifTrue:[

            tok := spec at:2.

            idx := 3.

            peekChar isNil ifTrue:[

                source next.