compiler/PPCTokenizingVisitor.st
changeset 502 1e45d3c96ec5
parent 459 4751c407bb40
child 503 ff58cd9f1f3c
child 515 b5316ef15274
equal deleted inserted replaced
464:f6d77fee9811 502:1e45d3c96ec5
    16         | tokenizerNode whitespaceNode |
    16         | tokenizerNode whitespaceNode |
    17         self change.
    17         self change.
    18         tokens addLast: self eofToken.
    18         tokens addLast: self eofToken.
    19         tokens do: [ :token | token unmarkForInline  ].
    19         tokens do: [ :token | token unmarkForInline  ].
    20         
    20         
    21         whitespaceNode := tokens detect: [ :e | e isTrimmingTokenNode ] ifFound: [:token | 
    21         whitespaceNode := tokens detect: [ :e | e isTrimmingTokenNode ] ifNone:[nil]. 
    22                 token whitespace copy
    22         whitespaceNode notNil ifTrue:[
    23                     unmarkForInline;
    23             whitespaceNode := whitespaceNode whitespace copy
    24                     name: 'consumeWhitespace';
    24                                 unmarkForInline;
    25                     yourself 
    25                                 name: 'consumeWhitespace';
    26             ] ifNone: [
    26                                 yourself 
    27          		PPCNilNode new
    27         ] ifFalse:[
    28                     name: 'consumeWhitespace';
    28             whitespaceNode := (PPCNilNode new)
    29                     yourself
    29                                 name: 'consumeWhitespace';
    30             ].
    30                                 yourself
    31         
    31         ].        
    32         tokenizerNode := PPCTokenChoiceNode new
    32         tokenizerNode := PPCTokenChoiceNode new
    33             children: tokens asArray;
    33             children: tokens asArray;
    34             name: 'nextToken';
    34             name: 'nextToken';
    35             yourself.
    35             yourself.
    36     
    36     
    40             whitespace: whitespaceNode;
    40             whitespace: whitespaceNode;
    41             name: #'mainParser';
    41             name: #'mainParser';
    42             yourself
    42             yourself
    43     ].
    43     ].
    44     ^ parserNode
    44     ^ parserNode
    45     
    45 
       
    46     "Modified: / 12-05-2015 / 01:37:57 / Jan Vrany <jan.vrany@fit.cvut.cz>"
    46 !
    47 !
    47 
    48 
    48 eofToken
    49 eofToken
    49     | ws  |
    50     | ws  |
    50     ws := PPCStarNode new
    51     ws := PPCStarNode new