compiler/PPCTokenizingVisitor.st
changeset 515 b5316ef15274
parent 502 1e45d3c96ec5
child 516 3b81c9e53352
child 524 f6f68d32de73
--- a/compiler/PPCTokenizingVisitor.st	Fri Jul 24 15:06:54 2015 +0100
+++ b/compiler/PPCTokenizingVisitor.st	Mon Aug 17 12:13:16 2015 +0100
@@ -13,41 +13,44 @@
 
 afterAccept: node retval: parserNode
     self isRoot ifTrue: [ 
-        | tokenizerNode whitespaceNode |
+        | tokensNode whitespaceNode |
         self change.
-        tokens addLast: self eofToken.
-        tokens do: [ :token | token unmarkForInline  ].
+"		tokens addLast: self eofToken."
+        tokens do: [ :token | token unmarkForInline ].
+        whitespaceNode := tokens detect: [ :e | e isTrimmingTokenNode ] ifNone: [ nil ].
+        whitespaceNode notNil ifTrue: [ 
+                whitespaceNode := whitespaceNode whitespace copy
+                    unmarkForInline;
+                    name: 'consumeWhitespace';
+                    yourself.
+                "whitespaceNode := PPCTokenWhitespaceNode new
+                    child: whitespaceNode;
+                    yourself"
+        ] ifFalse: [ 
+                whitespaceNode := PPCNilNode new
+                    name: 'consumeWhitespace';
+                    yourself 
+        ].
         
-        whitespaceNode := tokens detect: [ :e | e isTrimmingTokenNode ] ifNone:[nil]. 
-        whitespaceNode notNil ifTrue:[
-            whitespaceNode := whitespaceNode whitespace copy
-                                unmarkForInline;
-                                name: 'consumeWhitespace';
-                                yourself 
-        ] ifFalse:[
-            whitespaceNode := (PPCNilNode new)
-                                name: 'consumeWhitespace';
-                                yourself
-        ].        
-        tokenizerNode := PPCTokenChoiceNode new
+        tokensNode := PPCListNode new
             children: tokens asArray;
             name: 'nextToken';
             yourself.
-    
+            
         ^ PPCTokenizingParserNode new
             parser: parserNode;
-            tokenizer: tokenizerNode;
             whitespace: whitespaceNode;
-            name: #'mainParser';
-            yourself
-    ].
+            tokens: tokensNode;
+            name: #mainParser;
+            yourself ].
     ^ parserNode
-
+    
     "Modified: / 12-05-2015 / 01:37:57 / Jan Vrany <jan.vrany@fit.cvut.cz>"
 !
 
 eofToken
     | ws  |
+    self error: 'deprecated?'.
     ws := PPCStarNode new
         child: (PPCMessagePredicateNode new
             message: #isSeparator;
@@ -58,6 +61,7 @@
         child: PPCEndOfFileNode new;
         whitespace: ws;
         tokenClass: PPToken;
+        name: 'eof'
         yourself.
 ! !
 
@@ -86,6 +90,7 @@
 
 visitActionNode: node
     (node hasProperty: #trimmingToken) ifTrue: [ 
+        self halt: 'can this happen?'.
         self change.
         self addToken: node.
         
@@ -97,10 +102,21 @@
     ^ super visitActionNode: node
 !
 
+visitTokenConsumeNode: node
+    "
+        Seems, it might happen, that if I create the consume node, 
+        I will ge to it later. This would create a token consume node for the
+        child, thus having tokenConsumNode with tokenConsumNode as a child...
+    "
+    ^ node
+!
+
 visitTokenNode: node
     self change.
     self addToken: node.
     
+    self assert: node acceptsEpsilon not description: 'Sorry, but the epsilon tokens are not allowed'.
+    
     ^ PPCTokenConsumeNode new
         child: node;
         yourself.
@@ -112,6 +128,7 @@
     
     ^ PPCTokenConsumeNode new
         child: node;
+        name: node name;
         yourself.
 ! !