--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/compiler/PPCTokenizingVisitor.st	Sun May 10 06:28:36 2015 +0100
@@ -0,0 +1,105 @@
+"{ Package: 'stx:goodies/petitparser/compiler' }"
+
+"{ NameSpace: Smalltalk }"
+
+PPCRewritingVisitor subclass:#PPCTokenizingVisitor
+	instanceVariableNames:'tokens'
+	classVariableNames:''
+	poolDictionaries:''
+	category:'PetitCompiler-Visitors'
+!
+
+!PPCTokenizingVisitor methodsFor:'hooks'!
+
+afterAccept: node retval: retval
+    Halt if: [ node name = 'start' ].
+    self isRoot ifTrue: [ 
+        | tokenizerNode |
+        self change.
+        tokens addLast: self eofToken.
+        tokens do: [ :token | token unmarkForInline  ].
+        
+        tokenizerNode := PPCTokenChoiceNode new
+            children: tokens asArray;
+            name: 'nextToken';
+            yourself.
+    
+        ^ PPCTokenizingParserNode new
+            parser: retval;
+            tokenizer: tokenizerNode;
+            name: #'mainParser';
+            yourself
+    ].
+    ^ retval
+    
+!
+
+eofToken
+    | ws  |
+    ws := PPCStarNode new
+        child: (PPCMessagePredicateNode new
+            message: #isSeparator;
+            yourself);
+        yourself.
+    
+    ^ PPCTrimmingTokenNode new
+        child: PPCEndOfFileNode new;
+        whitespace: ws;
+        tokenClass: PPToken;
+        yourself.
+! !
+
+!PPCTokenizingVisitor methodsFor:'initialization'!
+
+initialize
+    super initialize.
+    tokens := OrderedCollection new.
+! !
+
+!PPCTokenizingVisitor methodsFor:'testing'!
+
+isRoot
+    ^ openSet size = 1
+! !
+
+!PPCTokenizingVisitor methodsFor:'tokens'!
+
+addToken: token
+    (tokens contains: [:e | e == token] ) ifFalse: [ 
+        tokens addLast: token
+    ]
+! !
+
+!PPCTokenizingVisitor methodsFor:'visiting'!
+
+visitActionNode: node
+    (node hasProperty: #trimmingToken) ifTrue: [ 
+        self change.
+        self addToken: node.
+        
+        ^ PPCTokenConsumeNode new
+            child: node;
+            yourself	
+    ].
+
+    ^ super visitActionNode: node
+!
+
+visitTokenNode: node
+    self change.
+    self addToken: node.
+    
+    ^ PPCTokenConsumeNode new
+        child: node;
+        yourself.
+!
+
+visitTrimmingTokenNode: node
+    self change.
+    self addToken: node.
+    
+    ^ PPCTokenConsumeNode new
+        child: node;
+        yourself.
+! !
+