--- a/compiler/PPCTokenizingCodeGenerator.st	Mon May 11 18:31:26 2015 +0100
+++ b/compiler/PPCTokenizingCodeGenerator.st	Tue May 12 01:33:33 2015 +0100
@@ -3,12 +3,31 @@
 "{ NameSpace: Smalltalk }"
 
 PPCCodeGenerator subclass:#PPCTokenizingCodeGenerator
-	instanceVariableNames:''
+	instanceVariableNames:'tokenGenerator'
 	classVariableNames:''
 	poolDictionaries:''
 	category:'PetitCompiler-Visitors'
 !
 
+!PPCTokenizingCodeGenerator methodsFor:'accessing'!
+
+guards
+    "When tokenizing, do not use guards"
+    ^ false
+!
+
+tokenGenerator
+    tokenGenerator isNil ifTrue: [ 
+        tokenGenerator := PPCTokenCodeGenerator on: compiler.
+        tokenGenerator arguments: arguments.
+    ].
+    ^ tokenGenerator
+!
+
+tokenGenerator: whatever
+    tokenGenerator := whatever
+! !
+
 !PPCTokenizingCodeGenerator methodsFor:'visiting'!
 
 visitChoiceNode: node
@@ -84,12 +103,16 @@
     | trimmingToken |
     self assert: (node children allSatisfy: [ :e | e isMarkedForInline not ]).
     
+    node children do: [ :child |
+        self tokenGenerator visit: child
+    ]
     
+"	
     trimmingToken := node children detect: [ :e | e isTrimmingTokenNode ] ifNone: [ nil ].
     trimmingToken isNil ifFalse: [ 
         compiler codeStoreValueOf: [ self visit: trimmingToken whitespace ] intoVariable: #whatever.
     ].
-    super visitChoiceNode: node.
+    super visitChoiceNode: node."
 !
 
 visitTokenConsumeNode: node
@@ -97,34 +120,12 @@
 !
 
 visitTokenNode: node
-    | id startVar endVar |
-    startVar := compiler allocateTemporaryVariableNamed: 'start'.
-    endVar := compiler allocateTemporaryVariableNamed: 'end'.
-
-    id := compiler idFor: node.
-    compiler toTokenRememberStrategy.	
-    
-    compiler codeAssign: 'context position + 1.' to: startVar.
-    compiler codeStoreValueOf: [ self visit: node child ] intoVariable: #whatever.
-    compiler add: 'error ifFalse: [ '.
-    compiler indent.	
-        compiler codeAssign: 'context position.' to: endVar.
-    
-        compiler codeTranscriptShow: 'current token type: ', id storeString.
-        compiler codeAssign: id storeString, '.' to: 'currentTokenType'.
-        compiler codeAssign: node tokenClass asString, ' on: (context collection) 
-                                                                    start: ', startVar, '  
-                                                                    stop: ', endVar, '
-                                                                    value: nil.'
-                    to: 'currentTokenValue := ', self retvalVar.
-        compiler codeReturn.
-    compiler dedent.
-    compiler add: '].'.		
-    compiler toNormalRememberStrategy.
+    self error: 'shoudl not happend!!'
 !
 
 visitTokenizingParserNode: node
     self visit: node tokenizer.
+    self visit: node whitespace.
     
 "	compiler codeNextToken."
     compiler codeHaltIfShiftPressed.
@@ -133,41 +134,6 @@
 !
 
 visitTrimmingTokenNode: node
-    |  id  startVar endVar |
-    
-    startVar := compiler allocateTemporaryVariableNamed: 'start'.
-    endVar := compiler allocateTemporaryVariableNamed:  'end'.
-    
-    id := compiler idFor: node.
-    compiler toTokenRememberStrategy.
-    
-    
-    compiler addComment: 'Consume Whitespace:'.
-    compiler codeStoreValueOf: [ self visit: node whitespace ] intoVariable: #whatever.
-    compiler nl.
-
-    compiler codeAssign: 'context position + 1.' to: startVar.
-    compiler codeStoreValueOf: [ self visit: node child ] intoVariable: #whatever.
-
-    compiler add: 'error ifFalse: [ '.
-    compiler indent.	
-        compiler codeAssign: 'context position.' to: endVar.
-    
-        compiler addComment: 'Consume Whitespace:'.
-        compiler codeStoreValueOf: [ self visit: node whitespace ] intoVariable: #whatever.
-        compiler nl.
-    
-    
-        compiler codeTranscriptShow: 'current token type: ', id storeString.
-        compiler codeAssign: id storeString, '.' to: 'currentTokenType'.
-        compiler codeAssign: node tokenClass asString, ' on: (context collection) 
-                                                                start: ', startVar, ' 
-                                                                stop: ', endVar, '
-                                                                value: nil.'
-                   to: 'currentTokenValue := ', self retvalVar.
-        compiler codeReturn.
-    compiler dedent.																
-    compiler add: '].'	.
-    compiler toNormalRememberStrategy.
+    self error: 'shoudl not happend!!'
 ! !