--- a/compiler/PPCTokenizingCodeGenerator.st	Sun May 10 06:28:36 2015 +0100
+++ b/compiler/PPCTokenizingCodeGenerator.st	Tue May 12 01:24:03 2015 +0100
@@ -3,18 +3,38 @@
 "{ NameSpace: Smalltalk }"
 
 PPCCodeGenerator subclass:#PPCTokenizingCodeGenerator
-	instanceVariableNames:''
+	instanceVariableNames:'tokenGenerator'
 	classVariableNames:''
 	poolDictionaries:''
 	category:'PetitCompiler-Visitors'
 !
 
+!PPCTokenizingCodeGenerator methodsFor:'accessing'!
+
+guards
+    "When tokenizing, do not use guards"
+    ^ false
+!
+
+tokenGenerator
+    ^ tokenGenerator isNil 
+        ifTrue: [ tokenGenerator := (PPCTokenCodeGenerator on: compiler)
+            arguments: arguments;
+            yourself ]
+        ifFalse: [ tokenGenerator ]
+!
+
+tokenGenerator: whatever
+    tokenGenerator := whatever
+! !
+
 !PPCTokenizingCodeGenerator methodsFor:'visiting'!
 
 visitChoiceNode: node
-"	true ifTrue: [ ^ super visitChoiceNode: node ]."
+"       true ifTrue: [ ^ super visitChoiceNode: node ]."
     "HACK alert: We are inside token..."
-    node firstSetWithTokens detect: [ :e | e isTokenNode not ] ifFound: [ ^ super visitChoiceNode: node ].
+
+    (node firstSetWithTokens contains: [ :e | e isTokenNode not ]) ifTrue: [ ^ super visitChoiceNode: node ].
          
     node children do: [ :child |
         | tokenMethodName |
@@ -43,6 +63,8 @@
     ].
 
     compiler codeError: 'no choice found'.
+
+    "Modified: / 10-05-2015 / 07:37:53 / Jan Vrany <jan.vrany@fit.cvut.cz>"
 !
 
 visitDeterministicChoiceNode: node
@@ -81,12 +103,16 @@
     | trimmingToken |
     self assert: (node children allSatisfy: [ :e | e isMarkedForInline not ]).
     
+    node children do: [ :child |
+        self tokenGenerator visit: child
+    ]
     
+"	
     trimmingToken := node children detect: [ :e | e isTrimmingTokenNode ] ifNone: [ nil ].
     trimmingToken isNil ifFalse: [ 
         compiler codeStoreValueOf: [ self visit: trimmingToken whitespace ] intoVariable: #whatever.
     ].
-    super visitChoiceNode: node.
+    super visitChoiceNode: node."
 !
 
 visitTokenConsumeNode: node
@@ -94,34 +120,12 @@
 !
 
 visitTokenNode: node
-    | id startVar endVar |
-    startVar := compiler allocateTemporaryVariableNamed: 'start'.
-    endVar := compiler allocateTemporaryVariableNamed: 'end'.
-
-    id := compiler idFor: node.
-    compiler toTokenRememberStrategy.	
-    
-    compiler codeAssign: 'context position + 1.' to: startVar.
-    compiler codeStoreValueOf: [ self visit: node child ] intoVariable: #whatever.
-    compiler add: 'error ifFalse: [ '.
-    compiler indent.	
-        compiler codeAssign: 'context position.' to: endVar.
-    
-        compiler codeTranscriptShow: 'current token type: ', id storeString.
-        compiler codeAssign: id storeString, '.' to: 'currentTokenType'.
-        compiler codeAssign: node tokenClass asString, ' on: (context collection) 
-                                                                    start: ', startVar, '  
-                                                                    stop: ', endVar, '
-                                                                    value: nil.'
-                    to: 'currentTokenValue := ', self retvalVar.
-        compiler codeReturn.
-    compiler dedent.
-    compiler add: '].'.		
-    compiler toNormalRememberStrategy.
+    self error: 'shoudl not happend!!'
 !
 
 visitTokenizingParserNode: node
     self visit: node tokenizer.
+    self visit: node whitespace.
     
 "	compiler codeNextToken."
     compiler codeHaltIfShiftPressed.
@@ -130,41 +134,6 @@
 !
 
 visitTrimmingTokenNode: node
-    |  id  startVar endVar |
-    
-    startVar := compiler allocateTemporaryVariableNamed: 'start'.
-    endVar := compiler allocateTemporaryVariableNamed:  'end'.
-    
-    id := compiler idFor: node.
-    compiler toTokenRememberStrategy.
-    
-    
-    compiler addComment: 'Consume Whitespace:'.
-    compiler codeStoreValueOf: [ self visit: node whitespace ] intoVariable: #whatever.
-    compiler nl.
-
-    compiler codeAssign: 'context position + 1.' to: startVar.
-    compiler codeStoreValueOf: [ self visit: node child ] intoVariable: #whatever.
-
-    compiler add: 'error ifFalse: [ '.
-    compiler indent.	
-        compiler codeAssign: 'context position.' to: endVar.
-    
-        compiler addComment: 'Consume Whitespace:'.
-        compiler codeStoreValueOf: [ self visit: node whitespace ] intoVariable: #whatever.
-        compiler nl.
-    
-    
-        compiler codeTranscriptShow: 'current token type: ', id storeString.
-        compiler codeAssign: id storeString, '.' to: 'currentTokenType'.
-        compiler codeAssign: node tokenClass asString, ' on: (context collection) 
-                                                                start: ', startVar, ' 
-                                                                stop: ', endVar, '
-                                                                value: nil.'
-                   to: 'currentTokenValue := ', self retvalVar.
-        compiler codeReturn.
-    compiler dedent.																
-    compiler add: '].'	.
-    compiler toNormalRememberStrategy.
+    self error: 'shoudl not happend!!'
 ! !