--- a/compiler/PPCTokenizingCodeGenerator.st	Thu Apr 30 23:43:14 2015 +0200
+++ b/compiler/PPCTokenizingCodeGenerator.st	Sun May 10 06:28:36 2015 +0100
@@ -11,52 +11,160 @@
 
 !PPCTokenizingCodeGenerator methodsFor:'visiting'!
 
-visitLLChoiceNode: node
-	| dictionary currentTokenVar |
-	dictionary := IdentityDictionary new.
-	
-	node children do: [ :child |
-		| firstSet |
-		firstSet := child firstSetSuchThat: [ :e | e isKindOf: PPCTokenNode ].
-		self assert: firstSet size = 1.
-		dictionary at: child 
-			put: firstSet anyOne.
-			
-	].
-	"Tokens are unique"
-	self assert: dictionary values asSet size = node children size.
-	
-	compiler addConstant: (dictionary values collect: [ :e | compiler idFor: e ])
-				as: #tokenMethods.
-	
-	currentTokenVar := compiler allocateTemporaryVariableNamed: 'currentToken'.
-	compiler codeAssign: 'self currentTokenType.' to: currentTokenVar.
-	node children do: [ :child |
-		| tokenMethodName |
-		tokenMethodName := compiler idFor: (dictionary at: child).
-		compiler add: currentTokenVar , ' = ',  tokenMethodName storeString.
-		compiler add: 'ifTrue: ['.
-		compiler codeStoreValueOf: [ self visit: child ] intoVariable: self retvalVar.
-		compiler codeReturn: self retvalVar.
-		compiler add: '].'
-	].
+visitChoiceNode: node
+"	true ifTrue: [ ^ super visitChoiceNode: node ]."
+    "HACK alert: We are inside token..."
+    node firstSetWithTokens detect: [ :e | e isTokenNode not ] ifFound: [ ^ super visitChoiceNode: node ].
+         
+    node children do: [ :child |
+        | tokenMethodName |
+        
+        child firstSetWithTokens do: [ :first |
+            "For each child, for each first compile this:"
+            tokenMethodName := compiler idFor: first.
+            compiler add: '(self currentTokenTypeIs: ', tokenMethodName storeString, ')'.
+            compiler addOnLine: ' ifTrue: ['.
+            compiler indent.
+            compiler codeStoreValueOf: [ self visit: child ] intoVariable: self retvalVar.
+            compiler add: 'error ifFalse: ['.
+            compiler indent.
+            compiler codeReturn: self retvalVar.
+            compiler dedent.
+            compiler add: '] ifTrue: ['.
+            compiler indent.
+            compiler codeClearError.
+            compiler codeAssign: 'nil.' to: 'currentTokenType'.
+            compiler add: 'context position: currentTokenValue start - 1.'.
+            compiler dedent.
+            compiler add: ']'.
+            compiler dedent.
+            compiler add: '].'
+        ]
+    ].
+
+    compiler codeError: 'no choice found'.
+!
 
-	compiler codeError: 'no choice found'.
+visitDeterministicChoiceNode: node
+    | dictionary |
+    dictionary := IdentityDictionary new.
+    
+    node children do: [ :child |
+        | firstSet |
+        firstSet := child firstSetWithTokens.
+        self assert: firstSet size = 1.
+        dictionary at: child 
+            put: firstSet anyOne.
+            
+    ].
+    "Tokens are unique"
+    self assert: dictionary values asSet size = node children size.
+    
+"	currentTokenVar := compiler allocateTemporaryVariableNamed: 'currentToken'.
+    compiler codeAssign: 'self currentTokenType.' to: currentTokenVar.
+"	node children do: [ :child |
+        | tokenMethodName |
+        tokenMethodName := compiler idFor: (dictionary at: child).
+        compiler add: '(self currentTokenTypeIs: ', tokenMethodName storeString, ')'.
+        compiler addOnLine: ' ifTrue: ['.
+        compiler indent.
+        compiler codeStoreValueOf: [ self visit: child ] intoVariable: self retvalVar.
+        compiler codeReturn: self retvalVar.
+        compiler dedent.
+        compiler add: '].'
+    ].
+
+    compiler codeError: 'no choice found'.
+!
+
+visitTokenChoiceNode: node
+    | trimmingToken |
+    self assert: (node children allSatisfy: [ :e | e isMarkedForInline not ]).
+    
+    
+    trimmingToken := node children detect: [ :e | e isTrimmingTokenNode ] ifNone: [ nil ].
+    trimmingToken isNil ifFalse: [ 
+        compiler codeStoreValueOf: [ self visit: trimmingToken whitespace ] intoVariable: #whatever.
+    ].
+    super visitChoiceNode: node.
 !
 
 visitTokenConsumeNode: node
-	compiler codeReturn: 'self consume: ', (compiler idFor: node child) storeString, '.'
+    compiler codeReturn: 'self consume: ', (compiler idFor: node child) storeString, '.'
 !
 
 visitTokenNode: node
-	| tokenType |
-	self assert: node isMarkedForInline.
+    | id startVar endVar |
+    startVar := compiler allocateTemporaryVariableNamed: 'start'.
+    endVar := compiler allocateTemporaryVariableNamed: 'end'.
+
+    id := compiler idFor: node.
+    compiler toTokenRememberStrategy.	
+    
+    compiler codeAssign: 'context position + 1.' to: startVar.
+    compiler codeStoreValueOf: [ self visit: node child ] intoVariable: #whatever.
+    compiler add: 'error ifFalse: [ '.
+    compiler indent.	
+        compiler codeAssign: 'context position.' to: endVar.
+    
+        compiler codeTranscriptShow: 'current token type: ', id storeString.
+        compiler codeAssign: id storeString, '.' to: 'currentTokenType'.
+        compiler codeAssign: node tokenClass asString, ' on: (context collection) 
+                                                                    start: ', startVar, '  
+                                                                    stop: ', endVar, '
+                                                                    value: nil.'
+                    to: 'currentTokenValue := ', self retvalVar.
+        compiler codeReturn.
+    compiler dedent.
+    compiler add: '].'.		
+    compiler toNormalRememberStrategy.
+!
+
+visitTokenizingParserNode: node
+    self visit: node tokenizer.
+    
+"	compiler codeNextToken."
+    compiler codeHaltIfShiftPressed.
+    compiler codeStoreValueOf: [ self visit: node parser ] intoVariable: self retvalVar.
+    compiler codeReturn.
+!
 
-	super visitTokenNode: node.
-	
-	tokenType := compiler idFor: node.
+visitTrimmingTokenNode: node
+    |  id  startVar endVar |
+    
+    startVar := compiler allocateTemporaryVariableNamed: 'start'.
+    endVar := compiler allocateTemporaryVariableNamed:  'end'.
+    
+    id := compiler idFor: node.
+    compiler toTokenRememberStrategy.
+    
+    
+    compiler addComment: 'Consume Whitespace:'.
+    compiler codeStoreValueOf: [ self visit: node whitespace ] intoVariable: #whatever.
+    compiler nl.
+
+    compiler codeAssign: 'context position + 1.' to: startVar.
+    compiler codeStoreValueOf: [ self visit: node child ] intoVariable: #whatever.
 
-	compiler codeAssign: tokenType storeString, '.' to: 'currentTokenType'.
-	compiler codeAssign: self retvalVar, '.' to: 'currentTokenValue'.
+    compiler add: 'error ifFalse: [ '.
+    compiler indent.	
+        compiler codeAssign: 'context position.' to: endVar.
+    
+        compiler addComment: 'Consume Whitespace:'.
+        compiler codeStoreValueOf: [ self visit: node whitespace ] intoVariable: #whatever.
+        compiler nl.
+    
+    
+        compiler codeTranscriptShow: 'current token type: ', id storeString.
+        compiler codeAssign: id storeString, '.' to: 'currentTokenType'.
+        compiler codeAssign: node tokenClass asString, ' on: (context collection) 
+                                                                start: ', startVar, ' 
+                                                                stop: ', endVar, '
+                                                                value: nil.'
+                   to: 'currentTokenValue := ', self retvalVar.
+        compiler codeReturn.
+    compiler dedent.																
+    compiler add: '].'	.
+    compiler toNormalRememberStrategy.
 ! !