--- a/compiler/PPCTokenCodeGenerator.st	Wed May 20 16:47:52 2015 +0100
+++ b/compiler/PPCTokenCodeGenerator.st	Thu May 21 14:35:34 2015 +0100
@@ -17,56 +17,135 @@
     return := super afterAccept: node retval: retval.
     return category: 'generated - tokens'.
     ^ return
+!
+
+fromTokenMode
+    compiler rememberStrategy: (PPCCompilerTokenizingRememberStrategy on: compiler).
+    compiler errorStrategy: (PPCCompilerTokenizingErrorStrategy on: compiler).
+!
+
+toTokenMode
+    compiler rememberStrategy: (PPCCompilerTokenRememberStrategy on: compiler).	
+    compiler errorStrategy: (PPCCompilerTokenErrorStrategy on: compiler).
 ! !
 
 !PPCTokenCodeGenerator methodsFor:'visiting'!
 
+visitOptionalNode: node
+    compiler codeStoreValueOf: [ self visit: node child ] intoVariable: self retvalVar.
+    compiler codeAssign: 'false.' to: 'error'.
+    compiler codeReturn.
+!
+
 visitTokenNode: node
-    | id startVar endVar |
+    | id startVar endVar  |
+    "Tokens cannot be inlined, 
+        - their result is true/false
+        - the return value is always stored in currentTokenValue
+        - the current token type is always stored in currentTokenType
+    "
+    self assert: node isMarkedForInline not.	
+    
     startVar := compiler allocateTemporaryVariableNamed: 'start'.
     endVar := compiler allocateTemporaryVariableNamed: 'end'.
 
     id := compiler idFor: node.
-    compiler rememberStrategy: (PPCCompilerTokenRememberStrategy on: compiler).	
+    self toTokenMode.
+
+    compiler add: 'currentTokenType isNil ifFalse: [ ^ currentTokenType == ', id storeString, '].'.	
+    compiler profileTokenRead: id.
+    
+    node allNodes size > 2 ifTrue: [ 
+        self addGuard: node ifTrue: nil  ifFalse: [ compiler addOnLine: '^ false' ].
+    ].
+
     
     compiler codeAssign: 'context position + 1.' to: startVar.
     compiler codeStoreValueOf: [ self visit: node child ] intoVariable: #whatever.
-    compiler add: 'error ifFalse: [ '.
-    compiler indent.	
-        compiler codeAssign: 'context position.' to: endVar.
+    compiler add: 'error ifTrue: [ ^ error := false ].'.
+
+    compiler codeAssign: 'context position.' to: endVar.
+
+    compiler codeTranscriptShow: 'current token type: ', id storeString.
+    compiler codeAssign: id storeString, '.' to: 'currentTokenType'.
+    compiler codeAssign: node tokenClass asString, ' on: (context collection) 
+                                                                start: ', startVar, '  
+                                                                stop: ', endVar, '
+                                                                value: nil.'
+                to: 'currentTokenValue := ', self retvalVar.
     
-        compiler codeTranscriptShow: 'current token type: ', id storeString.
-        compiler codeAssign: id storeString, '.' to: 'currentTokenType'.
-        compiler codeAssign: node tokenClass asString, ' on: (context collection) 
-                                                                    start: ', startVar, '  
-                                                                    stop: ', endVar, '
-                                                                    value: nil.'
-                    to: 'currentTokenValue := ', self retvalVar.
-        compiler codeReturn.
-    compiler dedent.
-    compiler add: '].'.		
-    compiler rememberStrategy: (PPCCompilerTokenizingRememberStrategy on: compiler).	
+        
+    compiler codeClearError.
+    compiler add: '^ true'.
+
+    self fromTokenMode.
+!
+
+visitTrimmingTokenCharacterNode: node
+    |  id     |
+
+    "Tokens cannot be inlined, 
+        - their result is true/false
+        - the return value is always stored in currentTokenValue
+        - the current token type is always stored in currentTokenType
+    "
+    self assert: node isMarkedForInline not.
+    
+    id := compiler idFor: node.
+    self toTokenMode.
+    
+    compiler add: 'currentTokenType isNil ifFalse: [ ^ currentTokenType == ', id storeString, '].'.
+    compiler profileTokenRead: id.
+
+    self addGuard: node ifTrue: nil ifFalse: [ compiler addOnLine: ' ^ false' ].
+
+    compiler add: 'context next.'.
+
+    compiler codeTranscriptShow: 'current token type: ', id storeString.
+    compiler codeAssign: id storeString, '.' to: 'currentTokenType'.
+    compiler codeAssign: node tokenClass asString, ' on: (context collection) 
+                                                            start: context position 
+                                                            stop: context position
+                                                            value: nil.'
+               to: 'currentTokenValue := ', self retvalVar.
+    
+    compiler addComment: 'Consume Whitespace:'.
+    compiler codeStoreValueOf: [ self visit: node whitespace ] intoVariable: #whatever.
+    compiler nl.
+    
+    compiler add: '^ true'.
+
+    self fromTokenMode.
 !
 
 visitTrimmingTokenNode: node
-    |  id  startVar endVar |
+    |  id  startVar endVar  |
+
+    "Tokens cannot be inlined, 
+        - their result is true/false
+        - the return value is always stored in currentTokenValue
+        - the current token type is always stored in currentTokenType
+    "
+    self assert: node isMarkedForInline not.
     
     startVar := compiler allocateTemporaryVariableNamed: 'start'.
     endVar := compiler allocateTemporaryVariableNamed:  'end'.
     
     id := compiler idFor: node.
-    compiler rememberStrategy: (PPCCompilerTokenRememberStrategy on: compiler).
-    
+    self toTokenMode.
     
-    compiler addComment: 'Consume Whitespace:'.
-    compiler codeStoreValueOf: [ self visit: node whitespace ] intoVariable: #whatever.
-    compiler nl.
+    compiler add: 'currentTokenType isNil ifFalse: [ ^ currentTokenType == ', id storeString, '].'.
+    compiler profileTokenRead: id.
+    
+    node allNodes size > 2 ifTrue: [ 
+        self addGuard: node ifTrue: nil  ifFalse: [ compiler addOnLine: '^ false' ].
+    ].
 
     compiler codeAssign: 'context position + 1.' to: startVar.
     compiler codeStoreValueOf: [ self visit: node child ] intoVariable: #whatever.
 
-    compiler add: 'error ifFalse: [ '.
-    compiler indent.	
+    compiler add: 'error ifTrue: [ ^ error := false ].'.
+
         compiler codeAssign: 'context position.' to: endVar.
     
         compiler addComment: 'Consume Whitespace:'.
@@ -81,10 +160,11 @@
                                                                 stop: ', endVar, '
                                                                 value: nil.'
                    to: 'currentTokenValue := ', self retvalVar.
-        compiler codeReturn.
-    compiler dedent.																
-    compiler add: '].'	.
-    compiler rememberStrategy: (PPCCompilerTokenizingRememberStrategy on: compiler).
+
+    compiler codeClearError.
+    compiler add: '^ true'.
+
+    self fromTokenMode.
 ! !
 
 !PPCTokenCodeGenerator class methodsFor:'documentation'!