--- a/SmallSense__TokenStream.st	Thu May 15 14:41:52 2014 +0100
+++ b/SmallSense__TokenStream.st	Thu May 15 17:25:42 2014 +0100
@@ -3,7 +3,8 @@
 "{ NameSpace: SmallSense }"
 
 Object subclass:#TokenStream
-	instanceVariableNames:'scanner tokens last position full positionMax'
+	instanceVariableNames:'scanner scannerSourceReadLimit tokens last position full
+		positionMax positionOfCursor'
 	classVariableNames:'BacklogSize'
 	poolDictionaries:''
 	category:'SmallSense-Utils-Matcher'
@@ -16,6 +17,31 @@
 	privateIn:TokenStream
 !
 
+!TokenStream class methodsFor:'documentation'!
+
+documentation
+"
+    A custom read-only stream that return high-level tokens produced
+    by underlying scanner object. The scanner must conform to API of
+    Scanner class.
+
+    Additionaly, when a cursor position is provided, TokenStream returns
+    a virtual token with type #CARET.
+
+    [author:]
+        Jan Vrany <jan.vrany@fit.cvut.cz>
+
+    [instance variables:]
+
+    [class variables:]
+
+    [see also:]
+        Scanner
+        TokenPatternMatcher.
+
+"
+! !
+
 !TokenStream class methodsFor:'initialization'!
 
 initialize
@@ -31,9 +57,16 @@
 !TokenStream class methodsFor:'instance creation'!
 
 on: scanner
-    ^ self new initializeOn: scanner
+    ^ self new initializeOn: scanner cursor: nil
 
     "Created: / 06-05-2014 / 15:25:41 / Jan Vrany <jan.vrany@fit.cvut.cz>"
+    "Modified: / 15-05-2014 / 15:08:43 / Jan Vrany <jan.vrany@fit.cvut.cz>"
+!
+
+on: scanner cursor: cursor
+    ^ self new initializeOn: scanner cursor: cursor
+
+    "Created: / 15-05-2014 / 15:08:25 / Jan Vrany <jan.vrany@fit.cvut.cz>"
 ! !
 
 !TokenStream methodsFor:'* As yet uncategorized *'!
@@ -59,14 +92,31 @@
 !TokenStream methodsFor:'initialization'!
 
 initializeOn: scannerObject
-    scanner := scannerObject.
+    ^ self initializeOn: scannerObject cursor: nil
+
+    "Created: / 06-05-2014 / 15:27:00 / Jan Vrany <jan.vrany@fit.cvut.cz>"
+    "Modified: / 15-05-2014 / 15:08:05 / Jan Vrany <jan.vrany@fit.cvut.cz>"
+!
+
+initializeOn: scannerArg cursor: positionOfCursorArg
+    scanner := scannerArg.
+    positionOfCursor := positionOfCursorArg.
     tokens := Array new: BacklogSize.
     last := 0.
     full := false.
     position := 0.
     positionMax := 0.
 
-    "Created: / 06-05-2014 / 15:27:00 / Jan Vrany <jan.vrany@fit.cvut.cz>"
+    positionOfCursor notNil ifTrue:[ 
+        scannerArg sourceStream isExternalStream ifTrue:[ 
+            self error: 'Cursor reporting us supported only on internal streams'.
+        ].
+        scannerSourceReadLimit := scannerArg sourceStream readLimit.
+        scannerArg sourceStream readLimit: (positionOfCursor min: scannerSourceReadLimit)
+    ].
+
+    "Created: / 15-05-2014 / 15:07:08 / Jan Vrany <jan.vrany@fit.cvut.cz>"
+    "Modified: / 15-05-2014 / 16:07:27 / Jan Vrany <jan.vrany@fit.cvut.cz>"
 ! !
 
 !TokenStream methodsFor:'stream api'!
@@ -104,6 +154,14 @@
                 type: #EOF;
                 yourself
     ].
+    t type == #EOF ifTrue:[ 
+        scanner sourceStream position == positionOfCursor ifTrue:[ 
+            t type: #CARET.
+            t startPosition: positionOfCursor.
+            t endPosition: positionOfCursor.
+            scanner sourceStream readLimit: scannerSourceReadLimit.
+        ].
+    ].
     position := positionMax := positionMax + 1.
     last := last + 1.
     last > tokens size ifTrue:[ 
@@ -114,7 +172,7 @@
     ^ t
 
     "Created: / 14-03-2012 / 22:53:21 / Jan Vrany <jan.vrany@fit.cvut.cz>"
-    "Modified: / 06-05-2014 / 15:57:16 / Jan Vrany <jan.vrany@fit.cvut.cz>"
+    "Modified: / 15-05-2014 / 16:24:25 / Jan Vrany <jan.vrany@fit.cvut.cz>"
 !
 
 next: anInteger 
@@ -151,6 +209,12 @@
     "Modified: / 07-08-2013 / 01:34:08 / Jan Vrany <jan.vrany@fit.cvut.cz>"
 !
 
+reset
+    self position: 0.
+
+    "Created: / 14-05-2014 / 16:19:08 / Jan Vrany <jan.vrany@fit.cvut.cz>"
+!
+
 skipSeparators
     self halt.