"
stx:goodies/smallsense - A productivity plugin for Smalltalk/X IDE
Copyright (C) 2013-2014 Jan Vrany

This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License. 

This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
Lesser General Public License for more details.

You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
"
"{ Package: 'stx:goodies/smallsense' }"

"{ NameSpace: SmallSense }"

Object subclass:#TokenStream
	instanceVariableNames:'scanner scannerSourceReadLimit tokens last position full
		positionMax positionOfCursor caretTokenReported'
	classVariableNames:'BacklogSize'
	poolDictionaries:''
	category:'SmallSense-Utils-Matcher'
!

Object subclass:#Token
	instanceVariableNames:'type value startPosition endPosition'
	classVariableNames:''
	poolDictionaries:''
	privateIn:TokenStream
!

!TokenStream class methodsFor:'documentation'!

copyright
"
stx:goodies/smallsense - A productivity plugin for Smalltalk/X IDE
Copyright (C) 2013-2014 Jan Vrany

This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License. 

This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
Lesser General Public License for more details.

You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
"
!

documentation
"
    A custom read-only stream that return high-level tokens produced
    by underlying scanner object. The scanner must conform to API of
    Scanner class.

    Additionaly, when a cursor position is provided, TokenStream returns
    a virtual token with type #CARET.

    [author:]
        Jan Vrany <jan.vrany@fit.cvut.cz>

    [instance variables:]

    [class variables:]

    [see also:]
        Scanner
        TokenPatternMatcher.

"
! !

!TokenStream class methodsFor:'initialization'!

initialize
    "Invoked at system start or when the class is dynamically loaded."

    "/ please change as required (and remove this comment)

    BacklogSize := 100.

    "Modified: / 07-08-2013 / 01:47:46 / Jan Vrany <jan.vrany@fit.cvut.cz>"
! !

!TokenStream class methodsFor:'instance creation'!

on: scanner
    ^ self new initializeOn: scanner cursor: nil

    "Created: / 06-05-2014 / 15:25:41 / Jan Vrany <jan.vrany@fit.cvut.cz>"
    "Modified: / 15-05-2014 / 15:08:43 / Jan Vrany <jan.vrany@fit.cvut.cz>"
!

on: scanner cursor: cursor
    ^ self new initializeOn: scanner cursor: cursor

    "Created: / 15-05-2014 / 15:08:25 / Jan Vrany <jan.vrany@fit.cvut.cz>"
! !

!TokenStream methodsFor:'accessing'!

sourceStream
    ^ scanner sourceStream

    "Created: / 07-08-2013 / 01:44:15 / Jan Vrany <jan.vrany@fit.cvut.cz>"
! !

!TokenStream methodsFor:'initialization'!

initializeOn: scannerObject
    ^ self initializeOn: scannerObject cursor: nil

    "Created: / 06-05-2014 / 15:27:00 / Jan Vrany <jan.vrany@fit.cvut.cz>"
    "Modified: / 15-05-2014 / 15:08:05 / Jan Vrany <jan.vrany@fit.cvut.cz>"
!

initializeOn: scannerArg cursor: positionOfCursorArg
    scanner := scannerArg.
    positionOfCursor := positionOfCursorArg.
    tokens := Array new: BacklogSize.
    last := 0.
    full := false.
    position := 0.
    positionMax := 0.
    caretTokenReported := false.

    positionOfCursor notNil ifTrue:[ 
        scannerArg sourceStream isExternalStream ifTrue:[ 
            self error: 'Cursor reporting us supported only on internal streams'.
        ].
        scannerSourceReadLimit := scannerArg sourceStream readLimit.
        scannerArg sourceStream readLimit: (positionOfCursor min: scannerSourceReadLimit)
    ].

    "Created: / 15-05-2014 / 15:07:08 / Jan Vrany <jan.vrany@fit.cvut.cz>"
    "Modified: / 15-05-2014 / 17:55:49 / Jan Vrany <jan.vrany@fit.cvut.cz>"
! !

!TokenStream methodsFor:'stream api'!

atEnd
    ^ self peek type == #EOF

    "Created: / 14-03-2012 / 22:53:39 / Jan Vrany <jan.vrany@fit.cvut.cz>"
    "Modified: / 07-08-2013 / 01:49:12 / Jan Vrany <jan.vrany@fit.cvut.cz>"
!

contents
    ^ Array streamContents:[ :s | 
        [ self atEnd ] whileFalse:[ 
            s nextPut: self next
        ].
    ].

    "Created: / 15-05-2014 / 07:49:59 / Jan Vrany <jan.vrany@fit.cvut.cz>"
!

next
    | i t |
    position < positionMax ifTrue:[
        i := last - (positionMax - position) + 1.
        i <= 0 ifTrue:[
            i := tokens size + i.
        ].
        position := position + 1.
        ^ tokens at: i.
    ].
    scanner nextToken.
    t := scanner token.
    "/ Smalltalk / JavaScript scanners does not return token objects, sigh...
    (t isCharacter or:[ t isString ]) ifTrue:[ 
        t := Token new
                type: scanner tokenType;
                value: scanner token;
                startPosition: scanner tokenStartPosition;
                endPosition: scanner tokenEndPosition;        
                yourself
    ].
    t isNil ifTrue:[ 
        t := Token new
                type: #EOF;
                yourself
    ].
    t type == #EOF ifTrue:[ 
        (caretTokenReported not and:[scanner sourceStream position == positionOfCursor]) ifTrue:[ 
            t type: #CARET.
            t value: $|.
            t startPosition: positionOfCursor.
            t endPosition: positionOfCursor.
            caretTokenReported := true.
            scanner sourceStream readLimit: scannerSourceReadLimit.
        ].
    ].
    position := positionMax := positionMax + 1.
    last := last + 1.
    last > tokens size ifTrue:[ 
        full ifFalse:[ full := true ].
        last := 1 
    ].
    tokens at: last put: t.
    ^ t

    "Created: / 14-03-2012 / 22:53:21 / Jan Vrany <jan.vrany@fit.cvut.cz>"
    "Modified: / 15-05-2014 / 17:56:33 / Jan Vrany <jan.vrany@fit.cvut.cz>"
!

next: anInteger 
    "Answer up to anInteger elements of my collection. Overridden for efficiency."

    ^ self shouldNotImplement

    "Modified: / 06-05-2014 / 15:51:18 / Jan Vrany <jan.vrany@fit.cvut.cz>"
!

peek
    | t |

    t := self next.
    position := position - 1.
    ^ t

    "Modified: / 07-08-2013 / 01:12:03 / Jan Vrany <jan.vrany@fit.cvut.cz>"
!

position
    ^position

    "Created: / 14-03-2012 / 22:52:51 / Jan Vrany <jan.vrany@fit.cvut.cz>"
    "Modified: / 07-08-2013 / 01:06:23 / Jan Vrany <jan.vrany@fit.cvut.cz>"
!

position: anInteger
    (positionMax - anInteger) > (tokens size) ifTrue:[
        Stream positionErrorSignal raiseErrorString:'Not that much tokens in backlog!! You may try to increase it.'.
        ^ self.
    ].
    position := anInteger.

    "Modified: / 07-08-2014 / 11:46:21 / Jan Vrany <jan.vrany@fit.cvut.cz>"
!

position: tokenPosition sourcePosition: sourcePosition
    position := positionMax := tokenPosition.
    tokens := tokens class new: tokens size.
    full := false.

    scanner sourceStream position: sourcePosition.
    positionOfCursor notNil ifTrue:[
        sourcePosition < positionOfCursor ifTrue:[
            scanner sourceStream readLimit: (positionOfCursor min: scannerSourceReadLimit).
            caretTokenReported := false.
        ].
    ].

    "Created: / 13-06-2014 / 17:06:46 / Jan Vrany <jan.vrany@fit.cvut.cz>"
!

reset
    (positionMax) > (tokens size) ifTrue:[
        scanner sourceStream reset.    
        tokens := Array new: BacklogSize.
        last := 0.
        full := false.
        position := 0.
        positionMax := 0.
    ].
    position := 0.

    "Created: / 14-05-2014 / 16:19:08 / Jan Vrany <jan.vrany@fit.cvut.cz>"
    "Modified: / 07-08-2014 / 11:57:43 / Jan Vrany <jan.vrany@fit.cvut.cz>"
!

skipSeparators
    self halt.

    "Created: / 15-03-2012 / 10:35:28 / Jan Vrany <jan.vrany@fit.cvut.cz>"
    "Modified: / 07-08-2013 / 01:15:18 / Jan Vrany <jan.vrany@fit.cvut.cz>"
!

uncheckedPeek
    "An unchecked version of peek that throws an error if we try to peek over the end of the stream, even faster than #peek."

    ^ self peek

    "Modified: / 14-03-2012 / 22:46:22 / Jan Vrany <jan.vrany@fit.cvut.cz>"
! !

!TokenStream::Token methodsFor:'accessing'!

endPosition
    ^ endPosition
!

endPosition:something
    endPosition := something.
!

startPosition
    ^ startPosition
!

startPosition:something
    startPosition := something.
!

stopPosition
    ^ endPosition

    "Created: / 04-04-2013 / 00:32:50 / Jan Vrany <jan.vrany@fit.cvut.cz>"
!

type
    ^ type
!

type:something
    type := something.
!

value
    ^ value
!

value:something
    value := something.
! !

!TokenStream::Token methodsFor:'printing & storing'!

printOn:aStream
    "append a printed representation of the receiver to the argument, aStream"

    aStream nextPut: ${.
    type printOn:aStream.
    aStream nextPutAll:' - '.
    value printOn:aStream.
    aStream nextPutAll:' <'.
    startPosition printOn:aStream.
    aStream nextPutAll:'-'.
    endPosition printOn:aStream.
    aStream nextPutAll:'>}'.

    "Modified: / 06-05-2014 / 15:33:28 / Jan Vrany <jan.vrany@fit.cvut.cz>"
! !

!TokenStream class methodsFor:'documentation'!

version_CVS

    ^ '$Header$'
!

version_HG

    ^ '$Changeset: <not expanded> $'
! !


TokenStream initialize!