"{ Package: 'stx:libbasic2' }"

"

 COPYRIGHT (c) 1996 by eXept Software AG

	      All Rights Reserved

 This software is furnished under a license and may be used

 only in accordance with the terms of that license and with the

 inclusion of the above copyright notice.   This software may not

 be provided or otherwise made available to, or used by, any

 other person.  No title to or ownership of the software is

 hereby transferred.

"

ReadStream subclass:#TokenizedStream

	instanceVariableNames:'inputStream token tokenType tokenPosition tokenName tokenLineNr

		tokenValue tokenRadix hereChar peekChar peekChar2

		beginCommentCharacter endCommentCharacter eolCommentCharacter

		eolCharacter outStream outCol actions types eolIsSignificant

		allowFloatNumbers numbersAreSigned'

	classVariableNames:'DefaultActions DefaultTypes'

	poolDictionaries:''

	category:'Streams-Misc'

!

!TokenizedStream class methodsFor:'documentation'!

copyright

"

 COPYRIGHT (c) 1996 by eXept Software AG

	      All Rights Reserved

 This software is furnished under a license and may be used

 only in accordance with the terms of that license and with the

 inclusion of the above copyright notice.   This software may not

 be provided or otherwise made available to, or used by, any

 other person.  No title to or ownership of the software is

 hereby transferred.

"

!

documentation

"

    a first version of a tokenStream.

    This is still being constructed and will probably be enhanced

    and may finally help a simplified Scanner class.

    For now, it may be useful when textual input files are to be read and

    parsed. For example, ascii data files are often in a simple free form format

    which requires some little processing.

    Dont blame me, if its not powerful enough for your needs.

    operation:

	characters are read from a real input stream

	and the tokenizer dispatches to a token reading method by the help

	of an actionTable, which is indexed by the tokenType.

	The tokenType itself is aquired via another table via

	the characters ascii code.

	By default, the table is setup to only read numbers

	and identifiers.

	Whitespace is ignored, and all other characters return themself.

    The returned tokens are either symbols (#Identifier, #Integer ..) or

    characters ($+ $, etc.)

    If its #Identifier, the name is found in tokenName (there is an access method for that).

    If its #Integer or #Float, the value is found in tokenValue.

    EndOfLine is either ignored or returned as #EOL (if eolIsSignificant is true).

    End of input as #EOF.

    Unrecognized input leads to #Error to be returned.

    Customized tokenizers can be setup, by modifying the action- or typeTables.

    See examples for more.

    [author:]

	Claus Gittinger

    [see also:]

	ReadStream

	Scanner

	Tgen::*

"

!

examples

"

    simple example; tokenizing some string:

									[exBegin]

	|s|

	s := TokenizedStream on:'hello world, how much is 3 + 2'.

	[s atEnd] whileFalse:[

	    Transcript showCR:(s next).

	].

									[exEnd]

    simple example2; tokenizing and checking:

									[exBegin]

	|s token|

	s := TokenizedStream on:'foo bar baz  3 + 2'.

	[s atEnd] whileFalse:[

	    token := s next.

	    token == #Identifier ifTrue:[

		Transcript showCR:(token , ' name=' , s tokenName).

	    ] ifFalse:[

		token == #Integer ifTrue:[

		    Transcript showCR:(token , ' value=' , s tokenValue printString).

		] ifFalse:[

		    Transcript showCR:token.

		]

	    ]

	].

									[exEnd]

    reading simple expressions:

									[exBegin]

	|s num1 num2|

	s := TokenizedStream on:'

3 + 2

4 + 6

1 + 2

'.

	[s atEnd] whileFalse:[

	    s next == #Integer ifTrue:[

		num1 := s tokenValue.

		s next == $+ ifTrue:[

		    s next == #Integer ifTrue:[

			num2 := s tokenValue.

			Transcript showCR:num1 printString

					  , ' + '

					  , num2 printString

					  , ' => '

					  , (num1 + num2) printString.

		    ]

		]

	    ]

	].

									[exEnd]

    with eol-comments:

									[exBegin]

	|s num1 num2|

	s := TokenizedStream on:'

3 + 2

; this is a comment

4 + 6

1 + 2

'.

	s eolCommentCharacter:$;.

	[s atEnd] whileFalse:[

	    s next == #Integer ifTrue:[

		num1 := s tokenValue.

		s next == $+ ifTrue:[

		    s next == #Integer ifTrue:[

			num2 := s tokenValue.

			Transcript showCR:num1 printString

					  , ' + '

					  , num2 printString

					  , ' => '

					  , (num1 + num2) printString.

		    ]

		]

	    ]

	].

									[exEnd]

    allowing float & negative numbers (the default):

									[exBegin]

	|s|

	s := TokenizedStream on:'1.23 4.56 7 8 9 -5 5 -5.0 5.0'.

	[s atEnd] whileFalse:[

	    s next.

	    Transcript showCR:(s tokenType displayString, ' value=' , s tokenValue printString).

	].

									[exEnd]

    not allowing float numbers :

    (notice, how the floats is scanned)

									[exBegin]

	|s|

	s := TokenizedStream on:'1.23 4.56 7 8 9 -5 5 -5.0 5.0 '.

	s allowFloatNumbers:false.

	[s atEnd] whileFalse:[

	    s next.

	    Transcript showCR:(s tokenType displayString , ' value= ' , s tokenValue printString).

	].

									[exEnd]

    not allowing negative numbers :

    (notice, how the signs is scanned)

									[exBegin]

	|s|

	s := TokenizedStream on:'1.23 4.56 7 8 9 -5 5 -5.0 5.0'.

	s numbersAreSigned:false.

	[s atEnd] whileFalse:[

	    s next.

	    Transcript showCR:(s tokenType displayString , ' value= ' , s tokenValue printString).

	].

									[exEnd]

    no radix numbers (the default):

    (notice, how the integer is scanned)

									[exBegin]

	|s|

	s := TokenizedStream on:'1234 0x1234 16r1234'.

	[s atEnd] whileFalse:[

	    s next.

	    Transcript showCR:(s tokenType displayString , ' value= ' , s tokenValue printString , ' name=' , s tokenName displayString).

	].

									[exEnd]

    C-style radix numbers:

    (notice, how the ST-style integer is scanned)

									[exBegin]

	|s|

	s := TokenizedStream on:'0x1234 16r1234'.

	s actionTable at:#digit put:[:s :char | s nextCNumber].

	[s atEnd] whileFalse:[

	    s next.

	    Transcript showCR:(s tokenType displayString , ' value= ' , s tokenValue printString , ' name=' , s tokenName displayString).

	].

									[exEnd]

    smalltalk-style radix numbers:

    (notice, how the C-style integer is scanned)

									[exBegin]

	|s|

	s := TokenizedStream on:'0x1234 16r1234'.

	s actionTable at:#digit put:[:s :char | s nextSmalltalkNumber].

	[s atEnd] whileFalse:[

	    s next.

	    Transcript showCR:(s tokenType displayString , ' value= ' , s tokenValue printString , ' name=' , s tokenName displayString).

	].

									[exEnd]

    scan the '/etc/services' file:

									[exBegin]

	|s t service port protocol|

	s := TokenizedStream on:'/etc/services' asFilename readStream.

	s eolCommentCharacter:$#.

	s typeTable at:($- asciiValue) put:#letter.

	[s atEnd] whileFalse:[

	    t := s next.

	    t == #Identifier ifTrue:[

		service := s tokenName.

		t := s next.

		t == #Integer ifTrue:[

		    port := s tokenValue.

		    s next == $/ ifTrue:[

			t := s next.

			t == #Identifier ifTrue:[

			    protocol := s tokenName.

			    Transcript showCR:('servive: ' , service , ' is ' , protocol , ' port=' , port printString).

			]

		    ]

		]

	    ].

	    s skipToEol

	]

									[exEnd]

"

! !

!TokenizedStream class methodsFor:'initialization'!

initialize

    DefaultActions := IdentityDictionary new.

    DefaultTypes := Array new:256.

    "kludge: action is nextColonOrAssign, but type is special"

    2 to:255 do:[:code |

	DefaultTypes at:code put:(Character value:code).

    ].

    ($0 asciiValue) to:($9 asciiValue) do:[:index |

	DefaultTypes at:index put:#digit.

    ].

    ($a asciiValue) to:($z asciiValue) do:[:index |

	DefaultTypes at:index put:#letter.

    ].

    ($A asciiValue) to:($Z asciiValue) do:[:index |

	DefaultTypes at:index put:#letter.

    ].

    DefaultActions at:#letter put:[:s :char | s nextIdentifier].

    DefaultActions at:#digit  put:[:s :char | s nextNumber].

    DefaultActions at:$-  put:[:s :char | s nextSignedNumber].

    "

     TokenizedStream initialize

    "

! !

!TokenizedStream class methodsFor:'instance creation'!

new

    ^ self basicNew initialize

    "Modified: 11.1.1997 / 19:18:48 / cg"

    "Created: 11.1.1997 / 19:22:40 / cg"

!

on:aStream

    ^ self readingFrom:(aStream readStream)

    "Modified: 11.1.1997 / 19:23:49 / cg"

!

readingFrom:aStream

    ^ self new inputStream:aStream

    "Created: 11.1.1997 / 19:18:58 / cg"

    "Modified: 11.1.1997 / 19:23:17 / cg"

! !

!TokenizedStream methodsFor:'accessing'!

actionTable

    ^ actions

    "Created: 1.2.1996 / 17:42:00 / cg"

!

allowFloatNumbers:aBoolean

    "if false, floating numbers are not read; a period is returned as

     a separate token. If true (the default), floating point numbers are allowed."

    allowFloatNumbers := aBoolean

    "Modified: 1.2.1996 / 18:14:27 / cg"

    "Created: 1.2.1996 / 18:27:41 / cg"

!

beginCommentCharacter:aCharacter

    beginCommentCharacter := aCharacter

    "Created: 1.2.1996 / 17:38:01 / cg"

!

endCommentCharacter:aCharacter

    endCommentCharacter := aCharacter

    "Created: 1.2.1996 / 17:38:06 / cg"

!

eolCommentCharacter:aCharacter

    eolCommentCharacter := aCharacter

    "Created: 1.2.1996 / 17:37:51 / cg"

!

eolIsSignificant:aBoolean

    "if false, EOL is treated like whiteSpace (the default);

     Otherwise, a #EOL token is returned"

    eolIsSignificant := aBoolean

    "Created: 1.2.1996 / 17:37:51 / cg"

    "Modified: 1.2.1996 / 18:14:27 / cg"

!

numbersAreSigned:aBoolean

    "if false, minus signs preceeding numbers are ignored.

     if true (the default) they are recognized"

    numbersAreSigned := aBoolean

    "Modified: 1.2.1996 / 18:14:27 / cg"

    "Created: 1.2.1996 / 18:56:34 / cg"

!

tokenName

    ^ tokenName

    "Created: 1.2.1996 / 17:46:48 / cg"

!

tokenType

    ^ tokenType

    "Created: 1.2.1996 / 17:26:24 / cg"

!

tokenValue

    ^ tokenValue

    "Created: 1.2.1996 / 17:26:30 / cg"

!

typeTable

    ^ types

    "Created: 1.2.1996 / 17:41:54 / cg"

! !

!TokenizedStream methodsFor:'initialization'!

initialize

    tokenLineNr := 1.

    eolCommentCharacter := beginCommentCharacter := endCommentCharacter := nil.

    eolCharacter := Character cr.

    eolIsSignificant := false.

    actions := DefaultActions shallowCopy.

    types := DefaultTypes shallowCopy.

    allowFloatNumbers := true.

    numbersAreSigned := true.

    "Modified: 11.1.1997 / 19:25:49 / cg"

!

inputStream:aStream

    inputStream := aStream

    "Created: 11.1.1997 / 19:25:41 / cg"

! !

!TokenizedStream methodsFor:'private'!

on:aStream

    self initialize.

    inputStream := aStream.

    "Modified: 11.1.1997 / 19:22:10 / cg"

! !

!TokenizedStream methodsFor:'reading'!

next

    ^ self nextToken

    "Created: 1.2.1996 / 17:21:47 / cg"

!

nextCNumber

    |nextChar value s|

    tokenRadix := 10.

    inputStream peek == $0 ifTrue:[

	inputStream next.

	inputStream peek == $x ifTrue:[

	    inputStream next.

	    tokenRadix := 16.

	] ifFalse:[

	    tokenRadix := 8

	]

    ].

    value := Integer readFrom:inputStream radix:tokenRadix.

    nextChar := inputStream peek.

    (allowFloatNumbers and:[tokenRadix == 10]) ifTrue:[

	(nextChar == $.) ifTrue:[

	    nextChar := inputStream nextPeek.