diff -r 99b850002359 -r f8c6732b927c TokenizedStream.st --- a/TokenizedStream.st Thu Feb 01 18:14:57 1996 +0100 +++ b/TokenizedStream.st Thu Feb 01 18:44:48 1996 +0100 @@ -2,7 +2,8 @@ instanceVariableNames:'source token tokenType tokenPosition tokenName tokenLineNr tokenValue tokenRadix hereChar peekChar peekChar2 beginCommentCharacter endCommentCharacter eolCommentCharacter - eolCharacter outStream outCol actions types eolIsSignificant' + eolCharacter outStream outCol actions types eolIsSignificant + allowFloatNumbers' classVariableNames:'DefaultActions DefaultTypes' poolDictionaries:'' category:'Streams' @@ -123,8 +124,69 @@ ] ]. + allowing float numbers (the default): - scan /etc/services file: + |s| + + s := TokenizedStream on:'1.23 4.56 7 8 9'. + [s atEnd] whileFalse:[ + s next. + Transcript showCr:(s tokenType displayString, ' value=' , s tokenValue printString). + ]. + + + not allowing float numbers : + + |s| + + s := TokenizedStream on:'1.23 4.56 7 8 9'. + s allowFloatNumbers:false. + + [s atEnd] whileFalse:[ + s next. + Transcript showCr:(s tokenType displayString , ' value= ' , s tokenValue printString). + ]. + + + no radix numbers (the default): + + |s| + + s := TokenizedStream on:'0x1234 16r1234'. + + [s atEnd] whileFalse:[ + s next. + Transcript showCr:(s tokenType displayString , ' value= ' , s tokenValue printString , ' name=' , s tokenName displayString). + ]. + + + C-style radix numbers: + + |s| + + s := TokenizedStream on:'0x1234 16r1234'. + s actionTable at:#digit put:[:s :char | s nextCNumber]. + + [s atEnd] whileFalse:[ + s next. + Transcript showCr:(s tokenType displayString , ' value= ' , s tokenValue printString , ' name=' , s tokenName displayString). + ]. + + + smalltalk-style radix numbers: + + |s| + + s := TokenizedStream on:'0x1234 16r1234'. + s actionTable at:#digit put:[:s :char | s nextSmalltalkNumber]. + + [s atEnd] whileFalse:[ + s next. + Transcript showCr:(s tokenType displayString , ' value= ' , s tokenValue printString , ' name=' , s tokenName displayString). + ]. + + + scan the '/etc/services' file: |s t service port protocol| @@ -156,9 +218,7 @@ !TokenizedStream class methodsFor:'initialization'! initialize - |block| - - DefaultActions := Array new:256. + DefaultActions := IdentityDictionary new. DefaultTypes := Array new:256. "kludge: action is nextColonOrAssign, but type is special" @@ -166,22 +226,20 @@ DefaultTypes at:code put:(Character value:code). ]. - block := [:s :char | s nextInteger]. ($0 asciiValue) to:($9 asciiValue) do:[:index | DefaultTypes at:index put:#digit. - DefaultActions at:index put:block ]. - block := [:s :char | s nextIdentifier]. ($a asciiValue) to:($z asciiValue) do:[:index | DefaultTypes at:index put:#letter. - DefaultActions at:index put:block ]. ($A asciiValue) to:($Z asciiValue) do:[:index | DefaultTypes at:index put:#letter. - DefaultActions at:index put:block ]. + DefaultActions at:#letter put:[:s :char | s nextIdentifier]. + DefaultActions at:#digit put:[:s :char | s nextNumber]. + " TokenizedStream initialize " @@ -201,6 +259,16 @@ "Created: 1.2.1996 / 17:42:00 / cg" ! +allowFloatNumbers:aBoolean + "if false, floating numbers are not read; a period is returned as + a separate token. If true (the default), floating point numbers are allowed." + + allowFloatNumbers := aBoolean + + "Modified: 1.2.1996 / 18:14:27 / cg" + "Created: 1.2.1996 / 18:27:41 / cg" +! + beginCommentCharacter:aCharacter beginCommentCharacter := aCharacter @@ -261,10 +329,11 @@ eolCharacter := Character cr. eolIsSignificant := false. - actions := DefaultActions. - types := DefaultTypes. + actions := DefaultActions shallowCopy. + types := DefaultTypes shallowCopy. + allowFloatNumbers := true. - "Modified: 1.2.1996 / 17:36:56 / cg" + "Modified: 1.2.1996 / 18:42:54 / cg" ! ! !TokenizedStream methodsFor:'private'! @@ -290,6 +359,64 @@ "Created: 1.2.1996 / 17:21:47 / cg" ! +nextCNumber + |nextChar value s| + + tokenRadix := 10. + source peek == $0 ifTrue:[ + source next. + source peek == $x ifTrue:[ + source next. + tokenRadix := 16. + ] ifFalse:[ + tokenRadix := 8 + ] + ]. + + value := Integer readFrom:source radix:tokenRadix. + nextChar := source peek. + + (allowFloatNumbers and:[tokenRadix == 10]) ifTrue:[ + (nextChar == $.) ifTrue:[ + nextChar := source nextPeek. + (nextChar notNil and:[nextChar isDigitRadix:tokenRadix]) ifTrue:[ + value := value asFloat + (self nextMantissa:tokenRadix). + nextChar := source peek + ] ifFalse:[ + nextChar == (Character cr) ifTrue:[ + tokenLineNr := tokenLineNr + 1. + ]. + peekChar := $. + ] + ]. + ((nextChar == $e) or:[nextChar == $E]) ifTrue:[ + nextChar := source nextPeek. + (nextChar notNil and:[(nextChar isDigitRadix:tokenRadix) or:['+-' includes:nextChar]]) ifTrue:[ + s := 1. + (nextChar == $+) ifTrue:[ + nextChar := source nextPeek + ] ifFalse:[ + (nextChar == $-) ifTrue:[ + nextChar := source nextPeek. + s := s negated + ] + ]. + value := value asFloat + * (10.0 raisedToInteger:((Integer readFrom:source radix:tokenRadix) * s)) + ] + ]. + ]. + tokenValue := value. + (value isMemberOf:Float) ifTrue:[ + tokenType := #Float + ] ifFalse:[ + tokenType := #Integer + ]. + ^ tokenType + + "Created: 1.2.1996 / 18:26:27 / cg" +! + nextIdentifier |nextChar string oldString index "{ Class: SmallInteger }" @@ -339,6 +466,130 @@ "Modified: 1.2.1996 / 16:37:28 / cg" ! +nextMantissa:radix + |nextChar value factor| + + value := 0. + factor := 1.0 / radix. + nextChar := source peek. + [(nextChar notNil and:[nextChar isDigitRadix:radix])] whileTrue:[ + value := value + (nextChar digitValue * factor). + factor := factor / radix. + nextChar := source nextPeek + ]. + ^ value + + "Created: 1.2.1996 / 18:31:38 / cg" +! + +nextNumber + |nextChar value s| + + tokenRadix := 10. + value := Integer readFrom:source radix:tokenRadix. + nextChar := source peek. + allowFloatNumbers ifTrue:[ + (nextChar == $.) ifTrue:[ + nextChar := source nextPeek. + (nextChar notNil and:[nextChar isDigitRadix:tokenRadix]) ifTrue:[ + value := value asFloat + (self nextMantissa:tokenRadix). + nextChar := source peek + ] ifFalse:[ + nextChar == (Character cr) ifTrue:[ + tokenLineNr := tokenLineNr + 1. + ]. + peekChar := $. + ] + ]. + ((nextChar == $e) or:[nextChar == $E]) ifTrue:[ + nextChar := source nextPeek. + (nextChar notNil and:[(nextChar isDigitRadix:tokenRadix) or:['+-' includes:nextChar]]) ifTrue:[ + s := 1. + (nextChar == $+) ifTrue:[ + nextChar := source nextPeek + ] ifFalse:[ + (nextChar == $-) ifTrue:[ + nextChar := source nextPeek. + s := s negated + ] + ]. + value := value asFloat + * (10.0 raisedToInteger:((Integer readFrom:source radix:tokenRadix) * s)) + ] + ]. + ]. + tokenValue := value. + (value isMemberOf:Float) ifTrue:[ + tokenType := #Float + ] ifFalse:[ + tokenType := #Integer + ]. + ^ tokenType + + "Modified: 1.2.1996 / 18:24:07 / cg" + "Created: 1.2.1996 / 18:31:03 / cg" +! + +nextSmalltalkNumber + |nextChar value s| + + tokenRadix := 10. + value := Integer readFrom:source radix:tokenRadix. + nextChar := source peek. + (nextChar == $r) ifTrue:[ + tokenRadix := value. + source next. + s := 1. + source peek == $- ifTrue:[ + source next. + s := -1 + ]. + value := Integer readFrom:source radix:tokenRadix. + value := value * s. + nextChar := source peek + ]. + allowFloatNumbers ifTrue:[ + (nextChar == $.) ifTrue:[ + nextChar := source nextPeek. + (nextChar notNil and:[nextChar isDigitRadix:tokenRadix]) ifTrue:[ + value := value asFloat + (self nextMantissa:tokenRadix). + nextChar := source peek + ] ifFalse:[ + nextChar == (Character cr) ifTrue:[ + tokenLineNr := tokenLineNr + 1. + ]. + peekChar := $. + ] + ]. + ((nextChar == $e) or:[nextChar == $E]) ifTrue:[ + nextChar := source nextPeek. + (nextChar notNil and:[(nextChar isDigitRadix:tokenRadix) or:['+-' includes:nextChar]]) ifTrue:[ + s := 1. + (nextChar == $+) ifTrue:[ + nextChar := source nextPeek + ] ifFalse:[ + (nextChar == $-) ifTrue:[ + nextChar := source nextPeek. + s := s negated + ] + ]. + value := value asFloat + * (10.0 raisedToInteger:((Integer readFrom:source radix:tokenRadix) * s)) + ] + ]. + ]. + tokenValue := value. + (value isMemberOf:Float) ifTrue:[ + tokenType := #Float + ] ifFalse:[ + tokenType := #Integer + ]. + ^ tokenType + + "Created: 1.2.1996 / 18:19:05 / cg" + "Modified: 1.2.1996 / 18:24:07 / cg" +! + nextString:separator |nextChar string pos index "{ Class: SmallInteger }" @@ -391,6 +642,8 @@ |skipping actionBlock| + tokenValue := tokenName := nil. + peekChar notNil ifTrue:[ hereChar := peekChar. peekChar := peekChar2. @@ -447,25 +700,24 @@ ]. tokenPosition := source position. + types notNil ifTrue:[ + tokenType := types at:(hereChar asciiValue). + ]. + actions notNil ifTrue:[ - actionBlock := actions at:(hereChar asciiValue). + actionBlock := actions at:tokenType ifAbsent:nil. actionBlock notNil ifTrue:[ ^ actionBlock value:self value:hereChar ] ]. - types notNil ifTrue:[ - source next. - tokenType := types at:(hereChar asciiValue). - tokenType notNil ifTrue:[ - ^ tokenType - ] + source next. + tokenType isNil ifTrue:[ + tokenType := #Error. ]. + ^ tokenType - tokenType := #Error. - ^ #Error - - "Modified: 1.2.1996 / 17:39:20 / cg" + "Modified: 1.2.1996 / 18:40:40 / cg" ! skipComment @@ -523,6 +775,6 @@ !TokenizedStream class methodsFor:'documentation'! version - ^ '$Header: /cvs/stx/stx/libbasic2/TokenizedStream.st,v 1.2 1996-02-01 17:14:57 cg Exp $' + ^ '$Header: /cvs/stx/stx/libbasic2/TokenizedStream.st,v 1.3 1996-02-01 17:44:48 cg Exp $' ! ! TokenizedStream initialize!