Mercurial Mercurial > hg > stx-libbasic2 / diff
summary | shortlog | changelog | graph | tags | bookmarks | branches | files | changeset | file | latest | revisions | annotate | diff | comparison | raw | help
Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
TokenizedStream.st
changeset 173 f8c6732b927c
parent 172 99b850002359
child 174 64e81af0a50b 
--- a/TokenizedStream.st	Thu Feb 01 18:14:57 1996 +0100
+++ b/TokenizedStream.st	Thu Feb 01 18:44:48 1996 +0100
@@ -2,7 +2,8 @@
 	instanceVariableNames:'source token tokenType tokenPosition tokenName tokenLineNr
 		tokenValue tokenRadix hereChar peekChar peekChar2
 		beginCommentCharacter endCommentCharacter eolCommentCharacter
-		eolCharacter outStream outCol actions types eolIsSignificant'
+		eolCharacter outStream outCol actions types eolIsSignificant
+		allowFloatNumbers'
 	classVariableNames:'DefaultActions DefaultTypes'
 	poolDictionaries:''
 	category:'Streams'
@@ -123,8 +124,69 @@
             ]
         ].
 
+    allowing float numbers (the default):
 
-    scan /etc/services file:
+        |s|
+
+        s := TokenizedStream on:'1.23 4.56 7 8 9'.
+        [s atEnd] whileFalse:[
+            s next.
+            Transcript showCr:(s tokenType displayString, ' value=' , s tokenValue printString).
+        ].
+
+
+    not allowing float numbers :
+
+        |s|
+
+        s := TokenizedStream on:'1.23 4.56 7 8 9'.
+        s allowFloatNumbers:false.
+
+        [s atEnd] whileFalse:[
+            s next.
+            Transcript showCr:(s tokenType displayString , ' value= ' , s tokenValue printString).
+        ].
+
+
+    no radix numbers (the default):
+
+        |s|
+
+        s := TokenizedStream on:'0x1234 16r1234'.
+
+        [s atEnd] whileFalse:[
+            s next.
+            Transcript showCr:(s tokenType displayString , ' value= ' , s tokenValue printString , ' name=' , s tokenName displayString).
+        ].
+
+
+    C-style radix numbers:
+
+        |s|
+
+        s := TokenizedStream on:'0x1234 16r1234'.
+        s actionTable at:#digit put:[:s :char | s nextCNumber].
+
+        [s atEnd] whileFalse:[
+            s next.
+            Transcript showCr:(s tokenType displayString , ' value= ' , s tokenValue printString , ' name=' , s tokenName displayString).
+        ].
+
+
+    smalltalk-style radix numbers:
+
+        |s|
+
+        s := TokenizedStream on:'0x1234 16r1234'.
+        s actionTable at:#digit put:[:s :char | s nextSmalltalkNumber].
+
+        [s atEnd] whileFalse:[
+            s next.
+            Transcript showCr:(s tokenType displayString , ' value= ' , s tokenValue printString , ' name=' , s tokenName displayString).
+        ].
+
+
+    scan the '/etc/services' file:
 
         |s t service port protocol|
 
@@ -156,9 +218,7 @@
 !TokenizedStream class methodsFor:'initialization'!
 
 initialize
-    |block|
-
-    DefaultActions := Array new:256.
+    DefaultActions := IdentityDictionary new.
     DefaultTypes := Array new:256.
 
     "kludge: action is nextColonOrAssign, but type is special"
@@ -166,22 +226,20 @@
         DefaultTypes at:code put:(Character value:code).
     ].
 
-    block := [:s :char | s nextInteger].
     ($0 asciiValue) to:($9 asciiValue) do:[:index |
         DefaultTypes at:index put:#digit.
-        DefaultActions at:index put:block
     ].
 
-    block := [:s :char | s nextIdentifier].
     ($a asciiValue) to:($z asciiValue) do:[:index |
         DefaultTypes at:index put:#letter.
-        DefaultActions at:index put:block
     ].
     ($A asciiValue) to:($Z asciiValue) do:[:index |
         DefaultTypes at:index put:#letter.
-        DefaultActions at:index put:block
     ].
 
+    DefaultActions at:#letter put:[:s :char | s nextIdentifier].
+    DefaultActions at:#digit  put:[:s :char | s nextNumber].
+
     "
      TokenizedStream initialize
     "
@@ -201,6 +259,16 @@
     "Created: 1.2.1996 / 17:42:00 / cg"
 !
 
+allowFloatNumbers:aBoolean
+    "if false, floating numbers are not read; a period is returned as
+     a separate token. If true (the default), floating point numbers are allowed."
+
+    allowFloatNumbers := aBoolean
+
+    "Modified: 1.2.1996 / 18:14:27 / cg"
+    "Created: 1.2.1996 / 18:27:41 / cg"
+!
+
 beginCommentCharacter:aCharacter
     beginCommentCharacter := aCharacter
 
@@ -261,10 +329,11 @@
     eolCharacter := Character cr.
     eolIsSignificant := false.
 
-    actions := DefaultActions.
-    types := DefaultTypes.
+    actions := DefaultActions shallowCopy.
+    types := DefaultTypes shallowCopy.
+    allowFloatNumbers := true.
 
-    "Modified: 1.2.1996 / 17:36:56 / cg"
+    "Modified: 1.2.1996 / 18:42:54 / cg"
 ! !
 
 !TokenizedStream methodsFor:'private'!
@@ -290,6 +359,64 @@
     "Created: 1.2.1996 / 17:21:47 / cg"
 !
 
+nextCNumber
+    |nextChar value s|
+
+    tokenRadix := 10.
+    source peek == $0 ifTrue:[
+        source next.
+        source peek == $x ifTrue:[
+            source next.
+            tokenRadix := 16.
+        ] ifFalse:[
+            tokenRadix := 8
+        ]
+    ].
+
+    value := Integer readFrom:source radix:tokenRadix.
+    nextChar := source peek.
+
+    (allowFloatNumbers and:[tokenRadix == 10]) ifTrue:[
+        (nextChar == $.) ifTrue:[
+            nextChar := source nextPeek.
+            (nextChar notNil and:[nextChar isDigitRadix:tokenRadix]) ifTrue:[
+                value := value asFloat + (self nextMantissa:tokenRadix).
+                nextChar := source peek
+            ] ifFalse:[
+                nextChar == (Character cr) ifTrue:[
+                    tokenLineNr := tokenLineNr + 1.
+                ].
+                peekChar := $.
+            ]
+        ].
+        ((nextChar == $e) or:[nextChar == $E]) ifTrue:[
+            nextChar := source nextPeek.
+            (nextChar notNil and:[(nextChar isDigitRadix:tokenRadix) or:['+-' includes:nextChar]]) ifTrue:[
+                s := 1.
+                (nextChar == $+) ifTrue:[
+                    nextChar := source nextPeek
+                ] ifFalse:[
+                    (nextChar == $-) ifTrue:[
+                        nextChar := source nextPeek.
+                        s := s negated
+                    ]
+                ].
+                value := value asFloat
+                         * (10.0 raisedToInteger:((Integer readFrom:source radix:tokenRadix) * s))
+            ]
+        ].
+    ].
+    tokenValue := value.
+    (value isMemberOf:Float) ifTrue:[
+        tokenType := #Float
+    ] ifFalse:[
+        tokenType := #Integer
+    ].
+    ^ tokenType
+
+    "Created: 1.2.1996 / 18:26:27 / cg"
+!
+
 nextIdentifier
     |nextChar string oldString 
      index "{ Class: SmallInteger }"
@@ -339,6 +466,130 @@
     "Modified: 1.2.1996 / 16:37:28 / cg"
 !
 
+nextMantissa:radix
+    |nextChar value factor|
+
+    value := 0.
+    factor := 1.0 / radix.
+    nextChar := source peek.
+    [(nextChar notNil and:[nextChar isDigitRadix:radix])] whileTrue:[
+        value := value + (nextChar digitValue * factor).
+        factor := factor / radix.
+        nextChar := source nextPeek
+    ].
+    ^ value
+
+    "Created: 1.2.1996 / 18:31:38 / cg"
+!
+
+nextNumber
+    |nextChar value s|
+
+    tokenRadix := 10.
+    value := Integer readFrom:source radix:tokenRadix.
+    nextChar := source peek.
+    allowFloatNumbers ifTrue:[
+        (nextChar == $.) ifTrue:[
+            nextChar := source nextPeek.
+            (nextChar notNil and:[nextChar isDigitRadix:tokenRadix]) ifTrue:[
+                value := value asFloat + (self nextMantissa:tokenRadix).
+                nextChar := source peek
+            ] ifFalse:[
+                nextChar == (Character cr) ifTrue:[
+                    tokenLineNr := tokenLineNr + 1.
+                ].
+                peekChar := $.
+            ]
+        ].
+        ((nextChar == $e) or:[nextChar == $E]) ifTrue:[
+            nextChar := source nextPeek.
+            (nextChar notNil and:[(nextChar isDigitRadix:tokenRadix) or:['+-' includes:nextChar]]) ifTrue:[
+                s := 1.
+                (nextChar == $+) ifTrue:[
+                    nextChar := source nextPeek
+                ] ifFalse:[
+                    (nextChar == $-) ifTrue:[
+                        nextChar := source nextPeek.
+                        s := s negated
+                    ]
+                ].
+                value := value asFloat
+                         * (10.0 raisedToInteger:((Integer readFrom:source radix:tokenRadix) * s))
+            ]
+        ].
+    ].
+    tokenValue := value.
+    (value isMemberOf:Float) ifTrue:[
+        tokenType := #Float
+    ] ifFalse:[
+        tokenType := #Integer
+    ].
+    ^ tokenType
+
+    "Modified: 1.2.1996 / 18:24:07 / cg"
+    "Created: 1.2.1996 / 18:31:03 / cg"
+!
+
+nextSmalltalkNumber
+    |nextChar value s|
+
+    tokenRadix := 10.
+    value := Integer readFrom:source radix:tokenRadix.
+    nextChar := source peek.
+    (nextChar == $r) ifTrue:[
+        tokenRadix := value.
+        source next.
+        s := 1.
+        source peek == $- ifTrue:[
+            source next.
+            s := -1
+        ].
+        value := Integer readFrom:source radix:tokenRadix.
+        value := value * s.
+        nextChar := source peek
+    ].
+    allowFloatNumbers ifTrue:[
+        (nextChar == $.) ifTrue:[
+            nextChar := source nextPeek.
+            (nextChar notNil and:[nextChar isDigitRadix:tokenRadix]) ifTrue:[
+                value := value asFloat + (self nextMantissa:tokenRadix).
+                nextChar := source peek
+            ] ifFalse:[
+                nextChar == (Character cr) ifTrue:[
+                    tokenLineNr := tokenLineNr + 1.
+                ].
+                peekChar := $.
+            ]
+        ].
+        ((nextChar == $e) or:[nextChar == $E]) ifTrue:[
+            nextChar := source nextPeek.
+            (nextChar notNil and:[(nextChar isDigitRadix:tokenRadix) or:['+-' includes:nextChar]]) ifTrue:[
+                s := 1.
+                (nextChar == $+) ifTrue:[
+                    nextChar := source nextPeek
+                ] ifFalse:[
+                    (nextChar == $-) ifTrue:[
+                        nextChar := source nextPeek.
+                        s := s negated
+                    ]
+                ].
+                value := value asFloat
+                         * (10.0 raisedToInteger:((Integer readFrom:source radix:tokenRadix) * s))
+            ]
+        ].
+    ].
+    tokenValue := value.
+    (value isMemberOf:Float) ifTrue:[
+        tokenType := #Float
+    ] ifFalse:[
+        tokenType := #Integer
+    ].
+    ^ tokenType
+
+    "Created: 1.2.1996 / 18:19:05 / cg"
+    "Modified: 1.2.1996 / 18:24:07 / cg"
+!
+
 nextString:separator
     |nextChar string pos
      index "{ Class: SmallInteger }"
@@ -391,6 +642,8 @@
 
     |skipping actionBlock|
 
+    tokenValue := tokenName := nil.
+
     peekChar notNil ifTrue:[
         hereChar := peekChar.
         peekChar := peekChar2.
@@ -447,25 +700,24 @@
     ].
     tokenPosition := source position.
 
+    types notNil ifTrue:[
+        tokenType := types at:(hereChar asciiValue).
+    ].
+
     actions notNil ifTrue:[
-        actionBlock := actions at:(hereChar asciiValue).
+        actionBlock := actions at:tokenType ifAbsent:nil.
         actionBlock notNil ifTrue:[
             ^ actionBlock value:self value:hereChar
         ]
     ].
 
-    types notNil ifTrue:[
-        source next.
-        tokenType := types at:(hereChar asciiValue).
-        tokenType notNil ifTrue:[
-            ^ tokenType
-        ]
+    source next.
+    tokenType isNil ifTrue:[
+        tokenType := #Error.
     ].
+    ^ tokenType
 
-    tokenType := #Error.
-    ^ #Error
-
-    "Modified: 1.2.1996 / 17:39:20 / cg"
+    "Modified: 1.2.1996 / 18:40:40 / cg"
 !
 
 skipComment
@@ -523,6 +775,6 @@
 !TokenizedStream class methodsFor:'documentation'!
 
 version
-    ^ '$Header: /cvs/stx/stx/libbasic2/TokenizedStream.st,v 1.2 1996-02-01 17:14:57 cg Exp $'
+    ^ '$Header: /cvs/stx/stx/libbasic2/TokenizedStream.st,v 1.3 1996-02-01 17:44:48 cg Exp $'
 ! !
 TokenizedStream initialize!
hg/stx-libbasic2