--- a/compiler/Dart__Scanner.st Fri Jan 11 10:12:41 2013 +0000
+++ b/compiler/Dart__Scanner.st Fri Jan 11 13:40:41 2013 +0000
@@ -106,7 +106,7 @@
TypeArray := Array new:256.
block := [:s :char | s nextNumber].
- ($0 asciiValue) to:($9 asciiValue) do:[:index |
+ ($0 codePoint) to:($9 codePoint) do:[:index |
ActionArray at:index put:block
].
@@ -116,34 +116,34 @@
].
block := [:s :char | s nextIdentifier].
- ($a asciiValue) to:($z asciiValue) do:[:index |
+ ($a codePoint) to:($z codePoint) do:[:index |
ActionArray at:index put:block
].
- ($A asciiValue) to:($Z asciiValue) do:[:index |
+ ($A codePoint) to:($Z codePoint) do:[:index |
ActionArray at:index put:block
].
- ActionArray at:$_ asciiValue put:block.
+ ActionArray at:$_ codePoint put:block.
- ActionArray at:$$ asciiValue put:block.
+ ActionArray at:$$ codePoint put:block.
- ActionArray at:($. asciiValue) put:[:s :char | s nextDotOrFloatOrEllipsis].
+ ActionArray at:($. codePoint) put:[:s :char | s nextDotOrFloatOrEllipsis].
- ActionArray at:($' asciiValue) put:[:s :char | s nextString:$' character:true].
- ActionArray at:($" asciiValue) put:[:s :char | s nextString:$" character:false].
- ActionArray at:($!! asciiValue) put:[:s :char | s nextMulti:#(($= #'!!=')) after:char].
- ActionArray at:($= asciiValue) put:[:s :char | s nextMulti:#(($= #'==')) after:char].
- ActionArray at:($< asciiValue) put:[:s :char | s nextMulti:#(($= #'<=') ($< #'<<')) after:char].
- ActionArray at:($> asciiValue) put:[:s :char | s nextMulti:#(($= #'>=') ($> #'>>' $> #'>>>' $= #'>>>=')) after:char].
+ ActionArray at:($' codePoint) put:[:s :char | s nextString:$' character:false].
+ ActionArray at:($" codePoint) put:[:s :char | s nextString:$" character:false].
+ ActionArray at:($!! codePoint) put:[:s :char | s nextMulti:#(($= #'!!=')) after:char].
+ ActionArray at:($= codePoint) put:[:s :char | s nextMulti:#(($= #'==')) after:char].
+ ActionArray at:($< codePoint) put:[:s :char | s nextMulti:#(($= #'<=') ($< #'<<')) after:char].
+ ActionArray at:($> codePoint) put:[:s :char | s nextMulti:#(($= #'>=') ($> #'>>' $> #'>>>' $= #'>>>=')) after:char].
- ActionArray at:($- asciiValue) put:[:s :char | s nextMulti:#(($- #'--') ($= #'-=')) after:char].
- ActionArray at:($+ asciiValue) put:[:s :char | s nextMulti:#(($+ #'++') ($= #'+=')) after:char].
- ActionArray at:($* asciiValue) put:[:s :char | s nextMulti:#(($= #'*=')) after:char].
- ActionArray at:($/ asciiValue) put:[:s :char | s nextMulti:#(($= #'/=') ($/ nil #skipEOLComment) ($* nil #skipComment)) after:char].
- ActionArray at:($% asciiValue) put:[:s :char | s nextMulti:#(($= #'%=')) after:char].
- ActionArray at:($& asciiValue) put:[:s :char | s nextMulti:#(($= #'&=') ($& #'&&')) after:char].
- ActionArray at:($^ asciiValue) put:[:s :char | s nextMulti:#(($= #'^=')) after:char].
- ActionArray at:($~ asciiValue) put:[:s :char | s nextMulti:#(($= #'~=')) after:char].
- ActionArray at:($| asciiValue) put:[:s :char | s nextMulti:#(($= #'|=') ($| #'||')) after:char].
+ ActionArray at:($- codePoint) put:[:s :char | s nextMulti:#(($- #'--') ($= #'-=')) after:char].
+ ActionArray at:($+ codePoint) put:[:s :char | s nextMulti:#(($+ #'++') ($= #'+=')) after:char].
+ ActionArray at:($* codePoint) put:[:s :char | s nextMulti:#(($= #'*=')) after:char].
+ ActionArray at:($/ codePoint) put:[:s :char | s nextMulti:#(($= #'/=') ($/ nil #skipEOLComment) ($* nil #skipComment)) after:char].
+ ActionArray at:($% codePoint) put:[:s :char | s nextMulti:#(($= #'%=')) after:char].
+ ActionArray at:($& codePoint) put:[:s :char | s nextMulti:#(($= #'&=') ($& #'&&')) after:char].
+ ActionArray at:($^ codePoint) put:[:s :char | s nextMulti:#(($= #'^=')) after:char].
+ ActionArray at:($~ codePoint) put:[:s :char | s nextMulti:#(($= #'~=')) after:char].
+ ActionArray at:($| codePoint) put:[:s :char | s nextMulti:#(($= #'|=') ($| #'||')) after:char].
"
self setupActions
@@ -151,7 +151,7 @@
"Created: / 14-05-1998 / 15:48:03 / cg"
"Modified: / 17-05-1998 / 21:03:37 / cg"
- "Modified: / 16-03-2012 / 23:49:40 / Jan Vrany <jan.vrany@fit.cvut.cz>"
+ "Modified: / 11-01-2013 / 13:04:40 / Jan Vrany <jan.vrany@fit.cvut.cz>"
!
setupKeywordTable
@@ -218,6 +218,21 @@
"Modified: / 10-01-2013 / 10:51:58 / Jan Vrany <jan.vrany@fit.cvut.cz>"
! !
+!Scanner class methodsFor:'utility scanning'!
+
+scan: aString
+ | tokens scanner |
+
+ tokens := OrderedCollection new.
+ scanner := (self for: aString).
+ [ scanner atEnd ] whileFalse:[
+ tokens add: scanner nextToken
+ ].
+ ^tokens
+
+ "Created: / 11-01-2013 / 12:57:02 / Jan Vrany <jan.vrany@fit.cvut.cz>"
+! !
+
!Scanner methodsFor:'accessing'!
token
@@ -1059,21 +1074,6 @@
value := something.
! !
-!Scanner::Token methodsFor:'printing & storing'!
-
-printOn:aStream
- "append a printed representation if the receiver to the argument, aStream"
-
- super printOn:aStream.
- aStream nextPutAll:'type: '.
- type printOn:aStream.
- aStream nextPutAll:'value: '.
- value printOn:aStream.
- aStream nextPutAll:'startPosition: '.
- startPosition printOn:aStream.
- aStream nextPutAll:'endPosition: '.
- endPosition printOn:aStream.
-! !
!Scanner class methodsFor:'documentation'!