hg/stx-libcomp: comparison Scanner.st

equal deleted inserted replaced

-:84a1ddf215a5
+:f8dd8ba75205
 tokenName tokenLineNr
 thisChar peekChar
 requestor exitBlock
 errorFlag
 ignoreErrors ignoreWarnings
-saveComments currentComments'
+saveComments currentComments
+warnSTXSpecialComment
+outStream outCol'
 classVariableNames:'typeArray ActionArray'
 poolDictionaries:''
 category:'System-Compiler'
 !
 Scanner comment:'
 COPYRIGHT (c) 1989 by Claus Gittinger
 All Rights Reserved
-Scanner reads from a stream and returns individual smalltalk tokens
-$Header: /cvs/stx/stx/libcomp/Scanner.st,v 1.7 1994-03-30 10:07:33 claus Exp $
 '!
+!Scanner class methodsFor:'documentation'!
+copyright
+"
+COPYRIGHT (c) 1989 by Claus Gittinger
+All Rights Reserved
+This software is furnished under a license and may be used
+only in accordance with the terms of that license and with the
+inclusion of the above copyright notice.   This software may not
+be provided or otherwise made available to, or used by, any
+other person.  No title to or ownership of the software is
+hereby transferred.
+"
+!
+version
+"
+$Header: /cvs/stx/stx/libcomp/Scanner.st,v 1.8 1994-06-02 20:26:16 claus Exp $
+"
+!
+documentation
+"
+Scanner reads from a stream and returns individual smalltalk tokens
+"
+! !
 !Scanner class methodsFor:'instance creation'!
 for:aStream
 "return a new scanner reading from aStream"
 source := aStream.
 currentComments := nil.
 saveComments := false.
 ignoreErrors := false.
 ignoreWarnings := false.
+warnSTXSpecialComment := true.
 ActionArray isNil ifTrue:[
 ActionArray := Array new:256.
 TypeArray := Array new:256.
 tokenLineNr := 1.
 currentComments := nil.
 saveComments := false.
 ignoreErrors := false.
 ignoreWarnings := false.
+warnSTXSpecialComment := true
 !
 notifying:anObject
 "set the requestor to be notified"
 one token too many"
 (tokenType == #EOF) ifFalse:[
 source position:tokenPosition
 ]
+!
+beginComment
+^ self
+!
+endComment:comment
+saveComments ifTrue:[
+currentComments isNil ifTrue:[
+currentComments := OrderedCollection with:comment
+] ifFalse:[
+currentComments add:comment
+]
+].
 ! !
 !Scanner methodsFor:'error handling'!
 showErrorMessage:aMessage position:pos
 skipComment
 |comment startPos|
 comment := ''.
+self beginComment.
+outStream notNil ifTrue:[
+outStream nextPut:Character doubleQuote.
+outCol := outCol + 1
+].
 startPos := source position.
 source next.
 thisChar := source peek.
 "special ST/X addition:
 thisChar == $/ ifTrue:[
 [thisChar notNil and:[thisChar ~~ Character cr]] whileTrue:[
 saveComments ifTrue:[
 comment := comment copyWith:thisChar
 ].
+outStream notNil ifTrue:[
+outStream nextPut:thisChar.
+outCol := outCol + 1
+].
 thisChar := source nextPeek.
 ].
 tokenLineNr := tokenLineNr + 1.
-self warning:'end-of-line comments are a nonstandard feature of ST/X'
+warnSTXSpecialComment ifTrue:[
-position:startPos
+self warning:'end-of-line comments are a nonstandard feature of ST/X'
-to:(source position)
+position:startPos to:(source position).
+"
+only warn once
+"
+warnSTXSpecialComment := false
+].
+outStream notNil ifTrue:[
+outStream cr.
+outCol := 1
+].
 ] ifFalse:[
 [thisChar notNil and:[thisChar ~~ (Character doubleQuote)]] whileTrue:[
 thisChar == (Character cr) ifTrue:[
 tokenLineNr := tokenLineNr + 1.
 ].
 saveComments ifTrue:[
 comment := comment copyWith:thisChar
 ].
+outStream notNil ifTrue:[
+outStream nextPut:thisChar.
+outCol := outCol + 1
+].
 thisChar := source nextPeek
 ].
 thisChar isNil ifTrue:[
 self warning:'unclosed comment' position:startPos to:(source position)
-].
+] ifFalse:[
+outStream notNil ifTrue:[
+outStream nextPut:(Character doubleQuote).
+outCol := outCol + 1
+].
+]
 ].
 saveComments ifTrue:[
 currentComments isNil ifTrue:[
 currentComments := OrderedCollection with:comment
 ] ifFalse:[
 currentComments add:comment
 ]
 ].
-"skip final dQuote"
+"skip final dQuote or cr"
 source next.
+self endComment:comment.
 !
 nextToken
 "return the next token from the source-stream"
 thisChar := peekChar.
 peekChar := nil
 ] ifFalse:[
 skipping := true.
 [skipping] whileTrue:[
-thisChar := source skipSeparatorsExceptCR.
+outStream notNil ifTrue:[
+[(thisChar := source peek) == Character space] whileTrue:[
+source next.
+outStream space.
+outCol := outCol + 1.
+]
+] ifFalse:[
+thisChar := source skipSeparatorsExceptCR.
+].
 thisChar == (Character cr) ifTrue:[
 tokenLineNr := tokenLineNr + 1.
-source next
+source next.
+outStream notNil ifTrue:[
+outStream cr.
+outCol := 1
+]
 ] ifFalse:[
 thisChar == (Character doubleQuote) ifTrue:[
 "start of a comment"
 self skipComment.
 tokenValue := tokenValue negated.
 ^ tokenType
 ]
 ].
 string := firstChar asString.
-((TypeArray at:(secondChar asciiValue)) == #special) ifTrue:[
+secondChar notNil ifTrue:[
-(secondChar == $-) ifTrue:[
+((TypeArray at:(secondChar asciiValue)) == #special) ifTrue:[
-"special- look if minus belongs to number following"
+(secondChar == $-) ifTrue:[
-p := source position.
+"special- look if minus belongs to number following"
+p := source position.
+source next.
+thirdChar := source peek.
+source position:p.
+thirdChar isDigit ifTrue:[
+tokenName := string.
+tokenType := #BinaryOperator.
+^ tokenType
+]
+].
 source next.
-thirdChar := source peek.
+string := string copyWith:secondChar
-source position:p.
+].
-thirdChar isDigit ifTrue:[
-tokenName := string.
-tokenType := #BinaryOperator.
-^ tokenType
-]
-].
-source next.
-string := string copyWith:secondChar
 ].
 tokenName := string.
 tokenType := #BinaryOperator.
 ^ tokenType
 !
 tokenType := #Keyword.
 ^ self
 ].
 peekChar := $_
 ].
+tokenName := string.
 firstChar := string at:1.
 (firstChar == $s) ifTrue:[
 (string = 'self') ifTrue:[tokenType := #Self. ^self].
 (string = 'super') ifTrue:[tokenType := #Super. ^self]
 ].
 (string = 'thisContext') ifTrue:[tokenType := #ThisContext. ^self]
 ].
 (firstChar == $f) ifTrue:[
 (string = 'false') ifTrue:[tokenType := #False. ^self]
 ].
-tokenName := string.
 tokenType := #Identifier.
 ^ tokenType
 !
 nextPrimitive

changeset 20	f8dd8ba75205
parent 17	f06d70d785dc
child 30	26e2f849916a