Scanner.st
changeset 2983 3d721bb07c48
parent 2961 d18fe23b336a
child 2987 1f2cd9585ea8
equal deleted inserted replaced
2982:e8da03e62efb 2983:3d721bb07c48
  2153 
  2153 
  2154     ^ ch == $"
  2154     ^ ch == $"
  2155 
  2155 
  2156     "Created: / 14.5.1998 / 20:51:42 / cg"
  2156     "Created: / 14.5.1998 / 20:51:42 / cg"
  2157     "Modified: / 14.5.1998 / 20:53:01 / cg"
  2157     "Modified: / 14.5.1998 / 20:53:01 / cg"
       
  2158 !
       
  2159 
       
  2160 isDoIt
       
  2161     ^ false
  2158 !
  2162 !
  2159 
  2163 
  2160 isSpecialOrExtendedSpecialCharacter:ch
  2164 isSpecialOrExtendedSpecialCharacter:ch
  2161     |code charType|
  2165     |code charType|
  2162 
  2166 
  3205     ].
  3209     ].
  3206     ^ self nextToken:$_
  3210     ^ self nextToken:$_
  3207 !
  3211 !
  3208 
  3212 
  3209 skipComment
  3213 skipComment
  3210     "skip over a comment; handles ST/X eol comments."
  3214     "skip over a comment; handles ST/X eol comments (quote-slash)
  3211 
  3215      and multiline-delimiter comment (quote-less-less)."
  3212     |commentStream commentType commentText startPos endPos stillInComment anyNonBlank|
  3216 
       
  3217     |commentStream commentType commentText startPos endPos stillInComment anyNonBlank
       
  3218      delimiter line|
  3213 
  3219 
  3214     anyNonBlank := false.
  3220     anyNonBlank := false.
  3215 
  3221 
  3216     saveComments ifTrue:[
  3222     saveComments ifTrue:[
  3217         commentStream := CharacterWriteStream on:''.
  3223         commentStream := CharacterWriteStream on:''.
  3224     ].
  3230     ].
  3225 
  3231 
  3226     startPos := source position1Based.
  3232     startPos := source position1Based.
  3227     source next.
  3233     source next.
  3228     hereChar := source peekOrNil.
  3234     hereChar := source peekOrNil.
       
  3235     commentType := #regularComment.
  3229 
  3236 
  3230     "
  3237     "
  3231      special ST/X addition:
  3238      special ST/X addition:
  3232      a $/ right after the initial double quote makes it an up-to-end-of-line comment,
  3239      a $/ right after the initial double quote makes it an up-to-end-of-line comment,
  3233      which is very useful to comment out parts of filed-in source code.
  3240      which is very useful to comment out parts of filed-in source code.
  3240 
  3247 
  3241         self skipToEndOfLineRememberingIn:commentStream.
  3248         self skipToEndOfLineRememberingIn:commentStream.
  3242         endPos := source position1Based.
  3249         endPos := source position1Based.
  3243         self markCommentFrom:startPos to:endPos.
  3250         self markCommentFrom:startPos to:endPos.
  3244         commentType := #eolComment.
  3251         commentType := #eolComment.
  3245         lineNr := lineNr + 1.
       
  3246         self warnSTXSpecialCommentAt:startPos to:endPos-1.
  3252         self warnSTXSpecialCommentAt:startPos to:endPos-1.
  3247         outStream notNil ifTrue:[
  3253         outStream notNil ifTrue:[
  3248             outStream cr.
  3254             outStream cr.
  3249             outCol := 1
  3255             outCol := 1
  3250         ].
  3256         ].
  3251         "skip cr"
  3257         "skip cr"
  3252         source next.
  3258         source next.
  3253     ] ifFalse:[
  3259     ] ifFalse:[
  3254         commentType := #regularComment.
  3260         (hereChar == $< and:[parserFlags allowSTXDelimiterComments]) ifTrue:[
  3255 
  3261             hereChar := source nextPeek.
  3256         hereChar == ${ ifTrue:[
  3262             (hereChar == $<) ifTrue:[
  3257             "
  3263                 "everything up to EOL is the delimiter; comment extends to everything up
  3258              special ST/X addition:
  3264                  to a line consisting of the delimiter alone"
  3259              a ${ right after the initial double quote starts a directive
  3265                 commentStream notNil ifTrue:[
  3260             "
  3266                     commentStream nextPutAll:'>>'
  3261             self parseDirective
       
  3262         ].
       
  3263 
       
  3264         stillInComment := true.
       
  3265         [stillInComment] whileTrue:[
       
  3266             stillInComment := false.
       
  3267 
       
  3268             [hereChar notNil and:[hereChar ~~ (Character doubleQuote)]] whileTrue:[
       
  3269                 hereChar isSeparator ifFalse:[
       
  3270                     anyNonBlank := true.
       
  3271                 ].
  3267                 ].
  3272                 hereChar == (Character cr) ifTrue:[
  3268                 hereChar := source nextPeek.
  3273                     lineNr := lineNr + 1.
  3269                 delimiter := String streamContents:[:s | self skipToEndOfLineRememberingIn:s].
       
  3270                 delimiter withoutSeparators isEmpty ifTrue:[
       
  3271                     self parseError:'invalid delimiter in comment'
       
  3272                 ].
       
  3273                 hereChar == Character cr ifTrue:[
       
  3274                     hereChar := source nextPeek.
  3274                 ].
  3275                 ].
  3275                 commentStream notNil ifTrue:[
  3276                 commentStream notNil ifTrue:[
  3276                     commentStream nextPut:hereChar
  3277                     commentStream nextPutLine:delimiter
  3277                 ].
  3278                 ].
  3278                 outStream notNil ifTrue:[
  3279                 stillInComment := true.
  3279                     outStream nextPut:hereChar.
  3280                 [stillInComment] whileTrue:[
  3280                     outCol := outCol + 1
  3281                     line := String streamContents:[:s | self skipToEndOfLineRememberingIn:s].
       
  3282                     commentStream notNil ifTrue:[
       
  3283                         commentStream nextPutLine:line
       
  3284                     ].
       
  3285                     hereChar == Character cr ifTrue:[
       
  3286                         hereChar := source nextPeek.
       
  3287                     ].
       
  3288                     stillInComment := (line ~= delimiter).    
  3281                 ].
  3289                 ].
  3282                 hereChar := source nextPeek
  3290                 commentType := #delimiterComment.
  3283             ].
       
  3284 
       
  3285             hereChar isNil ifTrue:[
       
  3286                 self markCommentFrom:startPos to:(source size).
       
  3287                 self warning:'unclosed comment' position:startPos to:(source position1Based)
       
  3288             ] ifFalse:[
  3291             ] ifFalse:[
  3289                 self markCommentFrom:startPos to:(source position1Based).
  3292                 commentStream notNil ifTrue:[
  3290                 outStream notNil ifTrue:[
  3293                     commentStream nextPut:$>
  3291                     outStream nextPut:(Character doubleQuote).
       
  3292                     outCol := outCol + 1
       
  3293                 ].
  3294                 ].
  3294             ].
  3295             ].
  3295             endPos := source position1Based.
  3296         ].
  3296             "skip final dQuote"
  3297 
  3297             source next.
  3298         (commentType == #regularComment) ifTrue:[
  3298 
  3299             hereChar == ${ ifTrue:[
  3299             (source peek == Character doubleQuote) ifTrue:[
  3300                 "
  3300                 stillInComment := true.
  3301                  special ST/X addition:
  3301                 hereChar := source nextPeek.
  3302                  a ${ right after the initial double quote starts a directive
  3302             ].
  3303                 "
  3303         ].
  3304                 self parseDirective
       
  3305             ].
       
  3306 
       
  3307             stillInComment := true.
       
  3308             [stillInComment] whileTrue:[
       
  3309                 stillInComment := false.
       
  3310 
       
  3311                 [hereChar notNil and:[hereChar ~~ (Character doubleQuote)]] whileTrue:[
       
  3312                     hereChar isSeparator ifFalse:[
       
  3313                         anyNonBlank := true.
       
  3314                     ].
       
  3315                     hereChar == (Character cr) ifTrue:[
       
  3316                         lineNr := lineNr + 1.
       
  3317                     ].
       
  3318                     commentStream notNil ifTrue:[
       
  3319                         commentStream nextPut:hereChar
       
  3320                     ].
       
  3321                     outStream notNil ifTrue:[
       
  3322                         outStream nextPut:hereChar.
       
  3323                         outCol := outCol + 1
       
  3324                     ].
       
  3325                     hereChar := source nextPeek
       
  3326                 ].
       
  3327 
       
  3328                 hereChar isNil ifTrue:[
       
  3329                     self markCommentFrom:startPos to:(source size).
       
  3330                     self warning:'unclosed comment' position:startPos to:(source position1Based)
       
  3331                 ] ifFalse:[
       
  3332                     self markCommentFrom:startPos to:(source position1Based).
       
  3333                     outStream notNil ifTrue:[
       
  3334                         outStream nextPut:(Character doubleQuote).
       
  3335                         outCol := outCol + 1
       
  3336                     ].
       
  3337                 ].
       
  3338                 endPos := source position1Based.
       
  3339                 "skip final dQuote"
       
  3340                 source next.
       
  3341 
       
  3342                 (source peek == Character doubleQuote) ifTrue:[
       
  3343                     stillInComment := true.
       
  3344                     hereChar := source nextPeek.
       
  3345                 ].
       
  3346             ].
       
  3347         ]
  3304     ].
  3348     ].
  3305 
  3349 
  3306     saveComments ifTrue:[
  3350     saveComments ifTrue:[
  3307         commentText := commentStream contents.
  3351         commentText := commentStream contents.
  3308         self endComment:commentText type:commentType.
  3352         self endComment:commentText type:commentType.
  3330             outStream nextPut:hereChar.
  3374             outStream nextPut:hereChar.
  3331             outCol := outCol + 1
  3375             outCol := outCol + 1
  3332         ].
  3376         ].
  3333         hereChar := source nextPeek.
  3377         hereChar := source nextPeek.
  3334     ].
  3378     ].
       
  3379     lineNr := lineNr + 1.
  3335 ! !
  3380 ! !
  3336 
  3381 
  3337 !Scanner::Comment methodsFor:'accessing'!
  3382 !Scanner::Comment methodsFor:'accessing'!
  3338 
  3383 
  3339 commentString
  3384 commentString
  3426 ! !
  3471 ! !
  3427 
  3472 
  3428 !Scanner class methodsFor:'documentation'!
  3473 !Scanner class methodsFor:'documentation'!
  3429 
  3474 
  3430 version
  3475 version
  3431     ^ '$Header: /cvs/stx/stx/libcomp/Scanner.st,v 1.285 2012-10-31 14:12:26 cg Exp $'
  3476     ^ '$Header: /cvs/stx/stx/libcomp/Scanner.st,v 1.286 2012-11-13 13:03:03 cg Exp $'
  3432 !
  3477 !
  3433 
  3478 
  3434 version_CVS
  3479 version_CVS
  3435     ^ '$Header: /cvs/stx/stx/libcomp/Scanner.st,v 1.285 2012-10-31 14:12:26 cg Exp $'
  3480     ^ '$Header: /cvs/stx/stx/libcomp/Scanner.st,v 1.286 2012-11-13 13:03:03 cg Exp $'
  3436 ! !
  3481 ! !
  3437 
  3482 
  3438 Scanner initialize!
  3483 Scanner initialize!