Updated to PetitCompiler-JanVrany.135, PetitCompiler-Tests-JanKurs.93, PetitCompiler-Extras-Tests-JanVrany.16, PetitCompiler-Benchmarks-JanKurs.12
Name: PetitCompiler-JanVrany.135
Author: JanVrany
Time: 22-07-2015, 06:53:29.127 PM
UUID: 890178b5-275d-46af-a2ad-1738998f07cb
Ancestors: PetitCompiler-JanVrany.134
Name: PetitCompiler-Tests-JanKurs.93
Author: JanKurs
Time: 20-07-2015, 11:30:10.283 PM
UUID: 6473e671-ad70-42ca-b6c3-654b78edc531
Ancestors: PetitCompiler-Tests-JanKurs.92
Name: PetitCompiler-Extras-Tests-JanVrany.16
Author: JanVrany
Time: 22-07-2015, 05:18:22.387 PM
UUID: 8f6f9129-dbba-49b1-9402-038470742f98
Ancestors: PetitCompiler-Extras-Tests-JanKurs.15
Name: PetitCompiler-Benchmarks-JanKurs.12
Author: JanKurs
Time: 06-07-2015, 02:10:06.901 PM
UUID: cb24f1ac-46a4-494d-9780-64576f0f0dba
Ancestors: PetitCompiler-Benchmarks-JanKurs.11, PetitCompiler-Benchmarks-JanVrany.e29bd90f388e.20150619081300
"{ Package: 'stx:goodies/petitparser/compiler' }"
"{ NameSpace: Smalltalk }"
PPCRewritingVisitor subclass:#PPCTokenizingVisitor
instanceVariableNames:'tokens'
classVariableNames:''
poolDictionaries:''
category:'PetitCompiler-Visitors'
!
!PPCTokenizingVisitor methodsFor:'hooks'!
afterAccept: node retval: parserNode
self isRoot ifTrue: [
| tokenizerNode whitespaceNode |
self change.
tokens addLast: self eofToken.
tokens do: [ :token | token unmarkForInline ].
whitespaceNode := tokens detect: [ :e | e isTrimmingTokenNode ] ifNone:[nil].
whitespaceNode notNil ifTrue:[
whitespaceNode := whitespaceNode whitespace copy
unmarkForInline;
name: 'consumeWhitespace';
yourself
] ifFalse:[
whitespaceNode := (PPCNilNode new)
name: 'consumeWhitespace';
yourself
].
tokenizerNode := PPCTokenChoiceNode new
children: tokens asArray;
name: 'nextToken';
yourself.
^ PPCTokenizingParserNode new
parser: parserNode;
tokenizer: tokenizerNode;
whitespace: whitespaceNode;
name: #'mainParser';
yourself
].
^ parserNode
"Modified: / 12-05-2015 / 01:37:57 / Jan Vrany <jan.vrany@fit.cvut.cz>"
!
eofToken
| ws |
ws := PPCStarNode new
child: (PPCMessagePredicateNode new
message: #isSeparator;
yourself);
yourself.
^ PPCTrimmingTokenNode new
child: PPCEndOfFileNode new;
whitespace: ws;
tokenClass: PPToken;
yourself.
! !
!PPCTokenizingVisitor methodsFor:'initialization'!
initialize
super initialize.
tokens := OrderedCollection new.
! !
!PPCTokenizingVisitor methodsFor:'testing'!
isRoot
^ openSet size = 1
! !
!PPCTokenizingVisitor methodsFor:'tokens'!
addToken: token
(tokens contains: [:e | e == token] ) ifFalse: [
tokens addLast: token
]
! !
!PPCTokenizingVisitor methodsFor:'visiting'!
visitActionNode: node
(node hasProperty: #trimmingToken) ifTrue: [
self change.
self addToken: node.
^ PPCTokenConsumeNode new
child: node;
yourself
].
^ super visitActionNode: node
!
visitTokenNode: node
self change.
self addToken: node.
^ PPCTokenConsumeNode new
child: node;
yourself.
!
visitTrimmingTokenNode: node
self change.
self addToken: node.
^ PPCTokenConsumeNode new
child: node;
yourself.
! !