--- a/compiler/PPCTokenizingVisitor.st Fri Jul 24 15:06:54 2015 +0100
+++ b/compiler/PPCTokenizingVisitor.st Mon Aug 17 12:13:16 2015 +0100
@@ -13,41 +13,44 @@
afterAccept: node retval: parserNode
self isRoot ifTrue: [
- | tokenizerNode whitespaceNode |
+ | tokensNode whitespaceNode |
self change.
- tokens addLast: self eofToken.
- tokens do: [ :token | token unmarkForInline ].
+" tokens addLast: self eofToken."
+ tokens do: [ :token | token unmarkForInline ].
+ whitespaceNode := tokens detect: [ :e | e isTrimmingTokenNode ] ifNone: [ nil ].
+ whitespaceNode notNil ifTrue: [
+ whitespaceNode := whitespaceNode whitespace copy
+ unmarkForInline;
+ name: 'consumeWhitespace';
+ yourself.
+ "whitespaceNode := PPCTokenWhitespaceNode new
+ child: whitespaceNode;
+ yourself"
+ ] ifFalse: [
+ whitespaceNode := PPCNilNode new
+ name: 'consumeWhitespace';
+ yourself
+ ].
- whitespaceNode := tokens detect: [ :e | e isTrimmingTokenNode ] ifNone:[nil].
- whitespaceNode notNil ifTrue:[
- whitespaceNode := whitespaceNode whitespace copy
- unmarkForInline;
- name: 'consumeWhitespace';
- yourself
- ] ifFalse:[
- whitespaceNode := (PPCNilNode new)
- name: 'consumeWhitespace';
- yourself
- ].
- tokenizerNode := PPCTokenChoiceNode new
+ tokensNode := PPCListNode new
children: tokens asArray;
name: 'nextToken';
yourself.
-
+
^ PPCTokenizingParserNode new
parser: parserNode;
- tokenizer: tokenizerNode;
whitespace: whitespaceNode;
- name: #'mainParser';
- yourself
- ].
+ tokens: tokensNode;
+ name: #mainParser;
+ yourself ].
^ parserNode
-
+
"Modified: / 12-05-2015 / 01:37:57 / Jan Vrany <jan.vrany@fit.cvut.cz>"
!
eofToken
| ws |
+ self error: 'deprecated?'.
ws := PPCStarNode new
child: (PPCMessagePredicateNode new
message: #isSeparator;
@@ -58,6 +61,7 @@
child: PPCEndOfFileNode new;
whitespace: ws;
tokenClass: PPToken;
+ name: 'eof'
yourself.
! !
@@ -86,6 +90,7 @@
visitActionNode: node
(node hasProperty: #trimmingToken) ifTrue: [
+ self halt: 'can this happen?'.
self change.
self addToken: node.
@@ -97,10 +102,21 @@
^ super visitActionNode: node
!
+visitTokenConsumeNode: node
+ "
+ Seems, it might happen, that if I create the consume node,
+ I will ge to it later. This would create a token consume node for the
+ child, thus having tokenConsumNode with tokenConsumNode as a child...
+ "
+ ^ node
+!
+
visitTokenNode: node
self change.
self addToken: node.
+ self assert: node acceptsEpsilon not description: 'Sorry, but the epsilon tokens are not allowed'.
+
^ PPCTokenConsumeNode new
child: node;
yourself.
@@ -112,6 +128,7 @@
^ PPCTokenConsumeNode new
child: node;
+ name: node name;
yourself.
! !