diff -r 46dd1237b20a -r 3b81c9e53352 compiler/PPCTokenizingVisitor.st --- a/compiler/PPCTokenizingVisitor.st Fri Jul 31 14:07:31 2015 +0100 +++ b/compiler/PPCTokenizingVisitor.st Mon Aug 17 12:56:02 2015 +0100 @@ -14,41 +14,44 @@ afterAccept: node retval: parserNode self isRoot ifTrue: [ - | tokenizerNode whitespaceNode | + | tokensNode whitespaceNode | self change. - tokens addLast: self eofToken. - tokens do: [ :token | token unmarkForInline ]. +" tokens addLast: self eofToken." + tokens do: [ :token | token unmarkForInline ]. + whitespaceNode := tokens detect: [ :e | e isTrimmingTokenNode ] ifNone: [ nil ]. + whitespaceNode notNil ifTrue: [ + whitespaceNode := whitespaceNode whitespace copy + unmarkForInline; + name: 'consumeWhitespace'; + yourself. + "whitespaceNode := PPCTokenWhitespaceNode new + child: whitespaceNode; + yourself" + ] ifFalse: [ + whitespaceNode := PPCNilNode new + name: 'consumeWhitespace'; + yourself + ]. - whitespaceNode := tokens detect: [ :e | e isTrimmingTokenNode ] ifNone:[nil]. - whitespaceNode notNil ifTrue:[ - whitespaceNode := whitespaceNode whitespace copy - unmarkForInline; - name: 'consumeWhitespace'; - yourself - ] ifFalse:[ - whitespaceNode := (PPCNilNode new) - name: 'consumeWhitespace'; - yourself - ]. - tokenizerNode := PPCTokenChoiceNode new + tokensNode := PPCListNode new children: tokens asArray; name: 'nextToken'; yourself. - + ^ PPCTokenizingParserNode new parser: parserNode; - tokenizer: tokenizerNode; whitespace: whitespaceNode; - name: #'mainParser'; - yourself - ]. + tokens: tokensNode; + name: #mainParser; + yourself ]. ^ parserNode - + "Modified: / 12-05-2015 / 01:37:57 / Jan Vrany " ! eofToken | ws | + self error: 'deprecated?'. ws := PPCStarNode new child: (PPCMessagePredicateNode new message: #isSeparator; @@ -59,6 +62,7 @@ child: PPCEndOfFileNode new; whitespace: ws; tokenClass: PPToken; + name: 'eof' yourself. ! ! @@ -87,6 +91,7 @@ visitActionNode: node (node hasProperty: #trimmingToken) ifTrue: [ + self halt: 'can this happen?'. self change. self addToken: node. @@ -98,10 +103,21 @@ ^ super visitActionNode: node ! +visitTokenConsumeNode: node + " + Seems, it might happen, that if I create the consume node, + I will ge to it later. This would create a token consume node for the + child, thus having tokenConsumNode with tokenConsumNode as a child... + " + ^ node +! + visitTokenNode: node self change. self addToken: node. + self assert: node acceptsEpsilon not description: 'Sorry, but the epsilon tokens are not allowed'. + ^ PPCTokenConsumeNode new child: node; yourself. @@ -113,6 +129,7 @@ ^ PPCTokenConsumeNode new child: node; + name: node name; yourself. ! !