11 |
11 |
12 !PPCTokenizingVisitor methodsFor:'hooks'! |
12 !PPCTokenizingVisitor methodsFor:'hooks'! |
13 |
13 |
14 afterAccept: node retval: parserNode |
14 afterAccept: node retval: parserNode |
15 self isRoot ifTrue: [ |
15 self isRoot ifTrue: [ |
16 | tokenizerNode whitespaceNode | |
16 | tokensNode whitespaceNode | |
17 self change. |
17 self change. |
18 tokens addLast: self eofToken. |
18 " tokens addLast: self eofToken." |
19 tokens do: [ :token | token unmarkForInline ]. |
19 tokens do: [ :token | token unmarkForInline ]. |
|
20 whitespaceNode := tokens detect: [ :e | e isTrimmingTokenNode ] ifNone: [ nil ]. |
|
21 whitespaceNode notNil ifTrue: [ |
|
22 whitespaceNode := whitespaceNode whitespace copy |
|
23 unmarkForInline; |
|
24 name: 'consumeWhitespace'; |
|
25 yourself. |
|
26 "whitespaceNode := PPCTokenWhitespaceNode new |
|
27 child: whitespaceNode; |
|
28 yourself" |
|
29 ] ifFalse: [ |
|
30 whitespaceNode := PPCNilNode new |
|
31 name: 'consumeWhitespace'; |
|
32 yourself |
|
33 ]. |
20 |
34 |
21 whitespaceNode := tokens detect: [ :e | e isTrimmingTokenNode ] ifNone:[nil]. |
35 tokensNode := PPCListNode new |
22 whitespaceNode notNil ifTrue:[ |
|
23 whitespaceNode := whitespaceNode whitespace copy |
|
24 unmarkForInline; |
|
25 name: 'consumeWhitespace'; |
|
26 yourself |
|
27 ] ifFalse:[ |
|
28 whitespaceNode := (PPCNilNode new) |
|
29 name: 'consumeWhitespace'; |
|
30 yourself |
|
31 ]. |
|
32 tokenizerNode := PPCTokenChoiceNode new |
|
33 children: tokens asArray; |
36 children: tokens asArray; |
34 name: 'nextToken'; |
37 name: 'nextToken'; |
35 yourself. |
38 yourself. |
36 |
39 |
37 ^ PPCTokenizingParserNode new |
40 ^ PPCTokenizingParserNode new |
38 parser: parserNode; |
41 parser: parserNode; |
39 tokenizer: tokenizerNode; |
|
40 whitespace: whitespaceNode; |
42 whitespace: whitespaceNode; |
41 name: #'mainParser'; |
43 tokens: tokensNode; |
42 yourself |
44 name: #mainParser; |
43 ]. |
45 yourself ]. |
44 ^ parserNode |
46 ^ parserNode |
45 |
47 |
46 "Modified: / 12-05-2015 / 01:37:57 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
48 "Modified: / 12-05-2015 / 01:37:57 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
47 ! |
49 ! |
48 |
50 |
49 eofToken |
51 eofToken |
50 | ws | |
52 | ws | |
|
53 self error: 'deprecated?'. |
51 ws := PPCStarNode new |
54 ws := PPCStarNode new |
52 child: (PPCMessagePredicateNode new |
55 child: (PPCMessagePredicateNode new |
53 message: #isSeparator; |
56 message: #isSeparator; |
54 yourself); |
57 yourself); |
55 yourself. |
58 yourself. |
56 |
59 |
57 ^ PPCTrimmingTokenNode new |
60 ^ PPCTrimmingTokenNode new |
58 child: PPCEndOfFileNode new; |
61 child: PPCEndOfFileNode new; |
59 whitespace: ws; |
62 whitespace: ws; |
60 tokenClass: PPToken; |
63 tokenClass: PPToken; |
|
64 name: 'eof' |
61 yourself. |
65 yourself. |
62 ! ! |
66 ! ! |
63 |
67 |
64 !PPCTokenizingVisitor methodsFor:'initialization'! |
68 !PPCTokenizingVisitor methodsFor:'initialization'! |
65 |
69 |
95 ]. |
100 ]. |
96 |
101 |
97 ^ super visitActionNode: node |
102 ^ super visitActionNode: node |
98 ! |
103 ! |
99 |
104 |
|
105 visitTokenConsumeNode: node |
|
106 " |
|
107 Seems, it might happen, that if I create the consume node, |
|
108 I will ge to it later. This would create a token consume node for the |
|
109 child, thus having tokenConsumNode with tokenConsumNode as a child... |
|
110 " |
|
111 ^ node |
|
112 ! |
|
113 |
100 visitTokenNode: node |
114 visitTokenNode: node |
101 self change. |
115 self change. |
102 self addToken: node. |
116 self addToken: node. |
|
117 |
|
118 self assert: node acceptsEpsilon not description: 'Sorry, but the epsilon tokens are not allowed'. |
103 |
119 |
104 ^ PPCTokenConsumeNode new |
120 ^ PPCTokenConsumeNode new |
105 child: node; |
121 child: node; |
106 yourself. |
122 yourself. |
107 ! |
123 ! |