equal
deleted
inserted
replaced
2 |
2 |
3 "{ NameSpace: Smalltalk }" |
3 "{ NameSpace: Smalltalk }" |
4 |
4 |
5 PPAbstractParserTest subclass:#PPCTokenizingCodeGeneratorTest |
5 PPAbstractParserTest subclass:#PPCTokenizingCodeGeneratorTest |
6 instanceVariableNames:'visitor node result compiler parser context arguments tokenizer |
6 instanceVariableNames:'visitor node result compiler parser context arguments tokenizer |
7 whitespace' |
7 whitespace configuration' |
8 classVariableNames:'' |
8 classVariableNames:'' |
9 poolDictionaries:'' |
9 poolDictionaries:'' |
10 category:'PetitCompiler-Tests-Visitors' |
10 category:'PetitCompiler-Tests-Visitors' |
11 ! |
11 ! |
12 |
12 |
23 scannerClass notNil ifTrue:[ |
23 scannerClass notNil ifTrue:[ |
24 scannerClass removeFromSystem |
24 scannerClass removeFromSystem |
25 ]. |
25 ]. |
26 ! |
26 ! |
27 |
27 |
28 compileTokenizer: aNode |
|
29 tokenizer := visitor visit: aNode |
|
30 ! |
|
31 |
|
32 compileTree: root |
28 compileTree: root |
33 |
|
34 | configuration | |
|
35 |
|
36 configuration := PPCPluggableConfiguration on: [ :_self | |
|
37 _self cacheFirstFollow. |
|
38 _self generateScanner. |
|
39 _self generate. |
|
40 |
|
41 ]. |
|
42 configuration arguments: arguments. |
|
43 configuration base: PPCConfiguration tokenizing. |
|
44 parser := configuration compile: root. |
29 parser := configuration compile: root. |
45 |
30 |
46 ! |
|
47 |
|
48 compileWs: aNode |
|
49 whitespace := visitor visit: aNode |
|
50 ! |
31 ! |
51 |
32 |
52 context |
33 context |
53 ^ context := PPCProfilingContext new |
34 ^ context := PPCProfilingContext new |
54 ! |
35 ! |
58 profile: true; |
39 profile: true; |
59 yourself. |
40 yourself. |
60 |
41 |
61 self cleanClass. |
42 self cleanClass. |
62 |
43 |
63 compiler := PPCTokenizingCodeGen new. |
44 configuration := PPCPluggableConfiguration on: [ :_self | |
64 compiler arguments: arguments. |
45 _self cacheFirstFollow. |
65 |
46 _self buildParserClazz. |
66 visitor := PPCTokenizingCodeGenerator new. |
47 _self unmarkConsumeTokensForInline. |
67 visitor compiler: compiler. |
48 _self createFSAs. |
68 visitor arguments: arguments. |
49 _self buildScannerTokens. |
|
50 _self buildScannerScans. |
|
51 _self generateScanner. |
|
52 _self generateParser. |
|
53 ] base: PPCConfiguration tokenizing. |
|
54 |
|
55 configuration arguments: arguments. |
69 ! |
56 ! |
70 |
57 |
71 tearDown |
58 tearDown |
72 "nothing to do now" |
59 "nothing to do now" |
73 ! ! |
60 ! ! |
76 |
63 |
77 assert: whatever parse: input |
64 assert: whatever parse: input |
78 result := super assert: whatever parse: input. |
65 result := super assert: whatever parse: input. |
79 ! |
66 ! |
80 |
67 |
81 assert: whatever recognizesToken: input |
|
82 whatever startSymbol: #nextToken. |
|
83 |
|
84 self assert: whatever parse: input. |
|
85 self assert: (result isKindOf: PPToken). |
|
86 |
|
87 whatever startSymbol: #start |
|
88 ! |
|
89 |
|
90 assert: whatever rejectsToken: input |
|
91 whatever startSymbol: #nextToken. |
|
92 |
|
93 self assert: whatever fail: input. |
|
94 |
|
95 whatever startSymbol: #start |
|
96 ! |
|
97 |
|
98 literalNode: literal |
68 literalNode: literal |
99 ^ PPCLiteralNode new |
69 ^ PPCLiteralNode new |
100 literal: literal; |
70 literal: literal; |
101 yourself |
|
102 ! |
|
103 |
|
104 tokenNode: child |
|
105 child markForInline. |
|
106 |
|
107 ^ PPCTokenNode new |
|
108 child: child; |
|
109 tokenClass: PPToken; |
|
110 yourself |
71 yourself |
111 ! |
72 ! |
112 |
73 |
113 tokenNodeForEOF |
74 tokenNodeForEOF |
114 | eof | |
75 | eof | |
169 tokenNode := PPCListNode new |
130 tokenNode := PPCListNode new |
170 children: { token1 . token2 . eof }; |
131 children: { token1 . token2 . eof }; |
171 name: 'nextToken'; |
132 name: 'nextToken'; |
172 yourself. |
133 yourself. |
173 |
134 |
174 wsNode := PPCTokenStarSeparatorNode new |
135 wsNode := PPCNilNode new |
175 name: 'consumeWhitespace'; |
136 name: 'consumeWhitespace'; |
176 child: PPCNilNode new; |
|
177 yourself. |
137 yourself. |
178 |
138 |
179 node := PPCTokenizingParserNode new |
139 node := PPCTokenizingParserNode new |
180 tokens: tokenNode; |
140 tokens: tokenNode; |
181 whitespace: wsNode; |
141 whitespace: wsNode; |
206 name: 'nextToken'; |
166 name: 'nextToken'; |
207 yourself. |
167 yourself. |
208 consumeNode := PPCTokenConsumeNode new |
168 consumeNode := PPCTokenConsumeNode new |
209 child: tokenNode; |
169 child: tokenNode; |
210 yourself. |
170 yourself. |
211 wsNode := PPCTokenStarSeparatorNode new |
171 wsNode := PPCNilNode new |
212 name: 'consumeWhitespace'; |
172 name: 'consumeWhitespace'; |
213 child: PPCSentinelNode instance; |
|
214 yourself. |
173 yourself. |
215 |
174 |
216 node := PPCTokenizingParserNode new |
175 node := PPCTokenizingParserNode new |
217 parser: consumeNode; |
176 parser: consumeNode; |
218 tokens: tokenizerNode; |
177 tokens: tokenizerNode; |
229 parser := parser class new. |
188 parser := parser class new. |
230 self assert: parser fail: 'foo'. |
189 self assert: parser fail: 'foo'. |
231 ! |
190 ! |
232 |
191 |
233 testTrimmingToken1 |
192 testTrimmingToken1 |
234 | token tokenConsume tokensNode eof wsNode | |
193 | token tokenConsume tokensNode eof wsNode separatorNode | |
235 |
194 |
236 token := self trimmingTokenNode: (self literalNode: 'foo'). |
195 token := self trimmingTokenNode: (self literalNode: 'foo'). |
237 eof := (self tokenNodeForEOF) yourself. |
196 eof := (self tokenNodeForEOF) yourself. |
238 |
197 |
239 tokenConsume := PPCTokenConsumeNode new |
198 tokenConsume := PPCTokenConsumeNode new |
243 tokensNode := PPCListNode new |
202 tokensNode := PPCListNode new |
244 children: { token . eof }; |
203 children: { token . eof }; |
245 name: 'nextToken'; |
204 name: 'nextToken'; |
246 yourself. |
205 yourself. |
247 |
206 |
248 wsNode := PPCTokenStarSeparatorNode new |
207 separatorNode := PPCLiteralNode new |
|
208 literal: ' '; |
|
209 name: 'separator'; |
|
210 yourself. |
|
211 |
|
212 wsNode := PPCStarNode new |
249 name: 'consumeWhitespace'; |
213 name: 'consumeWhitespace'; |
250 child: PPCNilNode new; |
214 child: separatorNode; |
251 yourself. |
215 yourself. |
252 |
216 |
253 node := PPCTokenizingParserNode new |
217 node := PPCTokenizingParserNode new |
254 tokens: tokensNode; |
218 tokens: tokensNode; |
255 whitespace: wsNode; |
219 whitespace: wsNode; |