1 "{ Package: 'stx:goodies/petitparser/compiler/tests' }" |
1 "{ Package: 'stx:goodies/petitparser/compiler/tests' }" |
2 |
2 |
3 "{ NameSpace: Smalltalk }" |
3 "{ NameSpace: Smalltalk }" |
4 |
4 |
5 PPAbstractParserTest subclass:#PPCTokenizingTest |
5 PPAbstractParserTest subclass:#PPCTokenizingTest |
6 instanceVariableNames:'parser result context node compiler id node2 id2 id1 node1 node3 |
6 instanceVariableNames:'parser result context node id node2 id2 id1 node1 node3 options |
7 options configuration' |
7 compiler' |
8 classVariableNames:'' |
8 classVariableNames:'' |
9 poolDictionaries:'' |
9 poolDictionaries:'' |
10 category:'PetitCompiler-Tests-Core-Tokenizing' |
10 category:'PetitCompiler-Tests-Core-Tokenizing' |
11 ! |
11 ! |
12 |
12 |
42 parse: whatever |
42 parse: whatever |
43 ^ result := super parse: whatever. |
43 ^ result := super parse: whatever. |
44 ! |
44 ! |
45 |
45 |
46 setUp |
46 setUp |
47 options := PPCCompilationOptions default |
47 options := (PPCCompilationOptions default) |
48 profile: true; |
48 profile:true; |
49 tokenize: true; |
49 tokenize:true; |
50 yourself. |
50 yourself. |
51 |
51 compiler := PPCCompiler new. |
52 configuration := PPCConfiguration new. |
52 compiler context options:options. |
53 configuration context options: options. |
|
54 |
|
55 self cleanClass. |
53 self cleanClass. |
56 |
54 |
57 "Modified: / 04-09-2015 / 16:21:36 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
55 "Modified: / 04-09-2015 / 16:21:36 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
58 ! |
56 ! |
59 |
57 |
87 self assert: parser parse: 'c' end: 0. |
85 self assert: parser parse: 'c' end: 0. |
88 |
86 |
89 ! |
87 ! |
90 |
88 |
91 testChoiceOrder |
89 testChoiceOrder |
92 parser := ( |
90 parser := (('a' asParser token , 'b' asParser token) / 'a' asParser token) |
93 'a' asParser token, 'b' asParser token / |
91 compileUsingCompiler:compiler. |
94 'a' asParser token) |
|
95 compileWithConfiguration: configuration. |
|
96 |
92 |
97 self assert: parser parse: 'ab'. |
93 self assert: parser parse: 'ab'. |
98 self assert: result first inputValue = 'a'. |
94 self assert: result first inputValue = 'a'. |
99 self assert: result second inputValue = 'b'. |
95 self assert: result second inputValue = 'b'. |
100 |
96 |
108 testChoiceOrder2 |
104 testChoiceOrder2 |
109 | p1 p2 | |
105 | p1 p2 | |
110 p1 := 'a' asParser token, 'b' asParser token. |
106 p1 := 'a' asParser token, 'b' asParser token. |
111 p2 := 'b' asParser token / 'a' asParser token. |
107 p2 := 'b' asParser token / 'a' asParser token. |
112 |
108 |
113 parser := p1 / p2 compileWithConfiguration: configuration. |
109 parser := p1 / p2 compileUsingCompiler:compiler. |
114 |
110 |
115 self assert: parser parse: 'ab'. |
111 self assert: parser parse: 'ab'. |
116 self assert: result first inputValue = 'a'. |
112 self assert: result first inputValue = 'a'. |
117 self assert: result second inputValue = 'b'. |
113 self assert: result second inputValue = 'b'. |
118 |
114 |
132 a2 := 'a' asParser token name: 't2'; yourself. |
128 a2 := 'a' asParser token name: 't2'; yourself. |
133 |
129 |
134 p1 := a1, 'b' asParser token. |
130 p1 := a1, 'b' asParser token. |
135 p2 := a2. |
131 p2 := a2. |
136 |
132 |
137 parser := p1 / p2 compileWithConfiguration: configuration. |
133 parser := p1 / p2 compileUsingCompiler:compiler. |
138 |
134 |
139 self assert: parser parse: 'ab'. |
135 self assert: parser parse: 'ab'. |
140 self assert: result first inputValue = 'a'. |
136 self assert: result first inputValue = 'a'. |
141 self assert: result second inputValue = 'b'. |
137 self assert: result second inputValue = 'b'. |
142 |
138 |
153 a2 := 'a' asParser token name: 't2'; yourself. |
149 a2 := 'a' asParser token name: 't2'; yourself. |
154 |
150 |
155 p1 := a1, 'b' asParser token. |
151 p1 := a1, 'b' asParser token. |
156 p2 := 'b' asParser token / a2. |
152 p2 := 'b' asParser token / a2. |
157 |
153 |
158 parser := p1 / p2 compileWithConfiguration: configuration. |
154 parser := p1 / p2 compileUsingCompiler:compiler. |
159 |
155 |
160 self assert: parser parse: 'ab'. |
156 self assert: parser parse: 'ab'. |
161 self assert: result first inputValue = 'a'. |
157 self assert: result first inputValue = 'a'. |
162 self assert: result second inputValue = 'b'. |
158 self assert: result second inputValue = 'b'. |
163 |
159 |
170 self assert: parser fail: 'c'. |
166 self assert: parser fail: 'c'. |
171 |
167 |
172 ! |
168 ! |
173 |
169 |
174 testCompileAnd |
170 testCompileAnd |
175 parser := (('foo' asParser token and) / ('bar' asParser token and)), 'bar' asParser token |
171 parser := (('foo' asParser token and) / ('bar' asParser token and)) |
176 compileWithConfiguration: configuration. |
172 , 'bar' asParser token compileUsingCompiler:compiler. |
177 |
173 |
178 self assert: parser parse: 'bar'. |
174 self assert: parser parse: 'bar'. |
179 self assert: result second inputValue = 'bar'. |
175 self assert: result second inputValue = 'bar'. |
180 ! |
176 ! |
181 |
177 |
182 testCompileChoice |
178 testCompileChoice |
183 parser := ('foo' asParser / 'bar' asParser) token compileWithConfiguration: configuration. |
179 parser := ('foo' asParser / 'bar' asParser) token |
|
180 compileUsingCompiler:compiler. |
184 |
181 |
185 self assert: parser parse: 'foo'. |
182 self assert: parser parse: 'foo'. |
186 self assert: result inputValue = 'foo'. |
183 self assert: result inputValue = 'foo'. |
187 self assert: parser parse: 'bar'. |
184 self assert: parser parse: 'bar'. |
188 self assert: result inputValue = 'bar'. |
185 self assert: result inputValue = 'bar'. |
189 self assert: parser fail: '_'. |
186 self assert: parser fail: '_'. |
190 |
187 |
191 ! |
188 ! |
192 |
189 |
193 testCompileChoice2 |
190 testCompileChoice2 |
194 parser := ('foo' asParser token trim / 'bar' asParser token trim) compileWithConfiguration: configuration. |
191 parser := ('foo' asParser token trim / 'bar' asParser token trim) |
|
192 compileUsingCompiler:compiler. |
195 |
193 |
196 self assert: parser parse: 'foo'. |
194 self assert: parser parse: 'foo'. |
197 self assert: result inputValue = 'foo'. |
195 self assert: result inputValue = 'foo'. |
198 self assert: parser parse: 'bar'. |
196 self assert: parser parse: 'bar'. |
199 self assert: result inputValue = 'bar'. |
197 self assert: result inputValue = 'bar'. |
200 self assert: parser fail: '_'. |
198 self assert: parser fail: '_'. |
201 |
199 |
202 ! |
200 ! |
203 |
201 |
204 testCompileComplex1 |
202 testCompileComplex1 |
205 parser := ('foo' asParser token, 'bar' asParser token) / |
203 parser := ('foo' asParser token , 'bar' asParser token) |
206 ('foo' asParser token, 'baz' asParser token) compileWithConfiguration: configuration. |
204 / ('foo' asParser token , 'baz' asParser token) |
|
205 compileUsingCompiler:compiler. |
207 |
206 |
208 self assert: parser parse: 'foobar'. |
207 self assert: parser parse: 'foobar'. |
209 self assert: result second inputValue = 'bar'. |
208 self assert: result second inputValue = 'bar'. |
210 |
209 |
211 self assert: parser parse: 'foobaz'. |
210 self assert: parser parse: 'foobaz'. |
214 self assert: parser fail: 'foobaq'. |
213 self assert: parser fail: 'foobaq'. |
215 |
214 |
216 ! |
215 ! |
217 |
216 |
218 testCompileComplex2 |
217 testCompileComplex2 |
219 parser := ('foo' asParser token, 'bar' asParser token) star, 'foo' asParser token |
218 parser := ('foo' asParser token , 'bar' asParser token) star , 'foo' asParser token |
220 compileWithConfiguration: configuration. |
219 compileUsingCompiler:compiler. |
221 |
220 |
222 self assert: parser parse: 'foobarfoobarfoo'. |
221 self assert: parser parse: 'foobarfoobarfoo'. |
223 self assert: parser parse: 'foo'. |
222 self assert: parser parse: 'foo'. |
224 |
223 |
225 self assert: parser fail: 'bar'. |
224 self assert: parser fail: 'bar'. |
226 |
225 |
227 ! |
226 ! |
228 |
227 |
229 testCompileComplex3 |
228 testCompileComplex3 |
230 parser := ('foo' asParser token, 'bar' asParser token) star, 'foo' asParser token / |
229 parser := (('foo' asParser token , 'bar' asParser token) star , 'foo' asParser token) |
231 ('foo' asParser token, 'baz' asParser token) |
230 / ('foo' asParser token , 'baz' asParser token) |
232 compileWithConfiguration: configuration. |
231 compileUsingCompiler:compiler. |
233 |
232 |
234 self assert: parser parse: 'foobarfoobarfoo'. |
233 self assert: parser parse: 'foobarfoobarfoo'. |
235 self assert: parser parse: 'foo'. |
234 self assert: parser parse: 'foo'. |
236 |
235 |
237 self assert: parser fail: 'bar'. |
236 self assert: parser fail: 'bar'. |
243 start := $( asParser token. |
242 start := $( asParser token. |
244 stop := $) asParser token. |
243 stop := $) asParser token. |
245 epsilon := '' asParser token. |
244 epsilon := '' asParser token. |
246 |
245 |
247 self should: [ |
246 self should: [ |
248 (start, epsilon, stop) compileWithConfiguration: configuration. |
247 (start , epsilon , stop) compileUsingCompiler:compiler. |
249 ] raise: Exception. |
248 ] raise: Exception. |
250 " |
249 " |
251 self assert: parser parse: '()'. |
250 self assert: parser parse: '()'. |
252 self assert: parser fail: '('. |
251 self assert: parser fail: '('. |
253 " |
252 " |
254 ! |
253 ! |
255 |
254 |
256 testCompileLiteral |
255 testCompileLiteral |
257 parser := 'foo' asParser token compileWithConfiguration: configuration. |
256 parser := 'foo' asParser token compileUsingCompiler:compiler. |
258 |
257 |
259 self assert: parser parse: 'foo'. |
258 self assert: parser parse: 'foo'. |
260 self assert: result inputValue = 'foo'. |
259 self assert: result inputValue = 'foo'. |
261 self assert: parser fail: 'boo'. |
260 self assert: parser fail: 'boo'. |
262 ! |
261 ! |
263 |
262 |
264 testCompileSequence |
263 testCompileSequence |
265 parser := ('foo' asParser token), ('bar' asParser token) |
264 parser := ('foo' asParser token) , ('bar' asParser token) |
266 compileWithConfiguration: configuration. |
265 compileUsingCompiler:compiler. |
267 |
266 |
268 self assert: parser parse: 'foobar'. |
267 self assert: parser parse: 'foobar'. |
269 self assert: result first inputValue = 'foo'. |
268 self assert: result first inputValue = 'foo'. |
270 self assert: result second inputValue = 'bar'. |
269 self assert: result second inputValue = 'bar'. |
271 ! |
270 ! |
272 |
271 |
273 testCompileSequence2 |
272 testCompileSequence2 |
274 parser := ('foo' asParser trimmingToken), ('bar' asParser trimmingToken) |
273 parser := ('foo' asParser trimmingToken) , ('bar' asParser trimmingToken) |
275 compileWithConfiguration: configuration. |
274 compileUsingCompiler:compiler. |
276 |
275 |
277 self assert: parser parse: 'foobar'. |
276 self assert: parser parse: 'foobar'. |
278 self assert: result first inputValue = 'foo'. |
277 self assert: result first inputValue = 'foo'. |
279 self assert: result second inputValue = 'bar'. |
278 self assert: result second inputValue = 'bar'. |
280 |
279 |
286 self assert: result first inputValue = 'foo'. |
285 self assert: result first inputValue = 'foo'. |
287 self assert: result second inputValue = 'bar'. |
286 self assert: result second inputValue = 'bar'. |
288 ! |
287 ! |
289 |
288 |
290 testCompileSequence3 |
289 testCompileSequence3 |
291 parser := ('foo' asParser trimmingToken), |
290 parser := ('foo' asParser trimmingToken) , ('bar' asParser trimmingToken) |
292 ('bar' asParser trimmingToken), |
291 , ('baz' asParser trimmingToken) compileUsingCompiler:compiler. |
293 ('baz' asParser trimmingToken) |
|
294 compileWithConfiguration: configuration. |
|
295 |
292 |
296 self assert: parser parse: 'foobarbaz'. |
293 self assert: parser parse: 'foobarbaz'. |
297 self assert: result first inputValue = 'foo'. |
294 self assert: result first inputValue = 'foo'. |
298 self assert: result second inputValue = 'bar'. |
295 self assert: result second inputValue = 'bar'. |
299 |
296 |
302 self assert: result second inputValue = 'bar'. |
299 self assert: result second inputValue = 'bar'. |
303 self assert: result third inputValue = 'baz'. |
300 self assert: result third inputValue = 'baz'. |
304 ! |
301 ! |
305 |
302 |
306 testCompileStar |
303 testCompileStar |
307 parser := 'foo' asParser token star compileWithConfiguration: configuration. |
304 parser := 'foo' asParser token star compileUsingCompiler:compiler. |
308 |
305 |
309 self assert: parser parse: 'foo'. |
306 self assert: parser parse: 'foo'. |
310 self assert: result first inputValue = 'foo'. |
307 self assert: result first inputValue = 'foo'. |
311 |
308 |
312 self assert: parser parse: 'boo' end: 0. |
309 self assert: parser parse: 'boo' end: 0. |
313 self assert: result isEmpty. |
310 self assert: result isEmpty. |
314 ! |
311 ! |
315 |
312 |
316 testCompileStar2 |
313 testCompileStar2 |
317 parser := ('foo' asParser token, 'bar' asParser token) star compileWithConfiguration: configuration. |
314 parser := ('foo' asParser token , 'bar' asParser token) star |
|
315 compileUsingCompiler:compiler. |
318 |
316 |
319 self assert: parser parse: 'foobar'. |
317 self assert: parser parse: 'foobar'. |
320 self assert: context tokenReads size = 1. |
318 self assert: context tokenReads size = 1. |
321 |
319 |
322 self assert: parser parse: 'bar' end: 0. |
320 self assert: parser parse: 'bar' end: 0. |
324 self assert: context tokenReads size = 1. |
322 self assert: context tokenReads size = 1. |
325 |
323 |
326 ! |
324 ! |
327 |
325 |
328 testCompileStar3 |
326 testCompileStar3 |
329 parser := 'a' asParser trimmingToken star, 'b' asParser trimmingToken |
327 parser := 'a' asParser trimmingToken star , 'b' asParser trimmingToken |
330 compileWithConfiguration: configuration. |
328 compileUsingCompiler:compiler. |
331 |
329 |
332 self assert: parser parse: 'ab'. |
330 self assert: parser parse: 'ab'. |
333 self assert: parser parse: 'aaab'. |
331 self assert: parser parse: 'aaab'. |
334 self assert: result size = 2. |
332 self assert: result size = 2. |
335 self assert: result first size = 3. |
333 self assert: result first size = 3. |
372 name: 'choice1'; |
370 name: 'choice1'; |
373 yourself. |
371 yourself. |
374 |
372 |
375 tricky := (a1 asParser, choice1) / (b2 asParser, choice2). |
373 tricky := (a1 asParser, choice1) / (b2 asParser, choice2). |
376 |
374 |
377 parser := tricky compileWithConfiguration: configuration. |
375 parser := tricky compileUsingCompiler:compiler. |
378 self assert: parser parse: '||'. |
376 self assert: parser parse: '||'. |
379 |
377 |
380 self assert: parser parse: '|]'. |
378 self assert: parser parse: '|]'. |
381 |
379 |
382 self assert: parser parse: ']|'. |
380 self assert: parser parse: ']|'. |
409 arrayLiteral setParser: '#(' asParser token, symbolLiteralArray, ')' asParser token. |
407 arrayLiteral setParser: '#(' asParser token, symbolLiteralArray, ')' asParser token. |
410 arrayLiteral name: 'arrayLiteral'. |
408 arrayLiteral name: 'arrayLiteral'. |
411 |
409 |
412 arrayItem := arrayLiteral / symbolLiteral. |
410 arrayItem := arrayLiteral / symbolLiteral. |
413 |
411 |
414 parser := arrayItem compileWithConfiguration: configuration. |
412 parser := arrayItem compileUsingCompiler:compiler. |
415 |
413 |
416 self assert: parser parse: '#(foo)'. |
414 self assert: parser parse: '#(foo)'. |
417 self assert: parser parse: '#foo'. |
415 self assert: parser parse: '#foo'. |
418 |
416 |
419 "Modified (comment): / 17-08-2015 / 23:07:35 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
417 "Modified (comment): / 17-08-2015 / 23:07:35 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
420 ! |
418 ! |
421 |
419 |
422 testCompileTrim |
420 testCompileTrim |
423 parser := 'foo' asParser token trim end compileWithConfiguration: configuration. |
421 parser := 'foo' asParser token trim end compileUsingCompiler:compiler. |
424 |
422 |
425 self assert: parser parse: 'foo'. |
423 self assert: parser parse: 'foo'. |
426 self assert: result inputValue = 'foo'. |
424 self assert: result inputValue = 'foo'. |
427 |
425 |
428 self assert: parser parse: 'foo '. |
426 self assert: parser parse: 'foo '. |
453 ! |
450 ! |
454 |
451 |
455 testTokenCharacter2 |
452 testTokenCharacter2 |
456 | token | |
453 | token | |
457 token := $a asParser token. |
454 token := $a asParser token. |
458 parser := token plus |
455 parser := token plus compileUsingCompiler:compiler. |
459 compileWithConfiguration: configuration. |
|
460 |
456 |
461 self assert: parser parse: 'aaa'. |
457 self assert: parser parse: 'aaa'. |
462 self assert: result first inputValue = 'a'. |
458 self assert: result first inputValue = 'a'. |
463 self assert: result second inputValue = 'a'. |
459 self assert: result second inputValue = 'a'. |
464 self assert: result third inputValue = 'a'. |
460 self assert: result third inputValue = 'a'. |
469 ! |
465 ! |
470 |
466 |
471 testTokenName |
467 testTokenName |
472 | token | |
468 | token | |
473 token := 'foo' asParser token name: 'fooToken'; yourself. |
469 token := 'foo' asParser token name: 'fooToken'; yourself. |
474 parser := token plus |
470 parser := token plus compileUsingCompiler:compiler. |
475 compileWithConfiguration: configuration. |
|
476 |
471 |
477 self assert: parser parse: 'foofoo'. |
472 self assert: parser parse: 'foofoo'. |
478 self assert: result first inputValue = 'foo'. |
473 self assert: result first inputValue = 'foo'. |
479 self assert: result second inputValue = 'foo'. |
474 self assert: result second inputValue = 'foo'. |
480 self assert: (parser scanner class methodDictionary includesKey: #fooToken). |
475 self assert: (parser scanner class methodDictionary includesKey: #fooToken). |
482 ! |
477 ! |
483 |
478 |
484 testWhitespace |
479 testWhitespace |
485 | token ws trimmingToken | |
480 | token ws trimmingToken | |
486 |
481 |
487 configuration removePass: PPCInliningVisitor. |
482 compiler removePass: PPCInliningVisitor. |
488 token := 'foo' asParser token. |
483 token := 'foo' asParser token. |
489 ws := #blank asParser star name: 'consumeWhitespace'; yourself. |
484 ws := #blank asParser star name: 'consumeWhitespace'; yourself. |
490 trimmingToken := ((ws, token, ws) ==> #second) |
485 trimmingToken := ((ws, token, ws) ==> #second) |
491 propertyAt: 'trimmingToken' put: true; |
486 propertyAt: 'trimmingToken' put: true; |
492 yourself. |
487 yourself. |
493 |
488 |
494 parser := trimmingToken plus |
489 parser := trimmingToken plus compileUsingCompiler:compiler. |
495 compileWithConfiguration: configuration. |
|
496 |
490 |
497 self assert: parser parse: ' foo '. |
491 self assert: parser parse: ' foo '. |
498 self assert: result first inputValue = 'foo'. |
492 self assert: result first inputValue = 'foo'. |
499 |
493 |
500 self assert: (context invocations select: [:e | e = #scan_consumeWhitespace ]) size = 3. |
494 self assert: (context invocations select: [:e | e = #scan_consumeWhitespace ]) size = 3. |
503 ! |
497 ! |
504 |
498 |
505 testWhitespace2 |
499 testWhitespace2 |
506 | token ws trimmingToken | |
500 | token ws trimmingToken | |
507 |
501 |
508 configuration removePass: PPCInliningVisitor. |
502 compiler removePass: PPCInliningVisitor. |
509 token := 'foo' asParser token. |
503 token := 'foo' asParser token. |
510 ws := #blank asParser star name: 'consumeWhitespace'; yourself. |
504 ws := #blank asParser star name: 'consumeWhitespace'; yourself. |
511 trimmingToken := ((ws, token, ws) ==> #second) |
505 trimmingToken := ((ws, token, ws) ==> #second) |
512 propertyAt: 'trimmingToken' put: true; |
506 propertyAt: 'trimmingToken' put: true; |
513 yourself. |
507 yourself. |
514 |
508 |
515 parser := trimmingToken plus |
509 parser := trimmingToken plus compileUsingCompiler:compiler. |
516 compileWithConfiguration: configuration. |
|
517 |
510 |
518 self assert: parser parse: ' foo foo '. |
511 self assert: parser parse: ' foo foo '. |
519 self assert: result first inputValue = 'foo'. |
512 self assert: result first inputValue = 'foo'. |
520 self assert: result second inputValue = 'foo'. |
513 self assert: result second inputValue = 'foo'. |
521 |
514 |
525 ! |
518 ! |
526 |
519 |
527 testWhitespace3 |
520 testWhitespace3 |
528 | token ws trimmingToken | |
521 | token ws trimmingToken | |
529 |
522 |
530 configuration removePass: PPCInliningVisitor. |
523 compiler removePass: PPCInliningVisitor. |
531 token := 'foo' asParser token. |
524 token := 'foo' asParser token. |
532 ws := #blank asParser star name: 'consumeWhitespace'; yourself. |
525 ws := #blank asParser star name: 'consumeWhitespace'; yourself. |
533 trimmingToken := ((ws, token, ws) ==> #second) |
526 trimmingToken := ((ws, token, ws) ==> #second) |
534 propertyAt: 'trimmingToken' put: true; |
527 propertyAt: 'trimmingToken' put: true; |
535 yourself. |
528 yourself. |
536 |
529 |
537 parser := trimmingToken plus |
530 parser := trimmingToken plus compileUsingCompiler:compiler. |
538 compileWithConfiguration: configuration. |
|
539 |
531 |
540 self assert: parser parse: ' foo foo foo '. |
532 self assert: parser parse: ' foo foo foo '. |
541 self assert: result first inputValue = 'foo'. |
533 self assert: result first inputValue = 'foo'. |
542 self assert: result second inputValue = 'foo'. |
534 self assert: result second inputValue = 'foo'. |
543 self assert: result third inputValue = 'foo'. |
535 self assert: result third inputValue = 'foo'. |