1 "{ Package: 'stx:goodies/petitparser/compiler/tests' }"

     2 

     3 "{ NameSpace: Smalltalk }"

     4 

     5 PPAbstractParserTest subclass:#PPCTokenizingTest

     6 	instanceVariableNames:'parser result context node compiler id node2 id2 id1 node1 node3

     7 		arguments configuration'

     8 	classVariableNames:''

     9 	poolDictionaries:''

    10 	category:'PetitCompiler-Tests-Core-Tokenizing'

    11 !

    12 

    13 !PPCTokenizingTest methodsFor:'as yet unclassified'!

    14 

    15 assert: p parse: whatever

    16     ^ result := super assert: p parse: whatever.

    17 !

    18 

    19 assert: p parse: whatever end: end

    20     ^ result := super assert: p parse: whatever end: end

    21 !

    22 

    23 cleanClass

    24     | parserClass |

    25     parserClass := (Smalltalk at: arguments name ifAbsent: [nil]).

    26     parserClass notNil ifTrue:[ 

    27         self flag: 'uncomment'.

    28 "		parserClass removeFromSystem"

    29     ].

    30 !

    31 

    32 context	

    33     ^ context := PPCProfilingContext new

    34 !

    35 

    36 parse: whatever

    37     ^ result := super parse: whatever.

    38 !

    39 

    40 setUp

    41     arguments := PPCArguments default

    42         profile: true;

    43         yourself.

    44         

    45     configuration := PPCTokenizingConfiguration new

    46         arguments: arguments;

    47         yourself.

    48         

    49     self cleanClass.

    50 !

    51 

    52 tearDown

    53     self cleanClass

    54 !

    55 

    56 testChoiceOrder

    57     parser := (

    58         'a' asParser token, 'b' asParser token / 

    59         'a' asParser token) 

    60         compileWithConfiguration: configuration.

    61     

    62     self assert: parser parse: 'ab'.

    63     self assert: result first inputValue = 'a'.

    64     self assert: result second inputValue = 'b'.

    65 

    66     self assert: parser parse: 'a'.

    67     self assert: result inputValue = 'a'.

    68 

    69     self assert: parser fail: '_'.

    70     

    71 !

    72 

    73 testChoiceOrder2

    74     | p1 p2 |

    75     p1 := 'a' asParser token, 'b' asParser token.

    76     p2 := 'b' asParser token / 'a' asParser token.

    77     

    78     parser := p1 / p2	compileWithConfiguration: configuration.

    79     

    80     self assert: parser parse: 'ab'.

    81     self assert: result first inputValue = 'a'.

    82     self assert: result second inputValue = 'b'.

    83 

    84     self assert: parser parse: 'a'.

    85     self assert: result inputValue = 'a'.

    86 

    87     self assert: parser parse: 'b'.

    88     self assert: result inputValue = 'b'.

    89 

    90     self assert: parser fail: 'c'.

    91     

    92 !

    93 

    94 testChoiceOrder3

    95     | p1 p2 a1 a2 |

    96     a1 := 'a' asParser token name: 't1'; yourself.

    97     a2 := 'a' asParser token name: 't2'; yourself.

    98     

    99     p1 := a1, 'b' asParser token.

   100     p2 := a2.

   101     

   102     parser := p1 / p2	compileWithConfiguration: configuration.

   103     

   104     self assert: parser parse: 'ab'.

   105     self assert: result first inputValue = 'a'.

   106     self assert: result second inputValue = 'b'.

   107 

   108     self assert: parser parse: 'a'.

   109     self assert: result inputValue = 'a'.

   110 

   111     self assert: parser fail: 'b'.

   112     

   113 !

   114 

   115 testChoiceOrder4

   116     | p1 p2 a1 a2 |

   117     a1 := 'a' asParser token name: 't1'; yourself.

   118     a2 := 'a' asParser token name: 't2'; yourself.

   119     

   120     p1 := a1, 'b' asParser token.

   121     p2 := 'b' asParser token / a2.

   122     

   123     parser := p1 / p2	compileWithConfiguration: configuration.

   124     

   125     self assert: parser parse: 'ab'.

   126     self assert: result first inputValue = 'a'.

   127     self assert: result second inputValue = 'b'.

   128 

   129     self assert: parser parse: 'a'.

   130     self assert: result inputValue = 'a'.

   131 

   132     self assert: parser parse: 'b'.

   133     self assert: result inputValue = 'b'.

   134 

   135     self assert: parser fail: 'c'.

   136     

   137 !

   138 

   139 testCompileAnd

   140     parser := (('foo' asParser token and) / ('bar' asParser token and)), 'bar' asParser token 

   141         compileWithConfiguration: configuration.

   142     

   143     self assert: parser parse: 'bar'.

   144     self assert: result second inputValue = 'bar'.

   145 !

   146 

   147 testCompileChoice

   148     parser := ('foo' asParser / 'bar' asParser) token compileWithConfiguration: configuration.

   149     

   150     self assert: parser parse: 'foo'.

   151     self assert: result inputValue = 'foo'.

   152     self assert: parser parse: 'bar'.

   153     self assert: result inputValue = 'bar'.

   154     self assert: parser fail: '_'.

   155     

   156 !

   157 

   158 testCompileChoice2

   159     parser := ('foo' asParser token trim / 'bar' asParser token trim) compileWithConfiguration: configuration.

   160     

   161     self assert: parser parse: 'foo'.

   162     self assert: result inputValue = 'foo'.

   163     self assert: parser parse: 'bar'.

   164     self assert: result inputValue = 'bar'.

   165     self assert: parser fail: '_'.

   166     

   167 !

   168 

   169 testCompileComplex1

   170     parser := ('foo' asParser token, 'bar' asParser token) / 

   171                  ('foo' asParser token, 'baz' asParser token) compileWithConfiguration: configuration.

   172     

   173     self assert: parser parse: 'foobar'.

   174     self assert: result second inputValue = 'bar'.

   175 

   176     self assert: parser parse: 'foobaz'.

   177     self assert: result second inputValue = 'baz'.

   178 

   179     self assert: parser fail: 'foobaq'.

   180     

   181 !

   182 

   183 testCompileComplex2

   184     parser := ('foo' asParser token, 'bar' asParser token) star, 'foo' asParser token

   185         compileWithConfiguration: configuration.

   186     

   187     self assert: parser parse: 'foobarfoobarfoo'.

   188     self assert: parser parse: 'foo'.

   189 

   190     self assert: parser fail: 'bar'.

   191     

   192 !

   193 

   194 testCompileComplex3

   195     parser :=	('foo' asParser token, 'bar' asParser token) star, 'foo' asParser token /

   196                 ('foo' asParser token, 'baz' asParser token)

   197         compileWithConfiguration: configuration.

   198     

   199     self assert: parser parse: 'foobarfoobarfoo'.

   200     self assert: parser parse: 'foo'.

   201 

   202     self assert: parser fail: 'bar'.

   203     

   204 !

   205 

   206 testCompileLiteral

   207     parser := 'foo' asParser token compileWithConfiguration: configuration.

   208     

   209     self assert: parser parse: 'foo'.

   210     self assert: result inputValue = 'foo'.

   211     self assert: parser fail: 'boo'.

   212 !

   213 

   214 testCompileSequence

   215     parser := ('foo' asParser token), ('bar' asParser token) 

   216         compileWithConfiguration: configuration.

   217     

   218     self assert: parser parse: 'foobar'.

   219     self assert: result first inputValue = 'foo'.

   220     self assert: result second inputValue = 'bar'.

   221 !

   222 

   223 testCompileStar

   224     parser := 'foo' asParser token star compileWithConfiguration: configuration.

   225     

   226     self assert: parser parse: 'foo'.

   227     self assert: result first inputValue = 'foo'.

   228     

   229     self assert: parser parse: 'boo' end: 0.

   230     self assert: result isEmpty.

   231 !

   232 

   233 testCompileStar2

   234     parser := ('foo' asParser token, 'bar' asParser token) star compileWithConfiguration: configuration.

   235     

   236     self assert: parser parse: 'foobar'.

   237     self assert: context tokenReads size = 3.

   238             

   239     self assert: parser parse: 'bar' end: 0.

   240     self assert: result isEmpty.

   241     self assert: context tokenReads size = 1.

   242         

   243 !

   244 

   245 testCompileTokenComplex2

   246     |  a b argumentsWith  |

   247     "based on the PPSmalltlakGrammar>>blockArgumentsWith"

   248     a := $| asParser smalltalkToken

   249         yourself.

   250     b := $] asParser smalltalkToken

   251         yourself.		

   252     argumentsWith := (a / b and ==> [:t | ]) wrapped

   253         name: 'argumentsWith'; 

   254         yourself.

   255 

   256     parser := argumentsWith compileWithConfiguration: configuration.

   257     self assert: parser parse: '|'.

   258 

   259     parser := argumentsWith compileWithConfiguration: configuration.

   260     self assert: parser parse: ']'.

   261 !

   262 

   263 testCompileTokenComplex3

   264     | choice1 choice2 a1 b1 a2 b2 tricky |

   265     a1 := $| asParser token

   266         yourself.

   267     b1 := $] asParser token

   268         yourself.		

   269     choice1 := (a1 / b1) wrapped

   270         name: 'choice1'; 

   271         yourself.

   272 

   273     a2 := $| asParser token

   274         yourself.

   275     b2 := $] asParser token

   276         yourself.		

   277     choice2 := (a2 / b2) wrapped

   278         name: 'choice1'; 

   279         yourself.

   280     

   281     tricky := (a1 asParser, choice1) / (b2 asParser, choice2).

   282 

   283     parser := tricky compileWithConfiguration: configuration.

   284     self assert: parser parse: '||'.

   285 

   286     parser := tricky compileWithConfiguration: configuration.

   287     self assert: parser parse: '|]'.

   288 

   289     parser := tricky compileWithConfiguration: configuration.

   290     self assert: parser parse: ']|'.

   291 

   292     parser := tricky compileWithConfiguration: configuration.

   293     self assert: parser parse: ']]'.

   294 !

   295 

   296 testCompileTrim

   297     parser := 'foo' asParser token trim end compileWithConfiguration: configuration.

   298     

   299     self assert: parser parse: 'foo'.

   300     self assert: result inputValue = 'foo'.

   301 

   302     self assert: parser parse: 'foo  '.

   303     self assert: result inputValue = 'foo'.

   304 

   305 

   306     self assert: parser parse: '  foo'.

   307     self assert: result inputValue = 'foo'.

   308 

   309     self assert: parser parse: '  foo   '.

   310     self assert: result inputValue = 'foo'.

   311 

   312     self assert: parser fail: 'boo'.

   313 !

   314 

   315 testTokenCharacter

   316     | token |

   317     token := $a asParser token.

   318     parser := token plus

   319         compileWithConfiguration: configuration.

   320 

   321     self assert: parser parse: 'a'.

   322     self assert: result first inputValue = 'a'.

   323     self assert: context invocations size = 5.

   324 !

   325 

   326 testTokenCharacter2

   327     | token |

   328     token := $a asParser token.

   329     parser := token plus

   330         compileWithConfiguration: configuration.

   331 

   332     self assert: parser parse: 'aaa'.

   333     self assert: result first inputValue = 'a'.

   334     self assert: result second inputValue = 'a'.

   335     self assert: result third inputValue = 'a'.

   336     self assert: context invocations size = 7.

   337 !

   338 

   339 testTokenName

   340     | token |

   341     token := 'foo' asParser token name: 'fooToken'; yourself.

   342     parser := token plus

   343         compileWithConfiguration: configuration.

   344 

   345     self assert: parser parse: 'foofoo'.

   346     self assert: result first inputValue = 'foo'.

   347     self assert: result second inputValue = 'foo'.

   348     self assert: (parser class methodDictionary includesKey: #fooToken).

   349 !

   350 

   351 testWhitespace

   352     | token ws trimmingToken |

   353     configuration arguments inline: false.

   354     

   355     token := 'foo' asParser token.

   356     ws := #blank asParser star name: 'consumeWhitespace'; yourself.

   357     trimmingToken := ((ws, token, ws) ==> #second) 

   358         propertyAt: 'trimmingToken' put: true; 

   359         yourself.

   360     

   361     parser := trimmingToken plus

   362         compileWithConfiguration: configuration.

   363 

   364     self assert: parser parse: ' foo '.

   365     self assert: result first inputValue = 'foo'.

   366 

   367     self assert: (context invocations select: [:e | e = #consumeWhitespace ]) size = 2.

   368 !

   369 

   370 testWhitespace2

   371     | token ws trimmingToken |

   372     configuration arguments inline: false.

   373         

   374     token := 'foo' asParser token.

   375     ws := #blank asParser star name: 'consumeWhitespace'; yourself.

   376     trimmingToken := ((ws, token, ws) ==> #second) 

   377         propertyAt: 'trimmingToken' put: true; 

   378         yourself.

   379     

   380     parser := trimmingToken plus

   381         compileWithConfiguration: configuration.

   382 

   383     self assert: parser parse: ' foo foo '.

   384     self assert: result first inputValue = 'foo'.

   385     self assert: result second inputValue = 'foo'.

   386 

   387     self assert: (context invocations select: [:e | e = #consumeWhitespace ]) size = 3.

   388 ! !

   389