Mercurial Mercurial > hg > stx-goodies-petitparser / comparison
summary | shortlog | changelog | graph | tags | bookmarks | branches | files | changeset | file | latest | revisions | annotate | diff | comparison | raw | help
Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
compiler/tests/PPCTokenizingTest.st
changeset 538 16e8536f5cfb
parent 537 fb212e14d1f4
equal deleted inserted replaced
537:fb212e14d1f4 538:16e8536f5cfb 
    42 parse: whatever
 
    42 parse: whatever
 
    43     ^ result := super parse: whatever.
 
    43     ^ result := super parse: whatever.
 
    44 !
 
    44 !
 
    45 
    45 
    46 setUp
 
    46 setUp
 
    47     options := (PPCCompilationOptions default)
 
    47     options := (PPCCompilationOptions new)
 
    48             profile:true;
 
    48             profile:true;
 
    49             tokenize:true;
 
    49             tokenize:true;
 
    50             yourself.
 
    50             yourself.
 
    51     compiler := PPCCompiler new.
 
    51     compiler := PPCCompiler new.
 
    52     compiler context options:options.
 
    52     compiler context options:options.
 
    53     self cleanClass.
 
    53     self cleanClass.
 
    54 
    54 
    55     "Modified: / 04-09-2015 / 16:21:36 / Jan Vrany <jan.vrany@fit.cvut.cz>"
 
    55     "Modified: / 07-09-2015 / 10:22:29 / Jan Vrany <jan.vrany@fit.cvut.cz>"
 
    56 !
 
    56 !
 
    57 
    57 
    58 tearDown
 
    58 tearDown
 
    59     "self cleanClass"
 
    59     "self cleanClass"
 
    60 !
 
    60 !
 
    65     a2 := 'b' asParser token name: 't2'; yourself.
 
    65     a2 := 'b' asParser token name: 't2'; yourself.
 
    66     
    66     
    67     p1 := a1 star.
 
    67     p1 := a1 star.
 
    68     p2 := a2.
 
    68     p2 := a2.
 
    69     
    69     
    70     parser := p1 / p2 compileUsingCompiler:compiler.
 
    70     parser := compiler compile: (p1 / p2).
 
    71 
    71 
    72     self assert: parser parse: ''.
 
    72     self assert: parser parse: ''.
 
    73     self assert: result isEmpty.
 
    73     self assert: result isEmpty.
 
    74     
    74     
    75     self assert: parser parse: 'a'.
 
    75     self assert: parser parse: 'a'.
 
    85     self assert: parser parse: 'c' end: 0.
 
    85     self assert: parser parse: 'c' end: 0.
 
    86     
    86     
    87 !
 
    87 !
 
    88 
    88 
    89 testChoiceOrder
 
    89 testChoiceOrder
 
    90     parser := (('a' asParser token , 'b' asParser token) / 'a' asParser token) 
    90     parser := compiler compile: (('a' asParser token , 'b' asParser token) / 'a' asParser token).
 
    91                     compileUsingCompiler:compiler.
 
    91 
    92     
    92     
    93     self assert: parser parse: 'ab'.
 
    93     self assert: parser parse: 'ab'.
 
    94     self assert: result first inputValue = 'a'.
 
    94     self assert: result first inputValue = 'a'.
 
    95     self assert: result second inputValue = 'b'.
 
    95     self assert: result second inputValue = 'b'.
 
    96 
    96 
    97     self assert: parser parse: 'a'.
 
    97     self assert: parser parse: 'a'.
 
    98     self assert: result inputValue = 'a'.
 
    98     self assert: result inputValue = 'a'.
 
    99 
    99 
   100     self assert: parser fail: '_'.
 
   100     self assert: parser fail: '_'.
 
   101     
   101 
       
   102     "Modified: / 07-09-2015 / 12:36:21 / Jan Vrany <jan.vrany@fit.cvut.cz>"
 
   102 !
 
   103 !
 
   103 
   104 
   104 testChoiceOrder2
 
   105 testChoiceOrder2
 
   105     | p1 p2 |
 
   106     | p1 p2 |
 
   106     p1 := 'a' asParser token, 'b' asParser token.
 
   107     p1 := 'a' asParser token, 'b' asParser token.
 
   107     p2 := 'b' asParser token / 'a' asParser token.
 
   108     p2 := 'b' asParser token / 'a' asParser token.
 
   108     
   109     
   109     parser := p1 / p2 compileUsingCompiler:compiler.
 
   110     parser := compiler compile: (p1 / p2).
 
   110     
   111     
   111     self assert: parser parse: 'ab'.
 
   112     self assert: parser parse: 'ab'.
 
   112     self assert: result first inputValue = 'a'.
 
   113     self assert: result first inputValue = 'a'.
 
   113     self assert: result second inputValue = 'b'.
 
   114     self assert: result second inputValue = 'b'.
 
   114 
   115 
   128     a2 := 'a' asParser token name: 't2'; yourself.
 
   129     a2 := 'a' asParser token name: 't2'; yourself.
 
   129     
   130     
   130     p1 := a1, 'b' asParser token.
 
   131     p1 := a1, 'b' asParser token.
 
   131     p2 := a2.
 
   132     p2 := a2.
 
   132     
   133     
   133     parser := p1 / p2 compileUsingCompiler:compiler.
 
   134     parser := compiler compile: (p1 / p2).
 
   134     
   135     
   135     self assert: parser parse: 'ab'.
 
   136     self assert: parser parse: 'ab'.
 
   136     self assert: result first inputValue = 'a'.
 
   137     self assert: result first inputValue = 'a'.
 
   137     self assert: result second inputValue = 'b'.
 
   138     self assert: result second inputValue = 'b'.
 
   138 
   139 
   149     a2 := 'a' asParser token name: 't2'; yourself.
 
   150     a2 := 'a' asParser token name: 't2'; yourself.
 
   150     
   151     
   151     p1 := a1, 'b' asParser token.
 
   152     p1 := a1, 'b' asParser token.
 
   152     p2 := 'b' asParser token / a2.
 
   153     p2 := 'b' asParser token / a2.
 
   153     
   154     
   154     parser := p1 / p2 compileUsingCompiler:compiler.
 
   155     parser := compiler compile: (p1 / p2).
 
   155     
   156     
   156     self assert: parser parse: 'ab'.
 
   157     self assert: parser parse: 'ab'.
 
   157     self assert: result first inputValue = 'a'.
 
   158     self assert: result first inputValue = 'a'.
 
   158     self assert: result second inputValue = 'b'.
 
   159     self assert: result second inputValue = 'b'.
 
   159 
   160 
   166     self assert: parser fail: 'c'.
 
   167     self assert: parser fail: 'c'.
 
   167     
   168     
   168 !
 
   169 !
 
   169 
   170 
   170 testCompileAnd
 
   171 testCompileAnd
 
   171     parser := (('foo' asParser token and) / ('bar' asParser token and)) 
   172     parser := compiler compile:(('foo' asParser token and) / ('bar' asParser token and)) 
   172                     , 'bar' asParser token compileUsingCompiler:compiler.
 
   173                     , 'bar' asParser token.
 
   173     
   174     
   174     self assert: parser parse: 'bar'.
 
   175     self assert: parser parse: 'bar'.
 
   175     self assert: result second inputValue = 'bar'.
 
   176     self assert: result second inputValue = 'bar'.
 
       
   177 
       
   178     "Modified: / 07-09-2015 / 12:36:30 / Jan Vrany <jan.vrany@fit.cvut.cz>"
 
   176 !
 
   179 !
 
   177 
   180 
   178 testCompileChoice
 
   181 testCompileChoice
 
   179     parser := ('foo' asParser / 'bar' asParser) token 
   182     parser := compiler compile: ('foo' asParser / 'bar' asParser) token.
 
   180                     compileUsingCompiler:compiler.
 
   183 
   181     
       
   182     self assert: parser parse: 'foo'.
 
   184     self assert: parser parse: 'foo'.
 
   183     self assert: result inputValue = 'foo'.
 
   185     self assert: result inputValue = 'foo'.
 
   184     self assert: parser parse: 'bar'.
 
   186     self assert: parser parse: 'bar'.
 
   185     self assert: result inputValue = 'bar'.
 
   187     self assert: result inputValue = 'bar'.
 
   186     self assert: parser fail: '_'.
 
   188     self assert: parser fail: '_'.
 
   187     
   189 
       
   190     "Modified: / 07-09-2015 / 12:35:42 / Jan Vrany <jan.vrany@fit.cvut.cz>"
 
   188 !
 
   191 !
 
   189 
   192 
   190 testCompileChoice2
 
   193 testCompileChoice2
 
   191     parser := ('foo' asParser token trim / 'bar' asParser token trim) 
   194     parser := compiler compile:('foo' asParser token trim / 'bar' asParser token trim).
 
   192                     compileUsingCompiler:compiler.
 
       
   193     
   195     
   194     self assert: parser parse: 'foo'.
 
   196     self assert: parser parse: 'foo'.
 
   195     self assert: result inputValue = 'foo'.
 
   197     self assert: result inputValue = 'foo'.
 
   196     self assert: parser parse: 'bar'.
 
   198     self assert: parser parse: 'bar'.
 
   197     self assert: result inputValue = 'bar'.
 
   199     self assert: result inputValue = 'bar'.
 
   198     self assert: parser fail: '_'.
 
   200     self assert: parser fail: '_'.
 
   199     
   201 
       
   202     "Modified: / 07-09-2015 / 12:36:37 / Jan Vrany <jan.vrany@fit.cvut.cz>"
 
   200 !
 
   203 !
 
   201 
   204 
   202 testCompileComplex1
 
   205 testCompileComplex1
 
   203     parser := ('foo' asParser token , 'bar' asParser token) 
   206     parser := compiler compile: ('foo' asParser token , 'bar' asParser token) 
   204                     / ('foo' asParser token , 'baz' asParser token) 
   207                     / ('foo' asParser token , 'baz' asParser token).
 
   205                         compileUsingCompiler:compiler.
 
       
   206     
   208     
   207     self assert: parser parse: 'foobar'.
 
   209     self assert: parser parse: 'foobar'.
 
   208     self assert: result second inputValue = 'bar'.
 
   210     self assert: result second inputValue = 'bar'.
 
   209 
   211 
   210     self assert: parser parse: 'foobaz'.
 
   212     self assert: parser parse: 'foobaz'.
 
   211     self assert: result second inputValue = 'baz'.
 
   213     self assert: result second inputValue = 'baz'.
 
   212 
   214 
   213     self assert: parser fail: 'foobaq'.
 
   215     self assert: parser fail: 'foobaq'.
 
   214     
   216 
       
   217     "Modified: / 07-09-2015 / 12:36:54 / Jan Vrany <jan.vrany@fit.cvut.cz>"
 
   215 !
 
   218 !
 
   216 
   219 
   217 testCompileComplex2
 
   220 testCompileComplex2
 
   218     parser := ('foo' asParser token , 'bar' asParser token) star , 'foo' asParser token 
   221     parser := compiler compile:('foo' asParser token , 'bar' asParser token) star , 'foo' asParser token.
 
   219                     compileUsingCompiler:compiler.
 
       
   220     
   222     
   221     self assert: parser parse: 'foobarfoobarfoo'.
 
   223     self assert: parser parse: 'foobarfoobarfoo'.
 
   222     self assert: parser parse: 'foo'.
 
   224     self assert: parser parse: 'foo'.
 
   223 
   225 
   224     self assert: parser fail: 'bar'.
 
   226     self assert: parser fail: 'bar'.
 
   225     
   227 
       
   228     "Modified: / 07-09-2015 / 12:37:01 / Jan Vrany <jan.vrany@fit.cvut.cz>"
 
   226 !
 
   229 !
 
   227 
   230 
   228 testCompileComplex3
 
   231 testCompileComplex3
 
   229     parser :=	(('foo' asParser token , 'bar' asParser token) star , 'foo' asParser token) 
   232     parser := compiler compile: (('foo' asParser token , 'bar' asParser token) star , 'foo' asParser token) 
   230                     / ('foo' asParser token , 'baz' asParser token) 
   233                     / ('foo' asParser token , 'baz' asParser token).
 
   231                         compileUsingCompiler:compiler.
 
   234 
   232     
   235     
   233     self assert: parser parse: 'foobarfoobarfoo'.
 
   236     self assert: parser parse: 'foobarfoobarfoo'.
 
   234     self assert: parser parse: 'foo'.
 
   237     self assert: parser parse: 'foo'.
 
   235 
   238 
   236     self assert: parser fail: 'bar'.
 
   239     self assert: parser fail: 'bar'.
 
   237     
   240 
       
   241     "Modified: / 07-09-2015 / 12:37:10 / Jan Vrany <jan.vrany@fit.cvut.cz>"
 
   238 !
 
   242 !
 
   239 
   243 
   240 testCompileEmptytoken
 
   244 testCompileEmptytoken
 
   241     | start stop epsilon |
 
   245     | start stop epsilon |
 
   242     start := $( asParser token.
 
   246     start := $( asParser token.
 
   243     stop := $) asParser token.
 
   247     stop := $) asParser token.
 
   244     epsilon := '' asParser token.
 
   248     epsilon := '' asParser token.
 
   245     
   249     
   246     self should: [
 
   250     self should: [
 
   247         (start , epsilon , stop) compileUsingCompiler:compiler.
 
   251         compiler compile: (start , epsilon , stop)
 
   248     ] raise: Exception.
 
   252     ] raise: Exception.
 
   249 "	
   253 "       
   250     self assert: parser parse: '()'.
 
   254     self assert: parser parse: '()'.
 
   251     self assert: parser fail: '('.
 
   255     self assert: parser fail: '('.
 
   252 "
 
   256 "
 
       
   257 
       
   258     "Modified: / 07-09-2015 / 12:40:19 / Jan Vrany <jan.vrany@fit.cvut.cz>"
 
   253 !
 
   259 !
 
   254 
   260 
   255 testCompileLiteral
 
   261 testCompileLiteral
 
   256     parser := 'foo' asParser token compileUsingCompiler:compiler.
 
   262     parser := compiler compile: ('foo' asParser token).
 
   257     
   263     
   258     self assert: parser parse: 'foo'.
 
   264     self assert: parser parse: 'foo'.
 
   259     self assert: result inputValue = 'foo'.
 
   265     self assert: result inputValue = 'foo'.
 
   260     self assert: parser fail: 'boo'.
 
   266     self assert: parser fail: 'boo'.
 
   261 !
 
   267 !
 
   262 
   268 
   263 testCompileSequence
 
   269 testCompileSequence
 
   264     parser := ('foo' asParser token) , ('bar' asParser token) 
   270     parser := compiler compile: ('foo' asParser token) , ('bar' asParser token).
 
   265                     compileUsingCompiler:compiler.
 
   271 
   266     
   272     
   267     self assert: parser parse: 'foobar'.
 
   273     self assert: parser parse: 'foobar'.
 
   268     self assert: result first inputValue = 'foo'.
 
   274     self assert: result first inputValue = 'foo'.
 
   269     self assert: result second inputValue = 'bar'.
 
   275     self assert: result second inputValue = 'bar'.
 
       
   276 
       
   277     "Modified: / 07-09-2015 / 12:40:11 / Jan Vrany <jan.vrany@fit.cvut.cz>"
 
   270 !
 
   278 !
 
   271 
   279 
   272 testCompileSequence2
 
   280 testCompileSequence2
 
   273     parser := ('foo' asParser trimmingToken) , ('bar' asParser trimmingToken) 
   281     parser := compiler compile: ('foo' asParser trimmingToken) , ('bar' asParser trimmingToken).
 
   274                     compileUsingCompiler:compiler.
 
   282 
   275     
   283     
   276     self assert: parser parse: 'foobar'.
 
   284     self assert: parser parse: 'foobar'.
 
   277     self assert: result first inputValue = 'foo'.
 
   285     self assert: result first inputValue = 'foo'.
 
   278     self assert: result second inputValue = 'bar'.
 
   286     self assert: result second inputValue = 'bar'.
 
   279 
   287 
   282     self assert: result second inputValue = 'bar'.
 
   290     self assert: result second inputValue = 'bar'.
 
   283 
   291 
   284     self assert: parser parse: '  foo  bar'.
 
   292     self assert: parser parse: '  foo  bar'.
 
   285     self assert: result first inputValue = 'foo'.
 
   293     self assert: result first inputValue = 'foo'.
 
   286     self assert: result second inputValue = 'bar'.
 
   294     self assert: result second inputValue = 'bar'.
 
       
   295 
       
   296     "Modified: / 07-09-2015 / 12:40:05 / Jan Vrany <jan.vrany@fit.cvut.cz>"
 
   287 !
 
   297 !
 
   288 
   298 
   289 testCompileSequence3
 
   299 testCompileSequence3
 
   290     parser := 	('foo' asParser trimmingToken) , ('bar' asParser trimmingToken) 
   300     parser :=   compiler compile: ('foo' asParser trimmingToken) , ('bar' asParser trimmingToken) 
   291                      , ('baz' asParser trimmingToken) compileUsingCompiler:compiler.
 
   301                      , ('baz' asParser trimmingToken).
 
   292     
   302     
   293     self assert: parser parse: 'foobarbaz'.
 
   303     self assert: parser parse: 'foobarbaz'.
 
   294     self assert: result first inputValue = 'foo'.
 
   304     self assert: result first inputValue = 'foo'.
 
   295     self assert: result second inputValue = 'bar'.
 
   305     self assert: result second inputValue = 'bar'.
 
   296 
   306 
   297     self assert: parser parse: ' foo  bar  baz  '.
 
   307     self assert: parser parse: ' foo  bar  baz  '.
 
   298     self assert: result first inputValue = 'foo'.
 
   308     self assert: result first inputValue = 'foo'.
 
   299     self assert: result second inputValue = 'bar'.
 
   309     self assert: result second inputValue = 'bar'.
 
   300     self assert: result third inputValue = 'baz'.
 
   310     self assert: result third inputValue = 'baz'.
 
       
   311 
       
   312     "Modified: / 07-09-2015 / 12:39:59 / Jan Vrany <jan.vrany@fit.cvut.cz>"
 
   301 !
 
   313 !
 
   302 
   314 
   303 testCompileStar
 
   315 testCompileStar
 
   304     parser := 'foo' asParser token star compileUsingCompiler:compiler.
 
   316     parser := compiler compile: ('foo' asParser token star).
 
   305     
   317     
   306     self assert: parser parse: 'foo'.
 
   318     self assert: parser parse: 'foo'.
 
   307     self assert: result first inputValue = 'foo'.
 
   319     self assert: result first inputValue = 'foo'.
 
   308     
   320     
   309     self assert: parser parse: 'boo' end: 0.
 
   321     self assert: parser parse: 'boo' end: 0.
 
   310     self assert: result isEmpty.
 
   322     self assert: result isEmpty.
 
   311 !
 
   323 !
 
   312 
   324 
   313 testCompileStar2
 
   325 testCompileStar2
 
   314     parser := ('foo' asParser token , 'bar' asParser token) star 
   326     parser := compiler compile: ('foo' asParser token , 'bar' asParser token) star.
 
   315                     compileUsingCompiler:compiler.
 
   327 
   316     
   328     
   317     self assert: parser parse: 'foobar'.
 
   329     self assert: parser parse: 'foobar'.
 
   318     self assert: context tokenReads size = 1.
 
   330     self assert: context tokenReads size = 1.
 
   319             
   331             
   320     self assert: parser parse: 'bar' end: 0.
 
   332     self assert: parser parse: 'bar' end: 0.
 
   321     self assert: result isEmpty.
 
   333     self assert: result isEmpty.
 
   322     self assert: context tokenReads size = 1.
 
   334     self assert: context tokenReads size = 1.
 
   323         
   335 
       
   336     "Modified: / 07-09-2015 / 12:39:51 / Jan Vrany <jan.vrany@fit.cvut.cz>"
 
   324 !
 
   337 !
 
   325 
   338 
   326 testCompileStar3
 
   339 testCompileStar3
 
   327     parser := 'a' asParser trimmingToken star , 'b' asParser trimmingToken 
   340     parser := compiler compile: ('a' asParser trimmingToken star , 'b' asParser trimmingToken).
 
   328                     compileUsingCompiler:compiler.
 
   341 
   329     
   342     
   330     self assert: parser parse: 'ab'.
 
   343     self assert: parser parse: 'ab'.
 
   331     self assert: parser parse: 'aaab'.
 
   344     self assert: parser parse: 'aaab'.
 
   332     self assert: result size = 2.
 
   345     self assert: result size = 2.
 
   333     self assert: result first size = 3.
 
   346     self assert: result first size = 3.
 
   334             
   347             
   335     self assert: parser fail: 'ac'.
 
   348     self assert: parser fail: 'ac'.
 
       
   349 
       
   350     "Modified: / 07-09-2015 / 12:39:45 / Jan Vrany <jan.vrany@fit.cvut.cz>"
 
   336 !
 
   351 !
 
   337 
   352 
   338 testCompileTokenComplex2
 
   353 testCompileTokenComplex2
 
   339     |  a b optionsWith  |
 
   354     |  a b optionsWith  |
 
   340     "based on the PPSmalltlakGrammar>>blockArgumentsWith"
 
   355     "based on the PPSmalltlakGrammar>>blockArgumentsWith"
 
   344         yourself.		
   359         yourself.		
   345     optionsWith := (a / b and ==> [:t | ]) wrapped
 
   360     optionsWith := (a / b and ==> [:t | ]) wrapped
 
   346         name: 'optionsWith'; 
   361         name: 'optionsWith'; 
   347         yourself.
 
   362         yourself.
 
   348 
   363 
   349     parser := optionsWith compileUsingCompiler:compiler.
 
   364     parser := compiler compile: (optionsWith).
 
   350     self assert: parser parse: '|'.
 
   365     self assert: parser parse: '|'.
 
   351 
   366 
   352     self assert: parser parse: ']' end: 0.
 
   367     self assert: parser parse: ']' end: 0.
 
   353 !
 
   368 !
 
   354 
   369 
   370         name: 'choice1'; 
   385         name: 'choice1'; 
   371         yourself.
 
   386         yourself.
 
   372     
   387     
   373     tricky := (a1 asParser, choice1) / (b2 asParser, choice2).
 
   388     tricky := (a1 asParser, choice1) / (b2 asParser, choice2).
 
   374 
   389 
   375     parser := tricky compileUsingCompiler:compiler.
 
   390     parser := compiler compile: (tricky).
 
   376     self assert: parser parse: '||'.
 
   391     self assert: parser parse: '||'.
 
   377 
   392 
   378     self assert: parser parse: '|]'.
 
   393     self assert: parser parse: '|]'.
 
   379 
   394 
   380     self assert: parser parse: ']|'.
 
   395     self assert: parser parse: ']|'.
 
   407     arrayLiteral setParser: '#(' asParser token, symbolLiteralArray, ')' asParser token.
 
   422     arrayLiteral setParser: '#(' asParser token, symbolLiteralArray, ')' asParser token.
 
   408     arrayLiteral name: 'arrayLiteral'.
 
   423     arrayLiteral name: 'arrayLiteral'.
 
   409 
   424 
   410     arrayItem := arrayLiteral / symbolLiteral.
 
   425     arrayItem := arrayLiteral / symbolLiteral.
 
   411 
   426 
   412     parser := arrayItem compileUsingCompiler:compiler.
 
   427     parser := compiler compile: (arrayItem).
 
   413 
   428 
   414     self assert: parser parse: '#(foo)'.
 
   429     self assert: parser parse: '#(foo)'.
 
   415     self assert: parser parse: '#foo'.
 
   430     self assert: parser parse: '#foo'.
 
   416 
   431 
   417     "Modified (comment): / 17-08-2015 / 23:07:35 / Jan Vrany <jan.vrany@fit.cvut.cz>"
 
   432     "Modified (comment): / 17-08-2015 / 23:07:35 / Jan Vrany <jan.vrany@fit.cvut.cz>"
 
   418 !
 
   433 !
 
   419 
   434 
   420 testCompileTrim
 
   435 testCompileTrim
 
   421     parser := 'foo' asParser token trim end compileUsingCompiler:compiler.
 
   436     parser := compiler compile: ('foo' asParser token trim end).
 
   422     
   437     
   423     self assert: parser parse: 'foo'.
 
   438     self assert: parser parse: 'foo'.
 
   424     self assert: result inputValue = 'foo'.
 
   439     self assert: result inputValue = 'foo'.
 
   425 
   440 
   426     self assert: parser parse: 'foo  '.
 
   441     self assert: parser parse: 'foo  '.
 
   437 !
 
   452 !
 
   438 
   453 
   439 testTokenCharacter
 
   454 testTokenCharacter
 
   440     | token |
 
   455     | token |
 
   441     token := $a asParser token.
 
   456     token := $a asParser token.
 
   442     parser := token plus compileUsingCompiler:compiler.
 
   457     parser := compiler compile: (token plus).
 
   443 
   458 
   444     self assert: parser parse: 'a'.
 
   459     self assert: parser parse: 'a'.
 
   445     self assert: result first inputValue = 'a'.
 
   460     self assert: result first inputValue = 'a'.
 
   446     self assert: context tokenReads size = 1.
 
   461     self assert: context tokenReads size = 1.
 
   447 
   462 
   450 !
 
   465 !
 
   451 
   466 
   452 testTokenCharacter2
 
   467 testTokenCharacter2
 
   453     | token |
 
   468     | token |
 
   454     token := $a asParser token.
 
   469     token := $a asParser token.
 
   455     parser := token plus compileUsingCompiler:compiler.
 
   470     parser := compiler compile: (token plus).
 
   456 
   471 
   457     self assert: parser parse: 'aaa'.
 
   472     self assert: parser parse: 'aaa'.
 
   458     self assert: result first inputValue = 'a'.
 
   473     self assert: result first inputValue = 'a'.
 
   459     self assert: result second inputValue = 'a'.
 
   474     self assert: result second inputValue = 'a'.
 
   460     self assert: result third inputValue = 'a'.
 
   475     self assert: result third inputValue = 'a'.
 
   465 !
 
   480 !
 
   466 
   481 
   467 testTokenName
 
   482 testTokenName
 
   468     | token |
 
   483     | token |
 
   469     token := 'foo' asParser token name: 'fooToken'; yourself.
 
   484     token := 'foo' asParser token name: 'fooToken'; yourself.
 
   470     parser := token plus compileUsingCompiler:compiler.
 
   485     parser := compiler compile: (token plus).
 
   471 
   486 
   472     self assert: parser parse: 'foofoo'.
 
   487     self assert: parser parse: 'foofoo'.
 
   473     self assert: result first inputValue = 'foo'.
 
   488     self assert: result first inputValue = 'foo'.
 
   474     self assert: result second inputValue = 'foo'.
 
   489     self assert: result second inputValue = 'foo'.
 
   475     self assert: (parser scanner class methodDictionary includesKey: #fooToken).
 
   490     self assert: (parser scanner class methodDictionary includesKey: #fooToken).
 
   484     ws := #blank asParser star name: 'consumeWhitespace'; yourself.
 
   499     ws := #blank asParser star name: 'consumeWhitespace'; yourself.
 
   485     trimmingToken := ((ws, token, ws) ==> #second) 
   500     trimmingToken := ((ws, token, ws) ==> #second) 
   486         propertyAt: 'trimmingToken' put: true; 
   501         propertyAt: 'trimmingToken' put: true; 
   487         yourself.
 
   502         yourself.
 
   488     
   503     
   489     parser := trimmingToken plus compileUsingCompiler:compiler.
 
   504     parser := compiler compile: (trimmingToken plus).
 
   490 
   505 
   491     self assert: parser parse: ' foo '.
 
   506     self assert: parser parse: ' foo '.
 
   492     self assert: result first inputValue = 'foo'.
 
   507     self assert: result first inputValue = 'foo'.
 
   493 
   508 
   494     self assert: (context invocations select: [:e | e = #scan_consumeWhitespace ]) size = 3.
 
   509     self assert: (context invocations select: [:e | e = #scan_consumeWhitespace ]) size = 3.
 
   504     ws := #blank asParser star name: 'consumeWhitespace'; yourself.
 
   519     ws := #blank asParser star name: 'consumeWhitespace'; yourself.
 
   505     trimmingToken := ((ws, token, ws) ==> #second) 
   520     trimmingToken := ((ws, token, ws) ==> #second) 
   506         propertyAt: 'trimmingToken' put: true; 
   521         propertyAt: 'trimmingToken' put: true; 
   507         yourself.
 
   522         yourself.
 
   508     
   523     
   509     parser := trimmingToken plus compileUsingCompiler:compiler.
 
   524     parser := compiler compile: (trimmingToken plus).
 
   510 
   525 
   511     self assert: parser parse: ' foo foo '.
 
   526     self assert: parser parse: ' foo foo '.
 
   512     self assert: result first inputValue = 'foo'.
 
   527     self assert: result first inputValue = 'foo'.
 
   513     self assert: result second inputValue = 'foo'.
 
   528     self assert: result second inputValue = 'foo'.
 
   514 
   529 
   525     ws := #blank asParser star name: 'consumeWhitespace'; yourself.
 
   540     ws := #blank asParser star name: 'consumeWhitespace'; yourself.
 
   526     trimmingToken := ((ws, token, ws) ==> #second) 
   541     trimmingToken := ((ws, token, ws) ==> #second) 
   527         propertyAt: 'trimmingToken' put: true; 
   542         propertyAt: 'trimmingToken' put: true; 
   528         yourself.
 
   543         yourself.
 
   529     
   544     
   530     parser := trimmingToken plus compileUsingCompiler:compiler.
 
   545     parser := compiler compile: (trimmingToken plus).
 
   531 
   546 
   532     self assert: parser parse: ' foo  foo  foo  '.
 
   547     self assert: parser parse: ' foo  foo  foo  '.
 
   533     self assert: result first inputValue = 'foo'.
 
   548     self assert: result first inputValue = 'foo'.
 
   534     self assert: result second inputValue = 'foo'.
 
   549     self assert: result second inputValue = 'foo'.
 
   535     self assert: result third inputValue = 'foo'.
 
   550     self assert: result third inputValue = 'foo'.
hg/stx-goodies-petitparser