1 "{ Package: 'stx:goodies/petitparser/compiler/tests' }"

     2 

     3 "{ NameSpace: Smalltalk }"

     4 

     5 TestCase subclass:#PPCDistinctScannerTest

     6 	instanceVariableNames:'configuration scanner'

     7 	classVariableNames:''

     8 	poolDictionaries:''

     9 	category:'PetitCompiler-Tests-Core-Tokenizing'

    10 !

    11 

    12 !PPCDistinctScannerTest methodsFor:'as yet unclassified'!

    13 

    14 aToken

    15     ^ 'a' asParser token 

    16         name: #token;

    17         yourself.

    18 !

    19 

    20 barToken

    21     ^ 'bar' asParser token 

    22         name: #bar;

    23         yourself.

    24 !

    25 

    26 fooToken

    27     ^ 'foo' asParser token 

    28         name: #foo;

    29         yourself.

    30 !

    31 

    32 fooTrimmingToken

    33     ^ 'foo' asParser trimmingToken 

    34         name: #foo;

    35         yourself.

    36 !

    37 

    38 idToken

    39     ^ #letter asParser plus token 

    40         name: #id;

    41         yourself.

    42 !

    43 

    44 idTrimmingToken

    45     ^ #letter asParser plus trimmingToken 

    46         name: #id;

    47         yourself.

    48 !

    49 

    50 overlappingToken

    51     ^ 'a' asParser token / 'a' asParser token

    52 !

    53 

    54 setUp

    55     configuration := PPCConfiguration tokenizing.

    56 !

    57 

    58 testConsumeToken

    59     | parser |

    60     parser := self fooToken, self overlappingToken.

    61     parser compileWithConfiguration: configuration.

    62     scanner := (Smalltalk at: configuration arguments scannerName) new.

    63 

    64     scanner stream: 'foobaz' asPetitStream.

    65     scanner perform: #'consume_foo'.

    66     

    67     self assert: scanner position = 3.

    68     self assert: scanner resultPosition = 3.

    69     self assert: scanner result isNil.

    70 !

    71 

    72 testConsumeToken2

    73     | parser |

    74     parser := self fooToken, self barToken.

    75     parser compileWithConfiguration: configuration.

    76     scanner := (Smalltalk at: configuration arguments scannerName) new.

    77 

    78     scanner stream: 'foobar' asPetitStream.

    79     scanner perform: #'consume_foo'.

    80     

    81     self assert: scanner position = 3.

    82     self assert: scanner resultPosition = 6.

    83     self assert: scanner result = #bar.

    84 !

    85 

    86 testScan

    87     | parser |

    88     parser := self aToken.

    89     parser compileWithConfiguration: configuration.

    90     

    91     scanner := (Smalltalk at: configuration arguments scannerName) new.

    92 

    93     scanner stream: 'a' asPetitStream.

    94     scanner perform: #'scan_token'.

    95     

    96     self assert: scanner position = 0.

    97     self assert: scanner resultPosition = 1.

    98     self assert: scanner result = #token.

    99 !

   100 

   101 testScan2

   102     | parser |

   103     parser := self fooToken.

   104     parser compileWithConfiguration: configuration.

   105     

   106     scanner := (Smalltalk at: configuration arguments scannerName) new.

   107 

   108     scanner stream: 'foo' asPetitStream.

   109     scanner perform: #'scan_foo'.

   110     

   111     self assert: scanner position = 0.

   112     self assert: scanner resultPosition = 3.

   113     self assert: scanner result = #foo.

   114 !

   115 

   116 testScan3

   117     | parser |

   118     parser := self fooToken.

   119     parser compileWithConfiguration: configuration.

   120     

   121     scanner := (Smalltalk at: configuration arguments scannerName) new.

   122 

   123     scanner stream: 'bar' asPetitStream.

   124     scanner perform: #'scan_foo'.

   125     

   126     self assert: scanner position = 0.

   127     self assert: scanner resultPosition = 0.

   128     self assert: scanner result isNil.

   129 !

   130 

   131 testScan4

   132     | parser |

   133     parser := self fooToken, self idToken.

   134     parser compileWithConfiguration: configuration.

   135     

   136     scanner := (Smalltalk at: configuration arguments scannerName) new.

   137 

   138     scanner stream: 'foothere' asPetitStream.

   139     scanner perform: #'scan_foo'.

   140     

   141     self assert: scanner position = 0.

   142     self assert: scanner resultPosition = 3.

   143     self assert: scanner result = #foo.

   144     

   145     scanner perform: #'consume_foo'.

   146 

   147     self assert: scanner position = 3.

   148     self assert: scanner resultPosition = 8.

   149     self assert: scanner result = #id.

   150 !

   151 

   152 testSequence

   153     | parser result |

   154     parser := self fooTrimmingToken, self idTrimmingToken.

   155     parser compileWithConfiguration: configuration.

   156     

   157     scanner := (Smalltalk at: configuration arguments scannerName) new.

   158 

   159     scanner stream: 'foo there ' asPetitStream.

   160     scanner perform: #'foo'.

   161     

   162     self assert: scanner position = 0.

   163     self assert: scanner resultPosition = 3.

   164     self assert: scanner stream position = 4.

   165     self assert: scanner result = #foo.	

   166     

   167     result := scanner perform: #'consume_foo'.

   168 

   169     self assert: scanner position = 4.

   170     self assert: scanner resultPosition = 9.

   171     self assert: scanner stream position = 10.

   172     self assert: scanner result = #id.

   173     

   174     self assert: (result isKindOf: PPToken).

   175     self assert: result start = 1.

   176     self assert: result stop = 3.

   177     self assert: result inputValue = 'foo'.

   178 !

   179 

   180 testToken

   181     | parser |

   182     parser := self fooToken, self idTrimmingToken.

   183     parser compileWithConfiguration: configuration.

   184     

   185     scanner := (Smalltalk at: configuration arguments scannerName) new.

   186 

   187     scanner stream: 'foo there' asPetitStream.

   188     scanner perform: #'foo'.

   189     

   190     self assert: scanner position = 0.

   191     self assert: scanner resultPosition = 3.

   192     self assert: scanner stream position = 3.

   193     self assert: scanner result = #foo.		

   194 !

   195 

   196 testTrimmingScan

   197     | parser |

   198     parser := self fooTrimmingToken, self idTrimmingToken.

   199     parser compileWithConfiguration: configuration.

   200     

   201     scanner := (Smalltalk at: configuration arguments scannerName) new.

   202 

   203     scanner stream: 'foo there' asPetitStream.

   204     scanner perform: #'scan_foo'.

   205     

   206     self assert: scanner position = 0.

   207     self assert: scanner resultPosition = 3.

   208     self assert: scanner result = #foo.	

   209 !

   210 

   211 testTrimmingToken

   212     | parser result |

   213     parser := self fooTrimmingToken, self idTrimmingToken.

   214     parser compileWithConfiguration: configuration.

   215     

   216     scanner := (Smalltalk at: configuration arguments scannerName) new.

   217 

   218     scanner stream: 'foo there' asPetitStream.

   219     result := scanner perform: #'foo'.

   220     

   221     self assert: scanner position = 0.

   222     self assert: scanner resultPosition = 3.

   223     self assert: scanner stream position = 4.

   224     self assert: scanner result = #foo.		

   225         

   226     self assert: result.

   227 ! !

   228