equal
deleted
inserted
replaced
51 overlappingToken |
51 overlappingToken |
52 ^ 'a' asParser token / 'a' asParser token |
52 ^ 'a' asParser token / 'a' asParser token |
53 ! |
53 ! |
54 |
54 |
55 setUp |
55 setUp |
56 compiler := PPCCompiler tokenizing. |
56 compiler := PPCCompiler newWithOptions: #( #tokenize: true ). |
|
57 |
|
58 "Modified: / 07-09-2015 / 11:36:36 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
57 ! |
59 ! |
58 |
60 |
59 testConsumeToken |
61 testConsumeToken |
60 | parser | |
62 | parser | |
61 parser := self fooToken, self overlappingToken. |
63 parser := self fooToken, self overlappingToken. |
62 parser compileUsingCompiler:compiler. |
64 compiler compile: parser. |
63 scanner := (Smalltalk at: compiler context options scannerName) new. |
65 scanner := (Smalltalk at: compiler context options scannerName) new. |
64 |
66 |
65 scanner stream: 'foobaz' asPetitStream. |
67 scanner stream: 'foobaz' asPetitStream. |
66 scanner perform: #'consume_foo'. |
68 scanner perform: #'consume_foo'. |
67 |
69 |
73 ! |
75 ! |
74 |
76 |
75 testConsumeToken2 |
77 testConsumeToken2 |
76 | parser | |
78 | parser | |
77 parser := self fooToken, self barToken. |
79 parser := self fooToken, self barToken. |
78 parser compileUsingCompiler:compiler. |
80 compiler compile: parser. |
79 scanner := (Smalltalk at: compiler context options scannerName) new. |
81 scanner := (Smalltalk at: compiler context options scannerName) new. |
80 |
82 |
81 scanner stream: 'foobar' asPetitStream. |
83 scanner stream: 'foobar' asPetitStream. |
82 scanner perform: #'consume_foo'. |
84 scanner perform: #'consume_foo'. |
83 |
85 |
89 ! |
91 ! |
90 |
92 |
91 testScan |
93 testScan |
92 | parser | |
94 | parser | |
93 parser := self aToken. |
95 parser := self aToken. |
94 parser compileUsingCompiler:compiler. |
96 compiler compile: parser. |
95 |
97 |
96 scanner := (Smalltalk at: compiler context options scannerName) new. |
98 scanner := (Smalltalk at: compiler context options scannerName) new. |
97 |
99 |
98 scanner stream: 'a' asPetitStream. |
100 scanner stream: 'a' asPetitStream. |
99 scanner perform: #'scan_token'. |
101 scanner perform: #'scan_token'. |
106 ! |
108 ! |
107 |
109 |
108 testScan2 |
110 testScan2 |
109 | parser | |
111 | parser | |
110 parser := self fooToken. |
112 parser := self fooToken. |
111 parser compileUsingCompiler:compiler. |
113 compiler compile: parser. |
112 |
114 |
113 scanner := (Smalltalk at: compiler context options scannerName) new. |
115 scanner := (Smalltalk at: compiler context options scannerName) new. |
114 |
116 |
115 scanner stream: 'foo' asPetitStream. |
117 scanner stream: 'foo' asPetitStream. |
116 scanner perform: #'scan_foo'. |
118 scanner perform: #'scan_foo'. |
123 ! |
125 ! |
124 |
126 |
125 testScan3 |
127 testScan3 |
126 | parser | |
128 | parser | |
127 parser := self fooToken. |
129 parser := self fooToken. |
128 parser compileUsingCompiler:compiler. |
130 compiler compile: parser. |
129 |
131 |
130 scanner := (Smalltalk at: compiler context options scannerName) new. |
132 scanner := (Smalltalk at: compiler context options scannerName) new. |
131 |
133 |
132 scanner stream: 'bar' asPetitStream. |
134 scanner stream: 'bar' asPetitStream. |
133 scanner perform: #'scan_foo'. |
135 scanner perform: #'scan_foo'. |
140 ! |
142 ! |
141 |
143 |
142 testScan4 |
144 testScan4 |
143 | parser | |
145 | parser | |
144 parser := self fooToken, self idToken. |
146 parser := self fooToken, self idToken. |
145 parser compileUsingCompiler:compiler. |
147 compiler compile: parser. |
146 |
148 |
147 scanner := (Smalltalk at: compiler context options scannerName) new. |
149 scanner := (Smalltalk at: compiler context options scannerName) new. |
148 |
150 |
149 scanner stream: 'foothere' asPetitStream. |
151 scanner stream: 'foothere' asPetitStream. |
150 scanner perform: #'scan_foo'. |
152 scanner perform: #'scan_foo'. |
163 ! |
165 ! |
164 |
166 |
165 testSequence |
167 testSequence |
166 | parser result | |
168 | parser result | |
167 parser := self fooTrimmingToken, self idTrimmingToken. |
169 parser := self fooTrimmingToken, self idTrimmingToken. |
168 parser compileUsingCompiler:compiler. |
170 compiler compile: parser. |
169 |
171 |
170 scanner := (Smalltalk at: compiler context options scannerName) new. |
172 scanner := (Smalltalk at: compiler context options scannerName) new. |
171 |
173 |
172 scanner stream: 'foo there ' asPetitStream. |
174 scanner stream: 'foo there ' asPetitStream. |
173 scanner perform: #'foo'. |
175 scanner perform: #'foo'. |
193 ! |
195 ! |
194 |
196 |
195 testToken |
197 testToken |
196 | parser | |
198 | parser | |
197 parser := self fooToken, self idTrimmingToken. |
199 parser := self fooToken, self idTrimmingToken. |
198 parser compileUsingCompiler:compiler. |
200 compiler compile: parser. |
199 |
201 |
200 scanner := (Smalltalk at: compiler context options scannerName) new. |
202 scanner := (Smalltalk at: compiler context options scannerName) new. |
201 |
203 |
202 scanner stream: 'foo there' asPetitStream. |
204 scanner stream: 'foo there' asPetitStream. |
203 scanner perform: #'foo'. |
205 scanner perform: #'foo'. |
211 ! |
213 ! |
212 |
214 |
213 testTrimmingScan |
215 testTrimmingScan |
214 | parser | |
216 | parser | |
215 parser := self fooTrimmingToken, self idTrimmingToken. |
217 parser := self fooTrimmingToken, self idTrimmingToken. |
216 parser compileUsingCompiler:compiler. |
218 compiler compile: parser. |
217 |
219 |
218 scanner := (Smalltalk at: compiler context options scannerName) new. |
220 scanner := (Smalltalk at: compiler context options scannerName) new. |
219 |
221 |
220 scanner stream: 'foo there' asPetitStream. |
222 scanner stream: 'foo there' asPetitStream. |
221 scanner perform: #'scan_foo'. |
223 scanner perform: #'scan_foo'. |
228 ! |
230 ! |
229 |
231 |
230 testTrimmingToken |
232 testTrimmingToken |
231 | parser result | |
233 | parser result | |
232 parser := self fooTrimmingToken, self idTrimmingToken. |
234 parser := self fooTrimmingToken, self idTrimmingToken. |
233 parser compileUsingCompiler:compiler. |
235 compiler compile: parser. |
234 |
236 |
235 scanner := (Smalltalk at: compiler context options scannerName) new. |
237 scanner := (Smalltalk at: compiler context options scannerName) new. |
236 |
238 |
237 scanner stream: 'foo there' asPetitStream. |
239 scanner stream: 'foo there' asPetitStream. |
238 result := scanner perform: #'foo'. |
240 result := scanner perform: #'foo'. |