1 "{ Package: 'stx:goodies/petitparser/compiler' }" |
1 "{ Package: 'stx:goodies/petitparser/compiler' }" |
2 |
2 |
3 "{ NameSpace: Smalltalk }" |
3 "{ NameSpace: Smalltalk }" |
4 |
4 |
5 PPCConfiguration subclass:#PPCTokenizingConfiguration |
5 PPCConfiguration subclass:#PPCTokenizingConfiguration |
6 instanceVariableNames:'codeGen' |
6 instanceVariableNames:'scannerClazz parserClazz idGen' |
7 classVariableNames:'' |
7 classVariableNames:'' |
8 poolDictionaries:'' |
8 poolDictionaries:'' |
9 category:'PetitCompiler-Core' |
9 category:'PetitCompiler-Core' |
10 ! |
10 ! |
11 |
11 |
12 |
|
13 !PPCTokenizingConfiguration methodsFor:'compiling'! |
12 !PPCTokenizingConfiguration methodsFor:'compiling'! |
14 |
13 |
15 buildClass: compiler |
14 arguments: args |
|
15 super arguments: args. |
|
16 ! |
|
17 |
|
18 buildClass: clazz |
16 | builder | |
19 | builder | |
17 builder := PPCClassBuilder new. |
20 builder := PPCClassBuilder new. |
18 |
21 |
19 builder compiledClassName: arguments parserName. |
22 builder compiledClassName: clazz name. |
20 builder compiledSuperclass: PPTokenizingCompiledParser. |
23 builder compiledSuperclass: clazz superclass. |
21 builder methodDictionary: compiler methodDictionary. |
24 builder methodDictionary: clazz methodDictionary. |
22 builder constants: compiler constants. |
25 builder constants: clazz constants. |
23 |
26 |
24 ^ builder compileClass. |
27 ^ builder compileClass. |
25 ! |
28 ! |
26 |
29 |
27 invokePhases |
30 invokePhases |
36 self specialize. |
39 self specialize. |
37 self inline. |
40 self inline. |
38 self merge. |
41 self merge. |
39 self check. |
42 self check. |
40 self cacheFirstFollow. |
43 self cacheFirstFollow. |
41 self generateScanner. "Please note that codeGen is shared between these two phases" |
44 self buildParserClazz. |
42 self generate. |
45 self unmarkConsumeTokensForInline. |
|
46 self createFSAs. |
|
47 self buildScannerTokens. |
|
48 self buildScannerScans. |
|
49 self generateScanner. |
|
50 self generateParser. |
43 ! ! |
51 ! ! |
44 |
52 |
45 !PPCTokenizingConfiguration methodsFor:'hooks'! |
53 !PPCTokenizingConfiguration methodsFor:'initialization'! |
46 |
54 |
47 codeCompiler |
55 fillInClazzes |
48 codeGen isNil ifTrue: [ codeGen := PPCTokenizingCodeGen on: arguments ]. |
56 parserClazz name: arguments parserName. |
49 ^ codeGen |
57 parserClazz superclass: PPTokenizingCompiledParser. |
|
58 |
|
59 scannerClazz name: arguments scannerName. |
|
60 scannerClazz superclass: arguments scannerSuperclass. |
|
61 |
50 ! |
62 ! |
51 |
63 |
52 codeCompilerOn: args |
64 initialize |
53 ^ PPCTokenizingCompiler on: args |
65 super initialize. |
54 ! |
66 |
55 |
67 parserClazz := PPCClass new. |
56 codeGeneratorVisitorOn: compiler |
68 scannerClazz := PPCClass new. |
57 ^ PPCTokenizingCodeGenerator on: compiler |
69 |
|
70 idGen := PPCIdGenerator new. |
|
71 |
|
72 "The parser and scanner share the same id generator in order |
|
73 to use same names for tokens. |
|
74 " |
|
75 parserClazz idGen: idGen. |
|
76 scannerClazz idGen: idGen. |
58 ! ! |
77 ! ! |
59 |
78 |
60 !PPCTokenizingConfiguration methodsFor:'phases'! |
79 !PPCTokenizingConfiguration methodsFor:'phases'! |
61 |
80 |
62 createLL1Choices |
81 buildParserClazz |
63 ir := PPCLL1Visitor new |
82 | rootMethod | |
|
83 rootMethod := PPCTokenizingCodeGenerator new |
|
84 clazz: parserClazz; |
64 arguments: arguments; |
85 arguments: arguments; |
65 visit: ir. |
86 visit: ir. |
66 self remember: #LL1 |
87 |
|
88 parserClazz propertyAt: #rootMethod put: rootMethod |
67 ! |
89 ! |
68 |
90 |
69 generateScanner |
91 buildScannerScans |
70 | generator scanner | |
92 | fsas generator | |
|
93 |
|
94 "TODO JK: Perhpas write separate visitor for this?" |
|
95 fsas := IdentitySet new. |
|
96 fsas addAll: (ir allNodes select: [ :node | node hasFsa ] thenCollect: [:node | node fsa]). |
|
97 fsas addAll: (ir allNodes select: [ :node | node hasNextFsa ] thenCollect: [:node | node nextFsa]). |
|
98 fsas := fsas reject: [ :fsa | fsa hasDistinctRetvals not ]. |
|
99 |
|
100 generator := PPCScannerCodeGenerator new |
|
101 clazz: scannerClazz; |
|
102 arguments: arguments; |
|
103 yourself. |
|
104 |
|
105 fsas do: [ :fsa | generator generate: fsa ]. |
|
106 ! |
|
107 |
|
108 buildScannerTokens |
|
109 | generator | |
71 generator := PPCTokenCodeGenerator new |
110 generator := PPCTokenCodeGenerator new |
72 compiler: self codeCompiler; |
111 clazz: scannerClazz; |
73 arguments: arguments; |
112 arguments: arguments; |
74 yourself. |
113 yourself. |
75 |
114 |
76 generator visit: ir. |
115 generator visit: ir. |
|
116 ! |
|
117 |
|
118 createFSAs |
|
119 ir := PPCFSAVisitor new |
|
120 idGen: idGen; |
|
121 visit: ir. |
|
122 |
|
123 self remember: (self copyTree: ir) as: #withFSAs |
|
124 ! |
|
125 |
|
126 createLL1Choices |
|
127 self flag: 'This phase needs revisit and update'. |
77 |
128 |
78 scanner := generator compileScanner. |
129 ir := PPCLL1Visitor new |
79 self codeCompiler addConstant: scanner as: #scannerClass. |
130 arguments: arguments; |
|
131 visit: ir. |
|
132 |
|
133 self remember: (self copyTree: ir) as: #LL1 |
|
134 ! |
|
135 |
|
136 generateParser |
|
137 | parserClass rootMethod | |
|
138 arguments generate ifFalse: [ ^ self ]. |
|
139 rootMethod := parserClazz propertyAt: #rootMethod. |
|
140 |
|
141 parserClazz name: arguments parserName. |
|
142 parserClazz superclass: arguments parserSuperclass. |
|
143 |
|
144 parserClass := self buildClass: parserClazz. |
|
145 parserClass startSymbol: rootMethod methodName. |
|
146 |
|
147 self remember: parserClass as: #parser. |
|
148 ir := parserClass new |
|
149 |
|
150 |
|
151 ! |
|
152 |
|
153 generateScanner |
|
154 | scanner | |
|
155 arguments generate ifFalse: [ ^ self ]. |
|
156 |
|
157 scannerClazz name: arguments scannerName. |
|
158 scannerClazz superclass: arguments scannerSuperclass. |
|
159 |
|
160 scanner := (self buildClass: scannerClazz). |
|
161 parserClazz addConstant: scanner as: #scannerClass. |
|
162 |
|
163 ir := scanner. |
|
164 |
|
165 self remember: scanner as: #scanner |
80 ! |
166 ! |
81 |
167 |
82 tokenize |
168 tokenize |
83 " |
169 " |
84 This will try transform the parser into the tokenizing parser |
170 This will try transform the parser into the tokenizing parser |
86 arguments tokenize ifFalse: [ ^ self ] . |
172 arguments tokenize ifFalse: [ ^ self ] . |
87 |
173 |
88 ir := PPCTokenizingVisitor new |
174 ir := PPCTokenizingVisitor new |
89 arguments: arguments; |
175 arguments: arguments; |
90 visit: ir. |
176 visit: ir. |
91 self remember: #tokenize |
177 |
|
178 |
|
179 self remember: (self copyTree: ir) as: #tokenize |
|
180 ! |
|
181 |
|
182 unmarkConsumeTokensForInline |
|
183 "TODO JK: Hack alert, use visitor, or at leas isTokenConsume" |
|
184 ir allNodesDo: [ :node | |
|
185 node class == PPCTokenConsumeNode ifTrue: [ |
|
186 node unmarkForInline |
|
187 ] |
|
188 ] |
92 ! ! |
189 ! ! |
93 |
190 |
94 !PPCTokenizingConfiguration class methodsFor:'documentation'! |
|
95 |
|
96 version_HG |
|
97 |
|
98 ^ '$Changeset: <not expanded> $' |
|
99 ! ! |
|
100 |
|