43 self createFSAs. |
42 self createFSAs. |
44 self buildScannerTokens. |
43 self buildScannerTokens. |
45 self buildScannerScans. |
44 self buildScannerScans. |
46 self generateScanner. |
45 self generateScanner. |
47 self generateParser. |
46 self generateParser. |
48 ! |
|
49 |
|
50 options: args |
|
51 super options: args. |
|
52 ! ! |
47 ! ! |
53 |
48 |
54 !PPCTokenizingConfiguration methodsFor:'phases'! |
49 !PPCTokenizingConfiguration methodsFor:'error handling'! |
55 |
50 |
56 buildParserClazz |
51 buildParserClazz |
57 | rootMethod | |
52 | rootMethod | |
58 rootMethod := PPCTokenizingCodeGenerator new |
|
59 clazz: context parserClass; |
|
60 options: options; |
|
61 visit: ir. |
|
62 |
|
63 context parserClass propertyAt: #rootMethod put: rootMethod |
|
64 |
53 |
65 "Modified: / 25-08-2015 / 00:07:38 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
54 rootMethod := (PPCTokenizingCodeGenerator new) |
|
55 clazz:context parserClass; |
|
56 options:context options; |
|
57 visit:ir. |
|
58 context parserClass propertyAt:#rootMethod put:rootMethod |
|
59 |
|
60 "Modified: / 25-08-2015 / 00:07:38 / Jan Vrany <jan.vrany@fit." |
|
61 "Modified: / 26-08-2015 / 19:57:26 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
66 ! |
62 ! |
67 |
63 |
68 buildScannerScans |
64 buildScannerScans |
69 | fsas generator | |
65 | fsas generator | |
70 |
66 |
71 "TODO JK: Perhpas write separate visitor for this?" |
67 "TODO JK: Perhpas write separate visitor for this?" |
72 fsas := IdentitySet new. |
68 fsas := IdentitySet new. |
73 fsas addAll: (ir allNodes select: [ :node | node hasFsa ] thenCollect: [:node | node fsa]). |
69 fsas addAll:(ir allNodes |
74 fsas addAll: (ir allNodes select: [ :node | node hasNextFsa ] thenCollect: [:node | node nextFsa]). |
70 select:[:node | node hasFsa ] |
75 fsas := fsas reject: [ :fsa | fsa hasDistinctRetvals not ]. |
71 thenCollect:[:node | node fsa ]). |
76 |
72 fsas addAll:(ir allNodes |
77 generator := PPCScannerCodeGenerator new |
73 select:[:node | node hasNextFsa ] |
78 clazz: context scannerClass; |
74 thenCollect:[:node | node nextFsa ]). |
79 options: options; |
75 fsas := fsas reject:[:fsa | fsa hasDistinctRetvals not ]. |
80 yourself. |
76 generator := (PPCScannerCodeGenerator new) |
81 |
77 clazz:context scannerClass; |
82 fsas do: [ :fsa | generator generate: fsa ]. |
78 options:context options; |
|
79 yourself. |
|
80 fsas do:[:fsa | |
|
81 generator generate:fsa |
|
82 ]. |
83 |
83 |
84 "Modified: / 25-08-2015 / 00:04:43 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
84 "Modified: / 25-08-2015 / 00:04:43 / Jan Vrany <jan.vrany@fit." |
|
85 "Modified: / 26-08-2015 / 19:57:38 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
85 ! |
86 ! |
86 |
87 |
87 buildScannerTokens |
88 buildScannerTokens |
88 | generator | |
89 | generator | |
89 generator := PPCTokenCodeGenerator new |
|
90 clazz: context scannerClass; |
|
91 options: options; |
|
92 yourself. |
|
93 |
90 |
94 generator visit: ir. |
91 generator := (PPCTokenCodeGenerator new) |
|
92 clazz:context scannerClass; |
|
93 options:context options; |
|
94 yourself. |
|
95 generator visit:ir. |
95 |
96 |
96 "Modified: / 25-08-2015 / 00:04:46 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
97 "Modified: / 25-08-2015 / 00:04:46 / Jan Vrany <jan.vrany@fit." |
|
98 "Modified: / 26-08-2015 / 19:57:46 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
97 ! |
99 ! |
|
100 |
|
101 generateParser |
|
102 | parserClass rootMethod | |
|
103 |
|
104 context options generate ifFalse:[ |
|
105 ^ self |
|
106 ]. |
|
107 rootMethod := context parserClass propertyAt:#rootMethod. |
|
108 context parserClass name:context options parserName. |
|
109 context parserClass superclass:context options parserSuperclass. |
|
110 parserClass := self buildClass:context parserClass. |
|
111 parserClass startSymbol:rootMethod methodName. |
|
112 self remember:parserClass as:#parser. |
|
113 ir := parserClass new |
|
114 |
|
115 "Modified: / 25-08-2015 / 00:05:49 / Jan Vrany" |
|
116 "Modified: / 26-08-2015 / 19:57:58 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
|
117 ! |
|
118 |
|
119 generateScanner |
|
120 | scanner | |
|
121 |
|
122 context options generate ifFalse:[ |
|
123 ^ self |
|
124 ]. |
|
125 context scannerClass name:context options scannerName. |
|
126 context scannerClass superclass:context options scannerSuperclass. |
|
127 scanner := (self buildClass:context scannerClass). |
|
128 context parserClass addConstant:scanner as:#scannerClass. |
|
129 ir := scanner. |
|
130 self remember:scanner as:#scanner |
|
131 |
|
132 "Modified: / 25-08-2015 / 00:06:49 / Jan Vrany" |
|
133 "Modified: / 26-08-2015 / 19:58:12 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
|
134 ! ! |
|
135 |
|
136 !PPCTokenizingConfiguration methodsFor:'phases'! |
98 |
137 |
99 createFSAs |
138 createFSAs |
100 ir := PPCFSAVisitor new |
139 ir := PPCFSAVisitor new |
101 idGen: context scannerClass idGen; |
140 idGen: context scannerClass idGen; |
102 visit: ir. |
141 visit: ir. |
105 |
144 |
106 "Modified: / 25-08-2015 / 00:07:23 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
145 "Modified: / 25-08-2015 / 00:07:23 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
107 ! |
146 ! |
108 |
147 |
109 createLL1Choices |
148 createLL1Choices |
110 self flag: 'This phase needs revisit and update'. |
149 self flag:'This phase needs revisit and update'. |
111 |
150 ir := (PPCLL1Visitor new) |
112 ir := PPCLL1Visitor new |
151 options:context options; |
113 options: options; |
152 visit:ir. |
114 visit: ir. |
153 self remember:(self copyTree:ir) as:#LL1 |
115 |
|
116 self remember: (self copyTree: ir) as: #LL1 |
|
117 ! |
|
118 |
|
119 generateParser |
|
120 | parserClass rootMethod | |
|
121 options generate ifFalse: [ ^ self ]. |
|
122 rootMethod := context parserClass propertyAt: #rootMethod. |
|
123 |
|
124 context parserClass name: options parserName. |
|
125 context parserClass superclass: options parserSuperclass. |
|
126 |
|
127 parserClass := self buildClass: context parserClass. |
|
128 parserClass startSymbol: rootMethod methodName. |
|
129 |
|
130 self remember: parserClass as: #parser. |
|
131 ir := parserClass new |
|
132 |
|
133 "Modified: / 25-08-2015 / 00:05:49 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
|
134 ! |
|
135 |
|
136 generateScanner |
|
137 | scanner | |
|
138 options generate ifFalse: [ ^ self ]. |
|
139 |
|
140 context scannerClass name: options scannerName. |
|
141 context scannerClass superclass: options scannerSuperclass. |
|
142 |
|
143 scanner := (self buildClass: context scannerClass). |
|
144 context parserClass addConstant: scanner as: #scannerClass. |
|
145 |
|
146 ir := scanner. |
|
147 |
|
148 self remember: scanner as: #scanner |
|
149 |
|
150 "Modified: / 25-08-2015 / 00:06:49 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
|
151 ! |
154 ! |
152 |
155 |
153 tokenize |
156 tokenize |
154 " |
157 " |
155 This will try transform the parser into the tokenizing parser |
158 This will try transform the parser into the tokenizing parser" |
156 " |
|
157 options tokenize ifFalse: [ ^ self ] . |
|
158 |
159 |
159 ir := PPCTokenizingVisitor new |
160 context options tokenize ifFalse:[ |
160 options: options; |
161 ^ self |
161 visit: ir. |
162 ]. |
162 |
163 ir := (PPCTokenizingVisitor new) |
163 |
164 options:context options; |
164 self remember: (self copyTree: ir) as: #tokenize |
165 visit:ir. |
|
166 self remember:(self copyTree:ir) as:#tokenize |
165 ! |
167 ! |
166 |
168 |
167 unmarkConsumeTokensForInline |
169 unmarkConsumeTokensForInline |
168 "TODO JK: Hack alert, use visitor, or at leas isTokenConsume" |
170 "TODO JK: Hack alert, use visitor, or at leas isTokenConsume" |
169 ir allNodesDo: [ :node | |
171 ir allNodesDo: [ :node | |