24 ifFalse: [ tokenGenerator ] |
24 ifFalse: [ tokenGenerator ] |
25 ! |
25 ! |
26 |
26 |
27 tokenGenerator: whatever |
27 tokenGenerator: whatever |
28 tokenGenerator := whatever |
28 tokenGenerator := whatever |
|
29 ! |
|
30 |
|
31 tokenGuards |
|
32 "When tokenizing, do not use guards" |
|
33 ^ arguments guards |
29 ! ! |
34 ! ! |
30 |
35 |
|
36 !PPCTokenizingCodeGenerator methodsFor:'guards'! |
|
37 |
|
38 addGuard: node ifTrue: trueBlock ifFalse: falseBlock |
|
39 | guard id | |
|
40 guard := PPCTokenGuard on: node. |
|
41 (self guards not or: guard makesSense not) ifTrue: [ ^ false ]. |
|
42 |
|
43 id := compiler idFor: guard firstToken. |
|
44 |
|
45 compiler add: 'self ', id asString. |
|
46 |
|
47 trueBlock isNil ifFalse: [ |
|
48 compiler addOnLine: ' ifTrue: ['. |
|
49 compiler indent. |
|
50 trueBlock value. |
|
51 compiler dedent. |
|
52 falseBlock isNil ifTrue: [ compiler addOnLine: '].' ] |
|
53 ifFalse: [ compiler add: ']'. ] |
|
54 ]. |
|
55 falseBlock isNil ifFalse: [ |
|
56 compiler addOnLine: ' ifFalse: ['. |
|
57 compiler indent. |
|
58 falseBlock value. |
|
59 compiler dedent. |
|
60 compiler addOnLine: '].'. |
|
61 ]. |
|
62 ^ true |
|
63 ! ! |
|
64 |
31 !PPCTokenizingCodeGenerator methodsFor:'visiting'! |
65 !PPCTokenizingCodeGenerator methodsFor:'visiting'! |
|
66 |
|
67 visitAndNode: node |
|
68 | mementoVar currentTokenVar | |
|
69 |
|
70 mementoVar := compiler allocateTemporaryVariableNamed: 'memento'. |
|
71 currentTokenVar := compiler allocateTemporaryVariableNamed: 'currentToken'. |
|
72 |
|
73 compiler smartRemember: node child to: mementoVar. |
|
74 compiler codeAssign: '{ currentTokenValue . currentTokenType }.' to: currentTokenVar. |
|
75 |
|
76 compiler codeStoreValueOf: [ self visit: node child ] intoVariable: self retvalVar. |
|
77 |
|
78 compiler smartRestore: node child from: mementoVar. |
|
79 compiler codeAssign: '(', currentTokenVar, ' at: 1).' to: 'currentTokenValue'. |
|
80 compiler codeAssign: '(', currentTokenVar, ' at: 2).' to: 'currentTokenType'. |
|
81 |
|
82 compiler codeReturn. |
|
83 ! |
32 |
84 |
33 visitChoiceNode: node |
85 visitChoiceNode: node |
34 " true ifTrue: [ ^ super visitChoiceNode: node ]." |
86 " true ifTrue: [ ^ super visitChoiceNode: node ]." |
35 "HACK alert: We are inside token..." |
87 | possibleError | |
36 |
88 possibleError := true. |
37 (node firstSetWithTokens contains: [ :e | e isTokenNode not ]) ifTrue: [ ^ super visitChoiceNode: node ]. |
89 |
38 |
|
39 node children do: [ :child | |
90 node children do: [ :child | |
40 | tokenMethodName | |
91 | tokenMethodName | |
41 |
92 |
42 child firstSetWithTokens do: [ :first | |
93 child acceptsEpsilon ifTrue: [ |
43 "For each child, for each first compile this:" |
94 possibleError := false. |
44 tokenMethodName := compiler idFor: first. |
|
45 compiler add: '(self currentTokenTypeIs: ', tokenMethodName storeString, ')'. |
|
46 compiler addOnLine: ' ifTrue: ['. |
|
47 compiler indent. |
|
48 compiler codeStoreValueOf: [ self visit: child ] intoVariable: self retvalVar. |
95 compiler codeStoreValueOf: [ self visit: child ] intoVariable: self retvalVar. |
49 compiler add: 'error ifFalse: ['. |
96 compiler codeReturn |
50 compiler indent. |
97 ] ifFalse: [ |
51 compiler codeReturn: self retvalVar. |
98 child firstSetWithTokens do: [ :first | |
52 compiler dedent. |
99 "For each child, for each first compile this:" |
53 compiler add: '] ifTrue: ['. |
100 |
54 compiler indent. |
101 tokenMethodName := compiler idFor: first. |
55 compiler codeClearError. |
102 compiler add: '(self ', tokenMethodName asString, ')'. |
56 compiler codeAssign: 'nil.' to: 'currentTokenType'. |
103 compiler addOnLine: ' ifTrue: ['. |
57 compiler add: 'context position: currentTokenValue start - 1.'. |
104 compiler indent. |
58 compiler dedent. |
105 compiler codeStoreValueOf: [ self visit: child ] intoVariable: self retvalVar. |
59 compiler add: ']'. |
106 compiler add: 'error ifFalse: ['. |
60 compiler dedent. |
107 compiler indent. |
61 compiler add: '].' |
108 compiler codeReturn: self retvalVar. |
|
109 compiler dedent. |
|
110 compiler add: '] ifTrue: ['. |
|
111 compiler indent. |
|
112 compiler codeClearError. |
|
113 compiler codeAssign: 'nil.' to: 'currentTokenType'. |
|
114 compiler add: 'context position: currentTokenValue start - 1.'. |
|
115 compiler dedent. |
|
116 compiler add: ']'. |
|
117 compiler dedent. |
|
118 compiler add: '].' |
|
119 ] |
62 ] |
120 ] |
63 ]. |
121 ]. |
64 |
122 possibleError ifTrue: [ |
65 compiler codeError: 'no choice found'. |
123 compiler codeError: 'no choice found'. |
|
124 ] |
66 |
125 |
67 "Modified: / 10-05-2015 / 07:37:53 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
126 "Modified: / 10-05-2015 / 07:37:53 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
68 ! |
127 ! |
69 |
128 |
70 visitDeterministicChoiceNode: node |
129 visitDeterministicChoiceNode: node |
73 |
132 |
74 node children do: [ :child | |
133 node children do: [ :child | |
75 | firstSet | |
134 | firstSet | |
76 firstSet := child firstSetWithTokens. |
135 firstSet := child firstSetWithTokens. |
77 self assert: firstSet size = 1. |
136 self assert: firstSet size = 1. |
78 dictionary at: child |
137 dictionary at: child put: firstSet anyOne. |
79 put: firstSet anyOne. |
|
80 |
|
81 ]. |
138 ]. |
82 "Tokens are unique" |
139 "Tokens are unique" |
83 self assert: dictionary values asSet size = node children size. |
140 self assert: dictionary values asSet size = node children size. |
84 |
141 |
85 " currentTokenVar := compiler allocateTemporaryVariableNamed: 'currentToken'. |
142 node children do: [ :child | |
86 compiler codeAssign: 'self currentTokenType.' to: currentTokenVar. |
|
87 " node children do: [ :child | |
|
88 | tokenMethodName | |
143 | tokenMethodName | |
89 tokenMethodName := compiler idFor: (dictionary at: child). |
144 tokenMethodName := compiler idFor: (dictionary at: child). |
90 compiler add: '(self currentTokenTypeIs: ', tokenMethodName storeString, ')'. |
145 compiler add: '(self ', tokenMethodName asString, ')'. |
91 compiler addOnLine: ' ifTrue: ['. |
146 compiler addOnLine: ' ifTrue: ['. |
92 compiler indent. |
147 compiler indent. |
93 compiler codeStoreValueOf: [ self visit: child ] intoVariable: self retvalVar. |
148 compiler codeStoreValueOf: [ self visit: child ] intoVariable: self retvalVar. |
94 compiler codeReturn: self retvalVar. |
149 compiler codeReturn: self retvalVar. |
95 compiler dedent. |
150 compiler dedent. |
96 compiler add: '].' |
151 compiler add: '].' |
97 ]. |
152 ]. |
98 |
153 |
99 compiler codeError: 'no choice found'. |
154 compiler codeError: 'no choice found'. |
114 ]. |
169 ]. |
115 super visitChoiceNode: node." |
170 super visitChoiceNode: node." |
116 ! |
171 ! |
117 |
172 |
118 visitTokenConsumeNode: node |
173 visitTokenConsumeNode: node |
|
174 | id | |
|
175 id := (compiler idFor: node child). |
|
176 compiler add: 'self ', id asString, ' ifTrue: ['. |
|
177 compiler indent. |
|
178 compiler codeAssign: 'nil.' to: 'currentTokenType'. |
|
179 compiler codeReturn: 'currentTokenValue'. |
|
180 compiler dedent. |
|
181 compiler add: '] ifFalse: ['. |
|
182 compiler indent. |
|
183 compiler codeError: id asString, ' expected'. |
|
184 compiler dedent. |
|
185 compiler add: '].'. |
|
186 |
|
187 " |
119 compiler codeReturn: 'self consume: ', (compiler idFor: node child) storeString, '.' |
188 compiler codeReturn: 'self consume: ', (compiler idFor: node child) storeString, '.' |
|
189 " |
120 ! |
190 ! |
121 |
191 |
122 visitTokenNode: node |
192 visitTokenNode: node |
123 self error: 'shoudl not happend!!' |
193 self error: 'shoudl not happend!!' |
124 ! |
194 ! |
125 |
195 |
126 visitTokenizingParserNode: node |
196 visitTokenizingParserNode: node |
127 self visit: node tokenizer. |
197 self visit: node tokenizer. |
128 self visit: node whitespace. |
198 self visit: node whitespace. |
129 |
199 |
130 " compiler codeNextToken." |
|
131 compiler codeHaltIfShiftPressed. |
200 compiler codeHaltIfShiftPressed. |
132 compiler codeStoreValueOf: [ self visit: node parser ] intoVariable: self retvalVar. |
201 compiler codeStoreValueOf: [ self visit: node parser ] intoVariable: self retvalVar. |
133 compiler codeReturn. |
202 compiler codeReturn. |
134 ! |
203 ! |
135 |
204 |