--- a/compiler/tests/PPCTokenizingTest.st Fri Jul 24 15:06:54 2015 +0100
+++ b/compiler/tests/PPCTokenizingTest.st Mon Aug 17 12:13:16 2015 +0100
@@ -21,11 +21,15 @@
!
cleanClass
- | parserClass |
- parserClass := (Smalltalk at: arguments name ifAbsent: [nil]).
+ | parserClass scannerClass |
+ parserClass := (Smalltalk at: arguments parserName ifAbsent: [nil]).
parserClass notNil ifTrue:[
- self flag: 'uncomment'.
-" parserClass removeFromSystem"
+ parserClass removeFromSystem
+ ].
+
+ scannerClass := (Smalltalk at: arguments scannerName ifAbsent: [nil]).
+ scannerClass notNil ifTrue:[
+ scannerClass removeFromSystem
].
!
@@ -50,7 +54,34 @@
!
tearDown
- self cleanClass
+ "self cleanClass"
+!
+
+testChoice
+ | p1 p2 a1 a2 |
+ a1 := 'a' asParser token name: 't1'; yourself.
+ a2 := 'b' asParser token name: 't2'; yourself.
+
+ p1 := a1 star.
+ p2 := a2.
+
+ parser := p1 / p2 compileWithConfiguration: configuration.
+
+ self assert: parser parse: ''.
+ self assert: result isEmpty.
+
+ self assert: parser parse: 'a'.
+ self assert: result first inputValue = 'a'.
+
+ self assert: parser parse: 'aa'.
+ self assert: result first inputValue = 'a'.
+ self assert: result second inputValue = 'a'.
+
+ self assert: parser parse: 'b' end: 0.
+ self assert: result isEmpty.
+
+ self assert: parser parse: 'c' end: 0.
+
!
testChoiceOrder
@@ -203,6 +234,21 @@
!
+testCompileEmptytoken
+ | start stop epsilon |
+ start := $( asParser token.
+ stop := $) asParser token.
+ epsilon := '' asParser token.
+
+ self should: [
+ (start, epsilon, stop) compileWithConfiguration: configuration.
+ ] raise: Exception.
+"
+ self assert: parser parse: '()'.
+ self assert: parser fail: '('.
+"
+!
+
testCompileLiteral
parser := 'foo' asParser token compileWithConfiguration: configuration.
@@ -220,6 +266,39 @@
self assert: result second inputValue = 'bar'.
!
+testCompileSequence2
+ parser := ('foo' asParser trimmingToken), ('bar' asParser trimmingToken)
+ compileWithConfiguration: configuration.
+
+ self assert: parser parse: 'foobar'.
+ self assert: result first inputValue = 'foo'.
+ self assert: result second inputValue = 'bar'.
+
+ self assert: parser parse: 'foo bar'.
+ self assert: result first inputValue = 'foo'.
+ self assert: result second inputValue = 'bar'.
+
+ self assert: parser parse: ' foo bar'.
+ self assert: result first inputValue = 'foo'.
+ self assert: result second inputValue = 'bar'.
+!
+
+testCompileSequence3
+ parser := ('foo' asParser trimmingToken),
+ ('bar' asParser trimmingToken),
+ ('baz' asParser trimmingToken)
+ compileWithConfiguration: configuration.
+
+ self assert: parser parse: 'foobarbaz'.
+ self assert: result first inputValue = 'foo'.
+ self assert: result second inputValue = 'bar'.
+
+ self assert: parser parse: ' foo bar baz '.
+ self assert: result first inputValue = 'foo'.
+ self assert: result second inputValue = 'bar'.
+ self assert: result third inputValue = 'baz'.
+!
+
testCompileStar
parser := 'foo' asParser token star compileWithConfiguration: configuration.
@@ -234,7 +313,7 @@
parser := ('foo' asParser token, 'bar' asParser token) star compileWithConfiguration: configuration.
self assert: parser parse: 'foobar'.
- self assert: context tokenReads size = 3.
+ self assert: context tokenReads size = 1.
self assert: parser parse: 'bar' end: 0.
self assert: result isEmpty.
@@ -256,7 +335,6 @@
parser := argumentsWith compileWithConfiguration: configuration.
self assert: parser parse: '|'.
- parser := argumentsWith compileWithConfiguration: configuration.
self assert: parser parse: ']'.
!
@@ -283,16 +361,41 @@
parser := tricky compileWithConfiguration: configuration.
self assert: parser parse: '||'.
- parser := tricky compileWithConfiguration: configuration.
self assert: parser parse: '|]'.
- parser := tricky compileWithConfiguration: configuration.
self assert: parser parse: ']|'.
- parser := tricky compileWithConfiguration: configuration.
self assert: parser parse: ']]'.
!
+testCompileTokenComplex4
+ | symbol symbolLiteralArray symbolLiteral arrayItem arrayLiteral |
+ "based on symbolLiteral symbolLiteralArray in SmalltalkGrammar"
+
+ symbol := PPDelegateParser new.
+ symbol setParser: 'foo' asParser.
+ symbol name: 'symbol'.
+
+ symbolLiteralArray := PPDelegateParser new.
+ symbolLiteralArray setParser: symbol token.
+ symbolLiteralArray name: 'symbolLiteralArray'.
+
+ symbolLiteral := PPDelegateParser new.
+ symbolLiteral setParser: $# asParser token, symbol token ==> [:e | e].
+ symbolLiteral name: 'symbolLiteral'.
+
+ arrayLiteral := PPDelegateParser new.
+ arrayLiteral setParser: '#(' asParser token, symbolLiteralArray, ')' asParser token.
+ arrayLiteral name: 'arrayLiteral'.
+
+ arrayItem := arrayLiteral / symbolLiteral.
+
+ parser := arrayItem compileWithConfiguration: configuration.
+
+ self assert: parser parse: '#(foo)'.
+ self assert: parser parse: '#foo'.
+!
+
testCompileTrim
parser := 'foo' asParser token trim end compileWithConfiguration: configuration.
@@ -320,7 +423,10 @@
self assert: parser parse: 'a'.
self assert: result first inputValue = 'a'.
- self assert: context invocations size = 5.
+ self assert: context tokenReads size = 1.
+
+ self flag: 'add the assertion here?'.
+" self assert: context invocations size = 5."
!
testTokenCharacter2
@@ -333,7 +439,10 @@
self assert: result first inputValue = 'a'.
self assert: result second inputValue = 'a'.
self assert: result third inputValue = 'a'.
- self assert: context invocations size = 7.
+
+ self assert: context tokenReads size = 1.
+ self flag: 'Add the assertion here?'.
+" self assert: context invocations size = 7."
!
testTokenName
@@ -364,7 +473,7 @@
self assert: parser parse: ' foo '.
self assert: result first inputValue = 'foo'.
- self assert: (context invocations select: [:e | e = #consumeWhitespace ]) size = 2.
+ self assert: (context invocations select: [:e | e = #consumeWhitespace ]) size = 3.
!
testWhitespace2
@@ -384,6 +493,27 @@
self assert: result first inputValue = 'foo'.
self assert: result second inputValue = 'foo'.
- self assert: (context invocations select: [:e | e = #consumeWhitespace ]) size = 3.
+ self assert: (context invocations select: [:e | e = #consumeWhitespace ]) size = 4.
+!
+
+testWhitespace3
+ | token ws trimmingToken |
+ configuration arguments inline: false.
+
+ token := 'foo' asParser token.
+ ws := #blank asParser star name: 'consumeWhitespace'; yourself.
+ trimmingToken := ((ws, token, ws) ==> #second)
+ propertyAt: 'trimmingToken' put: true;
+ yourself.
+
+ parser := trimmingToken plus
+ compileWithConfiguration: configuration.
+
+ self assert: parser parse: ' foo foo foo '.
+ self assert: result first inputValue = 'foo'.
+ self assert: result second inputValue = 'foo'.
+ self assert: result third inputValue = 'foo'.
+
+ self assert: (context invocations select: [:e | e = #consumeWhitespace ]) size = 5.
! !