--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/compiler/tests/PPCDistinctScannerTest.st	Mon Aug 24 15:34:14 2015 +0100
@@ -0,0 +1,228 @@
+"{ Package: 'stx:goodies/petitparser/compiler/tests' }"
+
+"{ NameSpace: Smalltalk }"
+
+TestCase subclass:#PPCDistinctScannerTest
+	instanceVariableNames:'configuration scanner'
+	classVariableNames:''
+	poolDictionaries:''
+	category:'PetitCompiler-Tests-Core-Tokenizing'
+!
+
+!PPCDistinctScannerTest methodsFor:'as yet unclassified'!
+
+aToken
+    ^ 'a' asParser token 
+        name: #token;
+        yourself.
+!
+
+barToken
+    ^ 'bar' asParser token 
+        name: #bar;
+        yourself.
+!
+
+fooToken
+    ^ 'foo' asParser token 
+        name: #foo;
+        yourself.
+!
+
+fooTrimmingToken
+    ^ 'foo' asParser trimmingToken 
+        name: #foo;
+        yourself.
+!
+
+idToken
+    ^ #letter asParser plus token 
+        name: #id;
+        yourself.
+!
+
+idTrimmingToken
+    ^ #letter asParser plus trimmingToken 
+        name: #id;
+        yourself.
+!
+
+overlappingToken
+    ^ 'a' asParser token / 'a' asParser token
+!
+
+setUp
+    configuration := PPCConfiguration tokenizing.
+!
+
+testConsumeToken
+    | parser |
+    parser := self fooToken, self overlappingToken.
+    parser compileWithConfiguration: configuration.
+    scanner := (Smalltalk at: configuration arguments scannerName) new.
+
+    scanner stream: 'foobaz' asPetitStream.
+    scanner perform: #'consume_foo'.
+    
+    self assert: scanner position = 3.
+    self assert: scanner resultPosition = 3.
+    self assert: scanner result isNil.
+!
+
+testConsumeToken2
+    | parser |
+    parser := self fooToken, self barToken.
+    parser compileWithConfiguration: configuration.
+    scanner := (Smalltalk at: configuration arguments scannerName) new.
+
+    scanner stream: 'foobar' asPetitStream.
+    scanner perform: #'consume_foo'.
+    
+    self assert: scanner position = 3.
+    self assert: scanner resultPosition = 6.
+    self assert: scanner result = #bar.
+!
+
+testScan
+    | parser |
+    parser := self aToken.
+    parser compileWithConfiguration: configuration.
+    
+    scanner := (Smalltalk at: configuration arguments scannerName) new.
+
+    scanner stream: 'a' asPetitStream.
+    scanner perform: #'scan_token'.
+    
+    self assert: scanner position = 0.
+    self assert: scanner resultPosition = 1.
+    self assert: scanner result = #token.
+!
+
+testScan2
+    | parser |
+    parser := self fooToken.
+    parser compileWithConfiguration: configuration.
+    
+    scanner := (Smalltalk at: configuration arguments scannerName) new.
+
+    scanner stream: 'foo' asPetitStream.
+    scanner perform: #'scan_foo'.
+    
+    self assert: scanner position = 0.
+    self assert: scanner resultPosition = 3.
+    self assert: scanner result = #foo.
+!
+
+testScan3
+    | parser |
+    parser := self fooToken.
+    parser compileWithConfiguration: configuration.
+    
+    scanner := (Smalltalk at: configuration arguments scannerName) new.
+
+    scanner stream: 'bar' asPetitStream.
+    scanner perform: #'scan_foo'.
+    
+    self assert: scanner position = 0.
+    self assert: scanner resultPosition = 0.
+    self assert: scanner result isNil.
+!
+
+testScan4
+    | parser |
+    parser := self fooToken, self idToken.
+    parser compileWithConfiguration: configuration.
+    
+    scanner := (Smalltalk at: configuration arguments scannerName) new.
+
+    scanner stream: 'foothere' asPetitStream.
+    scanner perform: #'scan_foo'.
+    
+    self assert: scanner position = 0.
+    self assert: scanner resultPosition = 3.
+    self assert: scanner result = #foo.
+    
+    scanner perform: #'consume_foo'.
+
+    self assert: scanner position = 3.
+    self assert: scanner resultPosition = 8.
+    self assert: scanner result = #id.
+!
+
+testSequence
+    | parser result |
+    parser := self fooTrimmingToken, self idTrimmingToken.
+    parser compileWithConfiguration: configuration.
+    
+    scanner := (Smalltalk at: configuration arguments scannerName) new.
+
+    scanner stream: 'foo there ' asPetitStream.
+    scanner perform: #'foo'.
+    
+    self assert: scanner position = 0.
+    self assert: scanner resultPosition = 3.
+    self assert: scanner stream position = 4.
+    self assert: scanner result = #foo.	
+    
+    result := scanner perform: #'consume_foo'.
+
+    self assert: scanner position = 4.
+    self assert: scanner resultPosition = 9.
+    self assert: scanner stream position = 10.
+    self assert: scanner result = #id.
+    
+    self assert: (result isKindOf: PPToken).
+    self assert: result start = 1.
+    self assert: result stop = 3.
+    self assert: result inputValue = 'foo'.
+!
+
+testToken
+    | parser |
+    parser := self fooToken, self idTrimmingToken.
+    parser compileWithConfiguration: configuration.
+    
+    scanner := (Smalltalk at: configuration arguments scannerName) new.
+
+    scanner stream: 'foo there' asPetitStream.
+    scanner perform: #'foo'.
+    
+    self assert: scanner position = 0.
+    self assert: scanner resultPosition = 3.
+    self assert: scanner stream position = 3.
+    self assert: scanner result = #foo.		
+!
+
+testTrimmingScan
+    | parser |
+    parser := self fooTrimmingToken, self idTrimmingToken.
+    parser compileWithConfiguration: configuration.
+    
+    scanner := (Smalltalk at: configuration arguments scannerName) new.
+
+    scanner stream: 'foo there' asPetitStream.
+    scanner perform: #'scan_foo'.
+    
+    self assert: scanner position = 0.
+    self assert: scanner resultPosition = 3.
+    self assert: scanner result = #foo.	
+!
+
+testTrimmingToken
+    | parser result |
+    parser := self fooTrimmingToken, self idTrimmingToken.
+    parser compileWithConfiguration: configuration.
+    
+    scanner := (Smalltalk at: configuration arguments scannerName) new.
+
+    scanner stream: 'foo there' asPetitStream.
+    result := scanner perform: #'foo'.
+    
+    self assert: scanner position = 0.
+    self assert: scanner resultPosition = 3.
+    self assert: scanner stream position = 4.
+    self assert: scanner result = #foo.		
+        
+    self assert: result.
+! !
+