"{ Package: 'stx:goodies/regression' }"

"{ NameSpace: RegressionTests }"

TestCase subclass:#CharacterEncoderTests

	instanceVariableNames:''

	classVariableNames:''

	poolDictionaries:''

	category:'tests-Regression-Collections'

!

!CharacterEncoderTests class methodsFor:'documentation'!

documentation

"

    documentation to be added.

    [author:]

	Claus Gittinger (cg@alan)

    [instance variables:]

    [class variables:]

    [see also:]

"

!

history

    "Created: / 11.2.2004 / 12:25:54 / cg"

! !

!CharacterEncoderTests methodsFor:'helpers'!

xTestReversability:encoderClass

    self xTestReversability:encoderClass skip:#()

    "

     self new xTestReversability: (CharacterEncoder::ISO8859_1)

    "

!

xTestReversability:encoderClass ignoreInvalid:ignoreInvalid

    self xTestReversability:encoderClass skip:#() ignoreInvalid:ignoreInvalid

    "

     self new xTestReversability: (CharacterEncoder::ISO8859_1)

    "

!

xTestReversability:encoderClass range:range skip:skippedCodes

    self

	xTestReversability:encoderClass

	range:range

	skip:skippedCodes

	ignoreInvalid:false

    "

     self new xTestReversability: (CharacterEncoder::ISO8859_1)

    "

!

xTestReversability:encoderClass range:range skip:skippedCodes ignoreInvalid:ignoreInvalid

    |encoder|

    encoder := encoderClass new.

    range do:[:code |

	|unicode code2 invalid|

	(skippedCodes includes:code) ifTrue:[

	    self should:[ encoder decode:code ] raise:CharacterEncoderError.

	] ifFalse:[

	    ignoreInvalid ifTrue:[

		invalid := false.

		CharacterEncoderError handle:[:ex |

		    invalid := true.

		] do:[

		    unicode := encoder decode:code.

		].

		invalid ifFalse:[

		    code2 := encoder encode:unicode.

		    self assert:(code == code2).

		].

	    ] ifFalse:[

		unicode := encoder decode:code.

		code2 := encoder encode:unicode.

		self assert:(code == code2).

	    ]

	]

    ].

    "

     self new xTestReversability: (CharacterEncoder::ISO8859_1)

     self new

	xTestReversability:(CharacterEncoder::JIS0208)

	range:(16r8140 to:16rEAA4)

	skip:#()

	ignoreInvalid:true

    "

!

xTestReversability:encoderClass skip:skippedCodes

    self

	xTestReversability:encoderClass

	range:(encoderClass minCode to:encoderClass maxCode)

	skip:skippedCodes

    "

     self new xTestReversability: (CharacterEncoder::ISO8859_1)

     self new xTestReversability: (CharacterEncoder::JIS0208)

    "

!

xTestReversability:encoderClass skip:skippedCodes ignoreInvalid:ignoreInvalid

    self

	xTestReversability:encoderClass

	range:(encoderClass minCode to:encoderClass maxCode)

	skip:skippedCodes

	ignoreInvalid:ignoreInvalid

    "

     self new xTestReversability: (CharacterEncoder::ISO8859_1)

     self new xTestReversability: (CharacterEncoder::JIS0208)

    "

! !

!CharacterEncoderTests methodsFor:'tests'!

test01

     self xTestReversability: (CharacterEncoderImplementations::ISO8859_1).

     self xTestReversability: (CharacterEncoderImplementations::ISO8859_2).

     self xTestReversability: (CharacterEncoderImplementations::ISO8859_3) skip:#( 16rA5 16rAE 16rBE 16rC3 16rD0 16rE3 16rF0 ).

     self xTestReversability: (CharacterEncoderImplementations::ISO8859_4).

     self xTestReversability: (CharacterEncoderImplementations::ISO8859_5).

     self xTestReversability: (CharacterEncoderImplementations::ISO8859_6) skip:#(       16rA1 16rA2 16rA3       16rA5 16rA6 16rA7

                                                                    16rA8 16rA9 16rAA 16rAB             16rAE 16rAF

                                                                    16rB0 16rB1 16rB2 16rB3 16rB4 16rB5 16rB6 16rB7

                                                                    16rB8 16rB9 16rBA       16rBC 16rBD 16rBE

                                                                    16rC0

                                                                                      16rDB 16rDC 16rDD 16rDE 16rDF

                                                                                      16rF3 16rF4 16rF5 16rF6 16rF7

                                                                    16rF8 16rF9 16rFA 16rFB 16rFC 16rFD 16rFE 16rFF

                                                                  ).

     self xTestReversability: (CharacterEncoderImplementations::ISO8859_7) skip:#( 16rAE 16rD2 16rFF ).

     self xTestReversability: (CharacterEncoderImplementations::ISO8859_8) skip:#( 16rA1 16rBF

                                                                    16rC0 16rC1 16rC2 16rC3 16rC4 16rC5 16rC6 16rC7

                                                                    16rC8 16rC9 16rCA 16rCB 16rCC 16rCD 16rCE 16rCF

                                                                    16rD0 16rD1 16rD2 16rD3 16rD4 16rD5 16rD6 16rD7

                                                                    16rD8 16rD9 16rDA 16rDB 16rDC 16rDD 16rDE

                                                                    16rFB 16rFC 16rFF

                                                                  ).

     self xTestReversability: (CharacterEncoderImplementations::ISO8859_9).

     self xTestReversability: (CharacterEncoderImplementations::ISO8859_10).

     self xTestReversability: (CharacterEncoderImplementations::ISO8859_11) skip:#( 16rDB 16rDC 16rDD 16rDE 16rFC 16rFD 16rFE 16rFF ).

     self xTestReversability: (CharacterEncoderImplementations::ISO8859_13).

     self xTestReversability: (CharacterEncoderImplementations::ISO8859_14).

     self xTestReversability: (CharacterEncoderImplementations::ISO8859_15).

     self xTestReversability: (CharacterEncoderImplementations::ISO8859_16).

     self xTestReversability: (CharacterEncoderImplementations::KOI8_U).

     self xTestReversability: (CharacterEncoderImplementations::KOI8_R).

"/     self xTestReversability: (CharacterEncoderImplementations::GSM0338) skip:( 16r80 to: 16rFF).

"/     self xTestReversability: (CharacterEncoderImplementations::JIS0208) ignoreInvalid:true.

    "

     self new test01

    "

    "Created: / 26-03-2019 / 16:21:34 / Claus Gittinger"

!

test02_UTF8

    |encoder|

    encoder := CharacterEncoder encoderToEncodeFrom:#unicode into:#utf8.

    "/ reversibility

    (0 to:16r1FFFF) do:[:eachCodePoint |

        |s1 s2 s3|

        s1 := (Character value:eachCodePoint) asString.

        s2 := encoder encodeString:s1.

        s3 := encoder decodeString:s2.

        self assert:(s1 = s3).

    ].

    "/ 00 .. 7F -> 0xxxxxxx

    #[16r00 16r01 16r02 16r04 16r08 16r10 16r20 16r40

                        16r03 16r07 16r0F 16r1F 16r3F 16r7F]

    do:[:eachCodePoint |

        |s1 s2 s3|

        s1 := (Character value:eachCodePoint) asString.

        s2 := encoder encodeString:s1.

        s3 := encoder decodeString:s2.

        self assert:(s1 = s2).

        self assert:(s2 = s3).

        self assert:(s2 size == 1).

    ].

    "/ 80 .. 7FF -> 110xxxxx 10xxxxxx

    #(16r80 16r100 16r200 16r400

            16r0FF 16r1FF 16r3FF 16r7FF)

    do:[:eachCodePoint |

        |s1 s2 s3|

        s1 := (Character value:eachCodePoint) asString.

        s2 := encoder encodeString:s1.

        self assert:(s2 size == 2).

        self assert:((s2 first codePoint bitAnd:2r11100000) == 2r11000000).

        self assert:((s2 second codePoint bitAnd:2r11000000) == 2r10000000).

        s3 := encoder decodeString:s2.

        self assert:(s1 = s3).

    ].

    "/ 800 .. FFFF -> 1110xxxx 10xxxxxx 10xxxxxx

    #(16r800 16r1000 16r2000 16r4000 16r8000

             16r0FFF 16r1FFF 16r3FFF 16r7FFF 16rFFFF)

    do:[:eachCodePoint |

        |s1 s2 s3|

        s1 := (Character value:eachCodePoint) asString.

        s2 := encoder encodeString:s1.

        self assert:(s2 size == 3).

        self assert:((s2 first codePoint bitAnd:2r11110000) == 2r11100000).

        self assert:((s2 second codePoint bitAnd:2r11000000) == 2r10000000).

        self assert:((s2 third codePoint bitAnd:2r11000000) == 2r10000000).

        s3 := encoder decodeString:s2.

        self assert:(s1 = s3).

    ].

    "/ 10000 .. 1FFFFF -> 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx

    #(16r10000 16r20000 16r40000 16r80000 16r10000

               16r1FFFF 16r3FFFF 16r7FFFF 16rFFFFF 16r1FFFFF)

    do:[:eachCodePoint |

        |s1 s2 s3|

        s1 := (Character value:eachCodePoint) asString.

        s2 := encoder encodeString:s1.

        self assert:(s2 size == 4).

        self assert:((s2 first codePoint bitAnd:2r11111000) == 2r11110000).

        self assert:((s2 second codePoint bitAnd:2r11000000) == 2r10000000).

        self assert:((s2 third codePoint bitAnd:2r11000000) == 2r10000000).

        self assert:((s2 fourth codePoint bitAnd:2r11000000) == 2r10000000).

        s3 := encoder decodeString:s2.

        self assert:(s1 = s3).

    ].

    "/ 200000 .. 3FFFFFF -> 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx

    #(16r200000 16r400000 16r800000 16r1000000 16r2000000

                16r3FFFFF 16r7FFFFF 16r0FFFFFF 16r1FFFFFF 16r3FFFFFF)

    do:[:eachCodePoint |

        |s1 s2 s3|

        s1 := (Character value:eachCodePoint) asString.

        s2 := encoder encodeString:s1.

        self assert:(s2 size == 5).

        self assert:((s2 first codePoint bitAnd:2r11111100) == 2r11111000).

        self assert:((s2 second codePoint bitAnd:2r11000000) == 2r10000000).

        self assert:((s2 third codePoint bitAnd:2r11000000) == 2r10000000).

        self assert:((s2 fourth codePoint bitAnd:2r11000000) == 2r10000000).

        self assert:((s2 fifth codePoint bitAnd:2r11000000) == 2r10000000).

        s3 := encoder decodeString:s2.

        self assert:(s1 = s3).

    ].

    "/ ST/X limitation: only 30 bit integers (to avoid largeInteger codePoint)

    "/ 4000000 .. 7FFFFFFF -> 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx

    #(16r4000000 16r8000000 16r10000000 16r20000000 "16r40000000"

                 16r7FFFFFF 16r0FFFFFFF 16r1FFFFFFF "16r3FFFFFFF 16r7FFFFFFF")

    do:[:eachCodePoint |

        |s1 s2 s3|

        s1 := (Character value:eachCodePoint) asString.

        s2 := encoder encodeString:s1.

        self assert:(s2 size == 6).

        self assert:((s2 first codePoint bitAnd:2r11111110) == 2r11111100).

        self assert:((s2 second codePoint bitAnd:2r11000000) == 2r10000000).

        self assert:((s2 third codePoint bitAnd:2r11000000) == 2r10000000).

        self assert:((s2 fourth codePoint bitAnd:2r11000000) == 2r10000000).

        self assert:((s2 fifth codePoint bitAnd:2r11000000) == 2r10000000).

        self assert:((s2 sixth codePoint bitAnd:2r11000000) == 2r10000000).

        s3 := encoder decodeString:s2.

        self assert:(s1 = s3).

    ].

    "

     self new test02_UTF8

    "

    "Created: / 26-03-2019 / 16:21:42 / Claus Gittinger"

!

test03_UTF8_2

    |encoder s|

    encoder := CharacterEncoder encoderToEncodeFrom:#unicode into:#utf8.

    self assert:(s := encoder encodeString:(Character value:16r1FFFFF)) asByteArray = #[16rF7 16rBF 16rBF 16rBF ].

    self assert:( encoder decodeString:s ) first = (Character value:16r1FFFFF).

    self assert:(s := encoder encodeString:(Character value:16r200000)) asByteArray = #[16rF8 16r88 16r80 16r80 16r80 ].

    self assert:( encoder decodeString:s ) first = (Character value:16r200000).

    self assert:( encoder encodeString:(Character value:16r2FFFFF)) asByteArray = #[16rF8 16r8B 16rBF 16rBF 16rBF ].

    self assert:( encoder encodeString:(Character value:16r3FFFFF)) asByteArray = #[16rF8 16r8F 16rBF 16rBF 16rBF ].

    self assert:( encoder encodeString:(Character value:16r4FFFFF)) asByteArray = #[16rF8 16r93 16rBF 16rBF 16rBF ].

    self assert:( encoder encodeString:(Character value:16r5FFFFF)) asByteArray = #[16rF8 16r97 16rBF 16rBF 16rBF ].

    self assert:( encoder encodeString:(Character value:16r6FFFFF)) asByteArray = #[16rF8 16r9B 16rBF 16rBF 16rBF ].

    self assert:( encoder encodeString:(Character value:16r7FFFFF)) asByteArray = #[16rF8 16r9F 16rBF 16rBF 16rBF ].

    self assert:( encoder encodeString:(Character value:16r800000)) asByteArray = #[16rF8 16rA0 16r80 16r80 16r80 ].

    self assert:( encoder encodeString:(Character value:16r3FFFFFF)) asByteArray = #[16rFB 16rBF 16rBF 16rBF 16rBF ].

    self assert:( encoder encodeString:(Character value:16r4000000)) asByteArray = #[16rFC 16r84 16r80 16r80 16r80 16r80 ].

    self assert:( encoder encodeString:(Character value:16r7FFFFFF)) asByteArray = #[16rFC 16r87 16rBF 16rBF 16rBF 16rBF ].

    self assert:( encoder encodeString:(Character value:16rFFFFFFF)) asByteArray = #[16rFC 16r8F 16rBF 16rBF 16rBF 16rBF ].

    self assert:(s := encoder encodeString:(Character value:16r3FFFFFFF)) asByteArray = #[16rFC 16rBF 16rBF 16rBF 16rBF 16rBF ].

    self assert:( encoder decodeString:s ) first = (Character value:16r3FFFFFFF).

    "/ STX <<only>> supports characters up to 31 bit.

    "/ self assert:( encoder encodeString:(Character value:16r7FFFFFFF)) asByteArray = #[16rFD 16rBF 16rBF 16rBF 16rBF 16rBF ].

    "

     self new test03_UTF8_2

    "

    "Created: / 26-03-2019 / 16:21:24 / Claus Gittinger"

!

test04_BOMDetection

     |s enc|

     s := #[1 2 3 4] readStream.

     enc := CharacterEncoder detectAndSkipBOMInStream:s.

     self assert:(enc == nil).

     self assert:(s position == 0).

     s := #[16rFF 2 3 4] readStream.

     enc := CharacterEncoder detectAndSkipBOMInStream:s.

     self assert:(enc == nil).

     self assert:(s position == 0).

     s := #[16rFF 16rFE 3 4] readStream.

     enc := CharacterEncoder detectAndSkipBOMInStream:s.

     self assert:(enc == #utf16le).

     self assert:(s position == 2).

     s := #[16rFE 16rFF 3 4] readStream.

     enc := CharacterEncoder detectAndSkipBOMInStream:s.

     self assert:(enc == #utf16be).

     self assert:(s position == 2).

     s := #[16rFF 16rFE 0 0 3 4] readStream.

     enc := CharacterEncoder detectAndSkipBOMInStream:s.

     self assert:(enc == #utf32le).

     self assert:(s position == 4).

     s := #[0 0 16rFE 16rFF 0 0 3 4] readStream.

     enc := CharacterEncoder detectAndSkipBOMInStream:s.

     self assert:(enc == #utf32be).

     self assert:(s position == 4).

    "

     self new test04_BOMDetection

    "

    "Created: / 26-03-2019 / 16:21:56 / Claus Gittinger"

! !

!CharacterEncoderTests class methodsFor:'documentation'!

version

    ^ '$Header$'

!

version_CVS

    ^ '$Header$'

! !