"

 COPYRIGHT (c) 1994 by Claus Gittinger

	      All Rights Reserved

 This software is furnished under a license and may be used

 only in accordance with the terms of that license and with the

 inclusion of the above copyright notice.   This software may not

 be provided or otherwise made available to, or used by, any

 other person.  No title to or ownership of the software is

 hereby transferred.

"

"{ Package: 'stx:libbasic' }"

ByteArray variableByteSubclass:#CharacterArray

	instanceVariableNames:''

	classVariableNames:'PreviousMatch DecoderTables EncoderTables DecodingFailedSignal

		EncodingFailedSignal'

	poolDictionaries:''

	category:'Collections-Text'

!

!CharacterArray class methodsFor:'documentation'!

copyright

"

 COPYRIGHT (c) 1994 by Claus Gittinger

	      All Rights Reserved

 This software is furnished under a license and may be used

 only in accordance with the terms of that license and with the

 inclusion of the above copyright notice.   This software may not

 be provided or otherwise made available to, or used by, any

 other person.  No title to or ownership of the software is

 hereby transferred.

"

!

documentation

"

    CharacterArray is a superclass for all kinds of Strings (i.e.

    (singleByte-)Strings, TwoByteStrings and whatever comes in the future.

    This class is abstract, meaning that there are no instances of it. 

    All this class does is provide common protocol for concrete subclasses.

    [author:]

        Claus Gittinger

    [see also:]

        String TwoByteString

        StringCollection

"

! !

!CharacterArray class methodsFor:'initialization'!

initialize

    DecodingFailedSignal isNil ifTrue:[    

        DecodingFailedSignal := Signal new mayProceed:true.

        DecodingFailedSignal notifierString:'decoding failure'.

        DecodingFailedSignal nameClass:self message:#decodingFailedSignal.

        EncodingFailedSignal := QuerySignal new mayProceed:true.

        EncodingFailedSignal notifierString:'encoding failure'.

        EncodingFailedSignal nameClass:self message:#encodingFailedSignal.

    ]

    "

     CharacterArray initialize

    "

    "Modified: 3.8.1997 / 18:15:59 / cg"

! !

!CharacterArray class methodsFor:'instance creation'!

basicNew

    "return a new empty string"

    ^ self basicNew:0

!

fromBytes:aByteCollection

    "return an instance of the receiver class,

     taking untranslated bytes from the argument, aByteCollection.

     Only useful, when reading twoByteStrings from external sources."

    |mySize nBytes newString dstIdx|

    nBytes := aByteCollection size.

    mySize := self basicNew bitsPerCharacter.

    mySize == 16 ifTrue:[

        newString := self basicNew:(nBytes // 2).

        dstIdx := 1.

        aByteCollection pairWiseDo:[:hi :lo |

            newString at:dstIdx put:(Character value:(hi bitShift:8)+lo).

            dstIdx := dstIdx + 1

        ].

        ^ newString.

    ].

    ^ (self basicNew:nBytes) replaceFrom:1 with:aByteCollection

    "

     TwoByteString fromBytes:#[16r21 16r21]

    "

    "Modified: 30.6.1997 / 20:08:37 / cg"

!

fromString:aString

    "return a copy of the argument, aString"

    ^ (self basicNew:(aString size)) replaceFrom:1 with:aString

    "TwoByteString fromString:'hello'"

!

fromStringCollection:aCollectionOfStrings

    "return new string formed by concatenating a copy of the argument, aString"

    ^ self fromStringCollection:aCollectionOfStrings separatedBy:''

    "

     String fromStringCollection:#('hello' 'world' 'how' 'about' 'this')

    "

    "Created: 20.11.1995 / 15:26:59 / cg"

!

fromStringCollection:aCollectionOfStrings separatedBy:aSeparatorString

    "return new string formed by concatenating a copy of the argument, aString"

    |newString first|

    newString := ''.

    first := true.

    aCollectionOfStrings do:[:s | 

	first ifFalse:[

	    newString := newString , aSeparatorString

	] ifTrue:[

	    first := false

	].

	newString := newString , s

    ].

    ^ newString

    "

     String fromStringCollection:#('hello' 'world' 'how' 'about' 'this') separatedBy:' '

    "

    "Created: 20.11.1995 / 15:32:17 / cg"

!

fromUTF8Bytes:aByteCollection

    "return a new string which represents the characters as decoded

     from the utf8 encoded bytes, aByteCollection.

     Returns either a normal String, or a TwoByteString instance.

     Only useful, when reading twoByteStrings from external sources.

     This only handles up-to 16bit characters"

    |sz nBitsRequired ascii s byte newString idx|

    nBitsRequired := 8.

    sz := 0.

    s := aByteCollection readStream.

    [s atEnd] whileFalse:[

        byte := ascii := s nextByte.

        (byte bitAnd:16r80) ~~ 0 ifTrue:[

            (byte bitAnd:16rE0) == 16rC0 ifTrue:[

                ascii := (byte bitAnd:16r1F).

                byte := s nextByte.

                ascii := (ascii bitShift:6) bitOr:(byte bitAnd:16r3F).

                ascii > 16rFF ifTrue:[

                    nBitsRequired := nBitsRequired max:16

                ].

                ascii <= 16r7F ifTrue:[

"/                    self error:'invalid utf encoding' mayProceed:true.

                ].

            ] ifFalse:[

                (byte bitAnd:16rF0) == 16rE0 ifTrue:[

                    ascii := (byte bitAnd:16r0F).

                    byte := s nextByte.

                    ascii := (ascii bitShift:6) bitOr:(byte bitAnd:16r3F).

                    byte := s nextByte.

                    ascii := (ascii bitShift:6) bitOr:(byte bitAnd:16r3F).

                    ascii > 16rFF ifTrue:[

                        nBitsRequired := nBitsRequired max:16

                    ].

                    ascii <= 16r7FF ifTrue:[

                        self error:'invalid utf encoding'.

                    ].

                ] ifFalse:[

                    self error:'bad/unsupported utf encoding'.

                    ^ nil.

                ]

            ].

        ].

        sz := sz + 1.

    ].

    nBitsRequired == 8 ifTrue:[

        newString := String new:sz

    ] ifFalse:[

        newString := UnicodeString new:sz

    ].

    s := aByteCollection readStream.

    idx := 1.

    [s atEnd] whileFalse:[

        byte := ascii := s nextByte.

        (byte bitAnd:16r80) ~~ 0 ifTrue:[

            (byte bitAnd:16rE0) == 16rC0 ifTrue:[

                ascii := (byte bitAnd:16r1F).

                byte := s nextByte.

                ascii := (ascii bitShift:6) bitOr:(byte bitAnd:16r3F).

            ] ifFalse:[

                ascii := (byte bitAnd:16r0F).

                byte := s nextByte.

                ascii := (ascii bitShift:6) bitOr:(byte bitAnd:16r3F).

                byte := s nextByte.

                ascii := (ascii bitShift:6) bitOr:(byte bitAnd:16r3F).

            ].

        ].

        newString at:idx put:(Character value:ascii).

        idx := idx + 1.

    ].

    ^ newString

    "

     CharacterArray fromUTF8Bytes:#[ 16r41 16r42 ]      

     CharacterArray fromUTF8Bytes:#[ 16rC1 16r02 ]      

     CharacterArray fromUTF8Bytes:#[ 16rE0 16r81 16r02 ]      

     CharacterArray fromUTF8Bytes:#[ 16rEF 16rBF 16rBF ]

   rfc2279 examples:

     CharacterArray fromUTF8Bytes:#[ 16r41 16rE2 16r89 16rA2 16rCE 16r91 16r2E ]      

     CharacterArray fromUTF8Bytes:#[ 16rED 16r95 16r9C 16rEA 16rB5 16rAD 16rEC 16r96 16rB4 ]      

     CharacterArray fromUTF8Bytes:#[ 16rE6 16r97 16rA5 16rE6 16r9C 16rAC 16rE8 16rAA 16r9E ]      

   invalid:

     CharacterArray fromUTF8Bytes:#[ 16rC0 16r80 ]      

     CharacterArray fromUTF8Bytes:#[ 16rE0 16r80 16r80 ]      

    "

!

new

    "return a new empty string"

    ^ self basicNew:0

! !

!CharacterArray class methodsFor:'Signal constants'!

decodingFailedSignal

    "return the signal, raised when decoding of a string is not possible

     due to invalid characters contained in the source.

     This may happen for example, if a non EUC coded 8-bit string

     is attempted to be decoded into a JIS string."

    ^ DecodingFailedSignal

    "Created: 28.6.1997 / 20:09:55 / cg"

    "Modified: 3.8.1997 / 18:16:47 / cg"

!

encodingFailedSignal

    "return the (query-) signal, raised when encoding of a string is not possible

     due to invalid characters contained in the source."

    ^ EncodingFailedSignal

    "Modified: 28.6.1997 / 20:09:35 / cg"

    "Created: 3.8.1997 / 18:16:40 / cg"

! !

!CharacterArray class methodsFor:'cleanup'!

lowSpaceCleanup

    "cleanup in low-memory situations"

    DecoderTables := EncoderTables := nil

    "

     CharacterArray lowSpaceCleanup

    "

    "Created: 22.2.1996 / 16:30:30 / cg"

    "Modified: 22.2.1996 / 17:58:05 / cg"

! !

!CharacterArray class methodsFor:'code tables'!

classForEncoding:encodingSymbol

    "return a class capable of representing text encoded

     by encodingSymbol"

    encodingSymbol == #jis ifTrue:[^ JISEncodedString].

    encodingSymbol == #big5 ifTrue:[^ BIG5EncodedString].

    encodingSymbol == #gb ifTrue:[^ GBEncodedString].

    encodingSymbol == #ksc ifTrue:[^ KSCEncodedString].

    encodingSymbol == #unicode ifTrue:[^ UnicodeString].

    "/

    "/ assume its an 8-bit code

    "/

    ^ String

    "Modified: 30.6.1997 / 19:48:05 / cg"

!

decoderTableFor:encoding

    "return a table to decode from an alien encoding into

     the internal (iso8859, ansi) format.

     This is an experimental interface - unfinished"

    |table unrepresentableCharacterCode x80Table|

    DecoderTables notNil ifTrue:[

        table := DecoderTables at:encoding ifAbsent:nil.

        table notNil ifTrue:[

            table == #identity ifTrue:[^ nil].

            ^ table

        ].

    ] ifFalse:[

        DecoderTables := IdentityDictionary new

    ].

    unrepresentableCharacterCode := 16rBF.

    "/

    "/ setup as identityTranslation

    "/

    table := (0 to:255) asByteArray.

    "/

    "/ hp -> iso8859

    "/

    "/ hp uses a funny encoding of the A1 .. FF characters

    "/

    "/   

    encoding == #hp ifTrue:[

        unrepresentableCharacterCode := 16rA0.

        x80Table :=

            #(

               16r0080 16r0081 16r0082 16r0083 16r0084 16r0085 16r0086 16r0087  "/ 80

               16r0088 16r0089 16r008a 16r008b 16r008c 16r008d 16r008e 16r008f

               16r0090 16r0091 16r0092 16r0093 16r0094 16r0095 16r0096 16r0097  "/ 90

               16r0098 16r0099 16r009a 16r009b 16r009c 16r009d 16r009e 16r009f

               16r00a0 16r00c0 16r00c2 16r00c8 16r00c9 16r00cb 16r00ce 16r00cf  "/ a0

               16r00b4 16rFFFF 16rFFFF 16r00a8 16rFFFF 16r00d9 16r00da 16r00a3

               16rFFFF 16r00dd 16r00fd 16r00b0 16r00c7 16r00e7 16r00d1 16r00f1  "/ b0

               16r00a1 16r00bf 16r00a4 16r00a3 16r00a5 16r00a7 16rFFFF 16rFFFF

               16r00e2 16r00ea 16r00f4 16r00fb 16r00e1 16r00e9 16r00f3 16r00fa  "/ c0

               16r00e0 16r00e8 16r00f2 16r00f9 16r00e4 16r00eb 16r00f6 16r00fc

               16r00c5 16r00ee 16r00d8 16r00c6 16r00e5 16r00ed 16r00f8 16r00e6  "/ d0

               16r00c4 16r00ec 16r00d6 16r00dc 16r00c9 16r00ef 16r00df 16r00d4

               16r00c1 16r00c3 16r00e3 16r00d0 16r00f0 16r00cd 16r00cc 16r00d3  "/ e0

"/                                        S ?       s ?             Y ?

               16r00d2 16r00d5 16r00f5 16rFFFF 16rFFFF 16r00da 16rFFFF 16r00ff 

               16r00de 16r00fe 16r00b7 16r00b5 16r00b6 16r00be 16r00ad 16r00bc  "/ f0

               16r00bd 16r00aa 16r00ba 16r00ab 16rFFFF 16r00bb 16r00b1 16rFFFF

            ).

    ].

    "/

    "/ mac -> iso8859

    "/

    "/ You will loose the following characters:

    "/ - all of them will be replaced by 8F

    "/

    "/  A0 (dagger)

    "/  A5 (dot-dot)

    "/  AA (tm - trademark)

    "/  AD (not equal)

    "/  B0 (infinity)

    "/  B3 (less-double-equal <=)

    "/  B4 (greater-equal >-)

    "/  B6 (math lowercase delta)

    "/  B7 (math sum)

    "/  B8 (math uppercase pi)

    "/  B9 (math lowercase pi)

    "/  BA (math integral)

    "/  BD (math omega)

    "/  C3 (math union)

    "/  C4 (ext latin f)

    "/  C5 (math almost equal)

    "/  C6 (math uppercase delta)

    "/  C9 (dot-dot-dot)

    "/  CE (OE ligature)

    "/  CF (oe ligature)

    "/  D1 (hyphen)

    "/  D2 (opening top dquote)

    "/  D3 (closing top dquote)

    "/  D4 (opening top quote)

    "/  D5 (closing top quote)

    "/  D7 (geometric: )

    "/  D9 (uppercase Y diacrit )

    "/  DA (slash2 )

    "/  DC (single oldStyle opening quote (<) )

    "/  DD (single oldStyle closing quote (>) )

    "/  DE (? )

    "/  DF (? )

    "/  E0 (double dagger )

    "/  E2 (opening bottom quote )

    "/  E3 (opening bottom dquote )

    "/  F0 (? )

    "/  F5 (latin l )

    "/  F6 (diacrit circumflex)

    "/  F7 (diacrit tilde)

    "/  F8 (diacrit top line)

    "/  F9 (diacrit inverse circumflex)

    "/  FA (diacrit dot)

    "/  FB (diacrit ring)

    "/  FC (diacrit cedille left)

    "/  FD (diacrit dquote)

    "/  FE (diacrit cedille right)

    "/  FF (diacrit circumflex2 ?)

    "/   

    encoding == #mac ifTrue:[

        x80Table :=

            #( 16r00c4 16r00c5 16r00c7 16r00c9 16r00d1 16r00d6 16r00dc 16r00e1  "/ 80

               16r00e0 16r00e2 16r00e4 16r00e3 16r00e5 16r00e7 16r00e9 16r00e8

               16r00ea 16r00eb 16r00ed 16r00ec 16r00ee 16r00ef 16r00f1 16r00f3  "/ 90

               16r00f2 16r00f4 16r00f6 16r00f5 16r00fa 16r00f9 16r00fb 16r00fc

               16rFFFF 16r00b0 16r00a2 16r00a3 16r00a7 16rFFFF 16r00b6 16r00df  "/ a0

               16r00ae 16r00a9 16rFFFF 16r00b4 16r00a8 16rFFFF 16r00c6 16r00d8

               16rFFFF 16r00b1 16rFFFF 16rFFFF 16r00a5 16r00b5 16rFFFF 16rFFFF  "/ b0

               16rFFFF 16rFFFF 16rFFFF 16r00aa 16r00ba 16rFFFF 16r00e6 16r00f8

               16r00bf 16r00a1 16r00ac 16rFFFF 16rFFFF 16rFFFF 16rFFFF 16r00ab  "/ c0

               16r00bb 16rFFFF 16r00a0 16r00c1 16r00c3 16r00d5 16rFFFF 16rFFFF

               16r00ad 16rFFFF 16rFFFF 16rFFFF 16rFFFF 16rFFFF 16r00f7 16rFFFF  "/ d0

               16rFFFF 16rFFFF 16rFFFF 16r00a4 16rFFFF 16rFFFF 16rFFFF 16rFFFF

               16rFFFF 16r00b7 16rFFFF 16rFFFF 16rFFFF 16r00c2 16r00ca 16r00c1  "/ e0

               16r00cb 16r00c8 16r00cd 16r00ce 16r00cf 16r00cc 16r00d3 16r00d4

               16rFFFF 16r00d2 16r00da 16r00db 16r00d9 16rFFFF 16rFFFF 16rFFFF  "/ f0

               16rFFFF 16rFFFF 16rFFFF 16rFFFF 16rFFFF 16rFFFF 16rFFFF 16rFFFF

            ).

    ].

    "/

    "/ next -> iso8859

    "/

    "/ You will loose the following characters:

    "/ - all of them will be replaced by 8F

    "/

    "/  A4 (diacrit /)

    "/  A9 (diacrit quote)

    "/  AA (top opening dquote)

    "/  AC (single oldStyle opening quote (<))

    "/  AD (single oldStyle closing quote (>))

    "/  AE (?)

    "/  AF (?)

    "/  B2 (dagger)

    "/  B3 (double dagger)

    "/  B7 (enter dot)

    "/  B8 (opening bottom quote)

    "/  B9 (opening bottom dquote)

    "/  BA (closing top dquote)

    "/  BC (dot-dot-dot)

    "/  BD (per-mille)

    "/  C0 (subscript 1)

    "/  C1 (accent grave)

    "/  C2 (accent degu)

    "/  C3 (accent circonflex)

    "/  C4 (accent tilde)

    "/  C5 (accent line)

    "/  C6 (accent circonflex reverse)

    "/  C7 (accent dot)

    "/  C8 (accent dot-dot)

    "/  D0 (hline)

    "/  E8 (engl. pound upper case L)

    "/  EA (OE ligature)

    "/  F5 (latin l)

    "/  F8 (engl. pound lower case l)

    "/  FE (?)