"

 COPYRIGHT (c) 1994 by Claus Gittinger

 COPYRIGHT (c) 2009 by eXept Software AG

              All Rights Reserved

 This software is furnished under a license and may be used

 only in accordance with the terms of that license and with the

 inclusion of the above copyright notice.   This software may not

 be provided or otherwise made available to, or used by, any

 other person.  No title to or ownership of the software is

 hereby transferred.

"

"{ Package: 'stx:libbasic2' }"

"{ NameSpace: Smalltalk }"

Object subclass:#PhoneticStringUtilities

	instanceVariableNames:''

	classVariableNames:''

	poolDictionaries:''

	category:'Collections-Text-Support'

!

Object subclass:#PhoneticStringComparator

	instanceVariableNames:''

	classVariableNames:''

	poolDictionaries:''

	privateIn:PhoneticStringUtilities

!

PhoneticStringUtilities::PhoneticStringComparator subclass:#ExtendedSoundexStringComparator

	instanceVariableNames:''

	classVariableNames:'CharacterTranslationDict'

	poolDictionaries:''

	privateIn:PhoneticStringUtilities

!

PhoneticStringUtilities::PhoneticStringComparator subclass:#KoelnerPhoneticCodeStringComparator

	instanceVariableNames:''

	classVariableNames:'CharacterTranslationDict'

	poolDictionaries:''

	privateIn:PhoneticStringUtilities

!

PhoneticStringUtilities::PhoneticStringComparator subclass:#SoundexStringComparator

	instanceVariableNames:''

	classVariableNames:'CharacterTranslationDict'

	poolDictionaries:''

	privateIn:PhoneticStringUtilities

!

PhoneticStringUtilities::SoundexStringComparator subclass:#MySQLSoundexStringComparator

	instanceVariableNames:''

	classVariableNames:''

	poolDictionaries:''

	privateIn:PhoneticStringUtilities

!

Object subclass:#NYSIISStringComparator

	instanceVariableNames:''

	classVariableNames:''

	poolDictionaries:''

	privateIn:PhoneticStringUtilities

!

PhoneticStringUtilities::PhoneticStringComparator subclass:#PhonemStringComparator

	instanceVariableNames:''

	classVariableNames:'CharacterTranslationDict'

	poolDictionaries:''

	privateIn:PhoneticStringUtilities

!

PhoneticStringUtilities::PhoneticStringComparator subclass:#DoubleMetaphoneStringComparator

	instanceVariableNames:'inputKey primaryTranslation secondaryTranslation startIndex

		currentIndex skipCount'

	classVariableNames:''

	poolDictionaries:''

	privateIn:PhoneticStringUtilities

!

PhoneticStringUtilities::SoundexStringComparator subclass:#MiracodeStringComparator

	instanceVariableNames:''

	classVariableNames:''

	poolDictionaries:''

	privateIn:PhoneticStringUtilities

!

!PhoneticStringUtilities class methodsFor:'documentation'!

copyright

"

 COPYRIGHT (c) 1994 by Claus Gittinger

 COPYRIGHT (c) 2009 by eXept Software AG

              All Rights Reserved

 This software is furnished under a license and may be used

 only in accordance with the terms of that license and with the

 inclusion of the above copyright notice.   This software may not

 be provided or otherwise made available to, or used by, any

 other person.  No title to or ownership of the software is

 hereby transferred.

"

!

documentation

"

    Utilities which are helpful to perform phonetic string searches or comparisons.

    These are all variations or improvements of the soundex algorithm, which usually fails

    to provide good results for non-english languages.

    soundexCode

        this algorithm was originally contained in the CharacterArray class;

    nysiis

        a modified soundex algorithm

    miracode

        another modified soundex algorithm ('american soundex') used in the 1880 census.

    mySQLSoundex

        another modified soundex algorithm used in mySQL.

    koelner phoneticCode 

        provides a functionality similar to soundex, but much more tuned towards the German language

    Double metaphone 

        works with most european languages.

    phonem

        described in Georg Wilde and Carsten Meyer, 'Doppelgaenger gesucht - Ein Programm fuer kontextsensitive phonetische Textumwandlung'

        from 'ct Magazin fuer Computer & Technik 25/1999'.

    More info for german readers is found in:

        http://www.uni-koeln.de/phil-fak/phonetik/Lehre/MA-Arbeiten/magister_wilz.pdf

"

!

sampleData

"

    for the 50 most common german names, we get:

                            ext. 

    name        soundex   soundex   metaphone   phonet  phonet2     phonix      daitsch phonem      koeln

    müller      M460    54600000    MLR         MÜLA    NILA        M4000000    689000  MYLR        657

    schmidt     S253    25300000    SKMTT       SHMIT   ZNIT        S5300000    463000  CMYD        8628

    schneider   S253    25360000    SKNTR       SHNEIDA ZNEITA      S5300000    463900  CNAYDR      8627

    fischer     F260    12600000    FSKR        FISHA   FIZA        F8000000    749000  VYCR        387

    weber       W160    16000000    WBR         WEBA    FEBA        $1000000    779000  VBR         317

    meyer       M600    56000000    MYR         MEIA    NEIA        M0000000    619000  MAYR        67

    wagner      W256    25600000    WKNR        WAKNA   FAKNA       $2500000    756900  VACNR       367

    schulz      S242    24200000    SKLS        SHULS   ZULZ        S4800000    484000  CULC        85

    becker      B260    12600000    BKR         BEKA    BEKA        B2000000    759000  BCR         147

    hoffmann    H155    15500000    HFMN        HOFMAN  UFNAN       $7550000    576600  OVMAN       036

    schäfer     S216    21600000    SKFR        SHEFA   ZEFA        S7000000    479000  CVR         837

"

! !

!PhoneticStringUtilities class methodsFor:'phonetic codes'!

koelnerPhoneticCodeOf:aString

    "return a koelner phonetic code.

     The koelnerPhonetic code is for the german language what the soundex code is for english;

     it returns simular strings for similar sounding words. 

     There are some differences to soundex, though: 

        its length is not limited to 4, but depends on the length of the original string;

        it does not start with the first character of the input.

     This algorithm is described by Postel 1969"

    ^ (KoelnerPhoneticCodeStringComparator new phoneticStringsFor:aString) first

    "

     #(

        'Müller'

        'Miller'

        'Mueller'

        'Mühler'

        'Mühlherr'

        'Mülherr'

        'Myler'

        'Millar'

        'Myller'

        'Müllar'

        'Müler'

        'Muehler'

        'Mülller'

        'Müllerr'

        'Muehlherr'

        'Muellar'

        'Mueler'

        'Mülleer'

        'Mueller'

        'Nüller'

        'Nyller'

        'Niler'

        'Czerny'

        'Tscherny'

        'Czernie'

        'Tschernie'

        'Schernie'

        'Scherny'

        'Scherno'

        'Czerne'

        'Zerny'

        'Tzernie'

        'Breschnew'

     ) do:[:w |

         Transcript show:w; show:'->'; showCR:(PhoneticStringUtilities koelnerPhoneticCodeOf:w)

     ].

    "

    "

     PhoneticStringUtilities koelnerPhoneticCodeOf:'Breschnew'. '17863'.

     PhoneticStringUtilities koelnerPhoneticCodeOf:'Breschneff'. '17863'.

     PhoneticStringUtilities koelnerPhoneticCodeOf:'Braeschneff'. '17863'.

     PhoneticStringUtilities koelnerPhoneticCodeOf:'Braessneff'. '17863'.

     PhoneticStringUtilities koelnerPhoneticCodeOf:'Pressneff'. '17863'.

     PhoneticStringUtilities koelnerPhoneticCodeOf:'Presznäph'. '17863'.

     PhoneticStringUtilities koelnerPhoneticCodeOf:'Preschnjiev'. '17863'.

    "

!

mySQLSoundexCodeOf:aString

    "return the mySQL soundex code. The mysql soundex coed is different from the miracode 'american' soundex

     (no 4char limitation; different order of duplicate vowel vs. duplicate code elimination)"

    ^ (MySQLSoundexStringComparator new phoneticStringsFor:aString) first

    "

     #(

        'Müller'

        'Miller'

        'Mueller'

        'Mühler'

        'Mühlherr'

        'Mülherr'

        'Myler'

        'Millar'

        'Myller'

        'Müllar'

        'Müler'

        'Muehler'

        'Mülller'

        'Müllerr'

        'Muehlherr'

        'Muellar'

        'Mueler'

        'Mülleer'

        'Mueller'

        'Nüller'

        'Nyller'

        'Niler'

        'Czerny'

        'Tscherny'

        'Czernie'

        'Tschernie'

        'Schernie'

        'Scherny'

        'Scherno'

        'Czerne'

        'Zerny'

        'Tzernie'

        'Breschnew'

     ) do:[:w |

         Transcript show:w; show:'->'; showCR:(PhoneticStringUtilities mySQLSoundexCodeOf:w)

     ].

    "

    "

     PhoneticStringUtilities mySQLSoundexCodeOf:'Breschnew'. 

     PhoneticStringUtilities mySQLSoundexCodeOf:'Breschneff'. 

     PhoneticStringUtilities mySQLSoundexCodeOf:'Braeschneff'. 

     PhoneticStringUtilities mySQLSoundexCodeOf:'Braessneff'.

     PhoneticStringUtilities mySQLSoundexCodeOf:'Pressneff'. 

     PhoneticStringUtilities mySQLSoundexCodeOf:'Presznäph'. 

     PhoneticStringUtilities mySQLSoundexCodeOf:'Preschnjiev'.

    "

!

soundexCodeOf:aString

    "return a soundex phonetic code or nil.

     Soundex (1918, 1922) returns similar codes for similar sounding words, making it a useful

     tool when searching for words where the correct spelling is unknown.

     (read Knuth or search the web if you don't know what a soundex code is).

     Caveat: 'similar sounding words' means: 'similar sounding in english'."

    ^ (SoundexStringComparator new phoneticStringsFor:aString) first

"/ old code - now use code in private class...

"/    |inStream codeStream ch last lch codeLength codes code lastCode|

"/

"/    inStream := aString readStream.

"/    inStream skipSeparators.

"/    inStream atEnd ifTrue:[

"/        ^ nil

"/    ].

"/

"/    ch := inStream next.

"/    ch isLetter ifFalse:[

"/        ^ nil

"/    ].

"/    codeLength := 0.

"/

"/    codes := Dictionary new.

"/    codes atAll:'bpfv'     put:$1.

"/    codes atAll:'cskgjqxz' put:$2.

"/    codes atAll:'dt'       put:$3.

"/    codes atAll:'l'        put:$4.

"/    codes atAll:'nm'       put:$5.

"/    codes atAll:'r'        put:$6.

"/

"/    codeStream := WriteStream on:(String new:4).

"/    codeStream nextPut:(ch asUppercase).

"/    last := ch asLowercase.

"/    lastCode := codes at:last ifAbsent:nil.

"/

"/    [inStream atEnd] whileFalse:[

"/        ch := inStream next.

"/        lch := ch asLowercase.

"/        lch = last ifFalse:[

"/            last := lch.

"/

"/            code := codes at:lch ifAbsent:nil.

"/            (code notNil and:[ code ~= lastCode]) ifTrue:[

"/                codeLength < 3 ifTrue:[

"/                    codeStream nextPut:code.

"/                    codeLength := codeLength + 1.

"/                    codeLength > 3 ifTrue:[^ codeStream contents].

"/                ].

"/            ].

"/            lastCode := code.

"/        ]

"/    ].

"/    [ codeLength < 3 ] whileTrue:[

"/        codeStream nextPut:$0.

"/        codeLength := codeLength + 1.

"/    ].

"/

"/    ^ codeStream contents

    "

     PhoneticStringUtilities soundexCodeOf:'claus'   

     PhoneticStringUtilities soundexCodeOf:'clause'   

     PhoneticStringUtilities soundexCodeOf:'close'   

     PhoneticStringUtilities soundexCodeOf:'smalltalk' 

     PhoneticStringUtilities soundexCodeOf:'smaltalk'  

     PhoneticStringUtilities soundexCodeOf:'smaltak'   

     PhoneticStringUtilities soundexCodeOf:'smaltok'   

     PhoneticStringUtilities soundexCodeOf:'smoltok'   

     PhoneticStringUtilities soundexCodeOf:'aa'        

     PhoneticStringUtilities soundexCodeOf:'by'        

     PhoneticStringUtilities soundexCodeOf:'bab'       

     PhoneticStringUtilities soundexCodeOf:'bob'       

     PhoneticStringUtilities soundexCodeOf:'bop'       

    "

! !

!PhoneticStringUtilities class methodsFor:'queries'!

isUtilityClass

    ^ self == PhoneticStringUtilities

! !

!PhoneticStringUtilities::PhoneticStringComparator class methodsFor:'constant'!

defaultClass

	^SoundexStringComparator

! !

!PhoneticStringUtilities::PhoneticStringComparator class methodsFor:'documentation'!

documentation

"

    abstract superclass for various phonetic comparators.

    They returns similar strings for similar sounding words, which can be used

    to find similar sounding words in a search list.

    Notice, that some comparators are better for particular languages.

"

! !

!PhoneticStringUtilities::PhoneticStringComparator class methodsFor:'instance creation'!

new

    ^ self basicNew initialize.

! !

!PhoneticStringUtilities::PhoneticStringComparator class methodsFor:'queries'!

isAbstract

    ^ self == PhoneticStringUtilities::PhoneticStringComparator

! !

!PhoneticStringUtilities::PhoneticStringComparator methodsFor:'api'!

does:aString soundLike:anotherString 

    |translations1 translations2|

    translations1 := self phoneticStringsFor:aString.

    translations2 := self phoneticStringsFor:anotherString.

    ^ translations1 contains:[:t1 | 

        translations2 contains:[:t2 | t1 = t2]]

    "

     PhoneticStringUtilities::SoundexStringComparator new

            does:'miller' soundLike:'miler'.   

     PhoneticStringUtilities::SoundexStringComparator new

            does:'miller' soundLike:'milner'.   

    "

!

phoneticStringsFor: aString

    "Should answer an array of alternate phonetic strings for the given input string."

    self subclassResponsibility

    "

     (PhoneticStringUtilities::SoundexStringComparator new

            phoneticStringsFor:'miller') first      

     'miller' asSoundexCode 

    "

! !

!PhoneticStringUtilities::PhoneticStringComparator methodsFor:'initialization'!

initialize

    "Invoked when a new instance is created."

    "/ please change as required (and remove this comment)

    "/ super initialize.   -- commented since inherited method does nothing

! !

!PhoneticStringUtilities::ExtendedSoundexStringComparator class methodsFor:'documentation'!

documentation

"

    There are many extended and enhanced soundex variants around;

    here is one, called 'extended soundex'. It is destribed for example in

    http://www.epidata.dk/documentation.php.

    An author or origin is unknown.

    The number of digits is increased to 5 or 8;

    The first character is not used literally; instead it is encoded like the rest.

    This might have a negative effect on names starting with a vovel, though.

    Overall, it can be doubted if this is really an enhancement after all.

"

! !

!PhoneticStringUtilities::ExtendedSoundexStringComparator methodsFor:'api'!

phoneticStringsFor:aString

    "generates both an extended soundex of length 5 and one of length 8"

    |first second u t prevCode|

    u := aString asUppercase.

    first := second := ''.

    u do:[:c | 

        t := self translate:c.

        (t notNil and:[ t ~= '0' and:[ t ~= prevCode ]]) ifTrue:[

            first := first , t.

            second := second , t.

            second size == 8 ifTrue:[

                ^ Array with:(first copyTo:5) with:second 

            ].

        ].

        prevCode := t

    ].

    [ first size < 5 ] whileTrue:[

        first := first , '0'.

        second := second , '0'.

    ].

    [ second size < 8 ] whileTrue:[

        second := second , '0'

    ].

    ^ Array with:first with:second

    "

     self basicNew phoneticStringsFor:'müller'  #('87900' '87900000')  

     self basicNew phoneticStringsFor:'miller'  #('87900' '87900000')   

     self basicNew phoneticStringsFor:'muller'  #('87900' '87900000')    

     self basicNew phoneticStringsFor:'muler'   #('87900' '87900000')

     self basicNew phoneticStringsFor:'schmidt'    #('38600' '38600000')

     self basicNew phoneticStringsFor:'schneider'  #('38690' '38690000')

     self basicNew phoneticStringsFor:'fischer'    #('23900' '23900000')

     self basicNew phoneticStringsFor:'weber'      #('19000' '19000000')

     self basicNew phoneticStringsFor:'meyer'      #('89000' '89000000')

     self basicNew phoneticStringsFor:'wagner'     #('48900' '48900000')

     self basicNew phoneticStringsFor:'schulz'     #('37500' '37500000')

     self basicNew phoneticStringsFor:'becker'     #('13900' '13900000')

     self basicNew phoneticStringsFor:'hoffmann'   #('28800' '28800000')

     self basicNew phoneticStringsFor:'schäfer'    #('32900' '32900000')

    "

! !

!PhoneticStringUtilities::ExtendedSoundexStringComparator methodsFor:'private'!

translate:aCharacter