"{ Package: 'stx:goodies/sunit' }"

Object subclass:#TestResultReporter

	instanceVariableNames:'result stream'

	classVariableNames:''

	poolDictionaries:''

!

!TestResultReporter class methodsFor:'documentation'!

documentation

"

    this is used as a last step in jenkins automated builds.

    WARNING:

        this is about to be obsoleted by the reporter classes found in the

        goodies/builder/ packages.

    Currently supported formats are:

        #xml_jUnit      - a junit-like xml format

        #xml_pythonUnit - a python unit-like xml format

        #xml            - same, for backward compatibility

        #tap            - perl TAP unit test format; 

                          very naive and simple, but there are tools for it...

        #xml_perfPublisher

                        - can be processed by the perfPublisher jenkins/hudson plugin

    public API entry:

        |aTestResult|

        aTestResult := aUnitTest suite run.

        TestResultReporter report:aTestResult format:#xml on: aStream

    modifications:

        cg: I think the name 'xml' is too unspecific; 

            I may want to add a whole bunch of additional xml-based formats. 

            So I changed #xml to xml_pythonUnit (but still support the original #xml for bw-compatibility).

            You may be especially interested in xml_perfPublisher, which is great to present

            nice reports in jenkins/hudson...

    [author:]

        Jan Vranji

        documentation, tap & perfPublisher format added by Claus Gittinger

    [see also:]

        TAP 

            http://testanything.org/wiki/index.php/Main_Page

            http://search.cpan.org/~petdance/Test-Harness-2.64/lib/Test/Harness/TAP.pod

        perfPublisher

            http://wiki.hudson-ci.org/display/HUDSON/PerfPublisher+Plugin

"

!

examples

"

    Smalltalk loadPackage:'exept:regression'

                                                                               [exBegin]

    |suite result testResult|

    suite := TestSuite named:'SelfTest'.

    #(

        'JavaScriptTests'

        'FloatTest'

        'IntegerTest'

    ) do:[:className |

        |fullName|

        fullName := ('RegressionTests::',className).

        suite addTest:(Smalltalk classNamed:fullName) suite.

    ].

    testResult := suite run.                                                                           

    TestResultReporter report:testResult format:#xml_jUnit on:Transcript. 

                                                                               [exEnd]

                                                                               [exBegin]

    |testResult|

    testResult := RegressionTests::IntegerTest suite run.

    TestResultReporter report:testResult format:#xml_jUnit on:Transcript. 

                                                                               [exEnd]

                                                                               [exBegin]

    |testResult|

    testResult := RegressionTests::IntegerTest suite run.

    TestResultReporter report:testResult format:#xml_pythonUnit on:Transcript. 

                                                                               [exEnd]

                                                                               [exBegin]

    |testResult|

    testResult := RegressionTests::IntegerTest suite run.

    TestResultReporter report:testResult format:#tap on:Transcript.      

                                                                               [exEnd]

                                                                               [exBegin]

    |testResult|

    testResult := RegressionTests::CollectionTests suite run.

    TestResultReporter report:testResult format:#xml_perfPublisher on:Transcript.

                                                                               [exEnd]

                                                                               [exBegin]

    |suite testResult|

    suite := RegressionTests::WritingToTranscriptOrStdoutTest suite.

    testResult := suite run.

    TestResultReporter report:testResult format:#xml_jUnit on:Transcript.

                                                                               [exEnd]

                                                                               [exBegin]

    |testResult|

    testResult := RegressionTests::NumberTest suite run.

    TestResultReporter report:testResult format:#xml_jUnit on:Transcript.

                                                                               [exEnd]

                                                                               [exBegin]

    |testResult|

    testResult := RegressionTests::NumberTest suite run.

    TestResultReporter report:testResult format:#xml_pythonUnit on:Transcript.

                                                                               [exEnd]

                                                                               [exBegin]

    |testResult|

    testResult := RegressionTests::NumberTest suite run.

    TestResultReporter report:testResult format:#xml_perfPublisher on:Transcript.

                                                                               [exEnd]

                                                                               [exBegin]

    |testResult|

    testResult := RegressionTests::NumberTest suite run.

    TestResultReporter report:testResult format:#tap on:Transcript.

                                                                               [exEnd]

  test an error:

                                                                               [exBegin]

    |testResult|

    (RegressionTests::AssociationTests includesSelector:#'test_xx_willFail') ifFalse:[

        Class withoutUpdatingChangesDo:[

            RegressionTests::AssociationTests compile:'test_xx_willFail self assert:5 > 7.'.

            RegressionTests::AssociationTests compile:'test_xx_willErr self error:''err by purpose''.'

        ].

    ].

    testResult := RegressionTests::AssociationTests suite run.

    TestResultReporter report:testResult format:#xml_jUnit on:Transcript.

                                                                               [exEnd]

"

!

format_tap

"

    sample output for one of the st/x regression-tests looks like:

1..49

ok 1 - RegressionTests::IntegerTest-testComparing (0ms)

ok 2 - RegressionTests::IntegerTest-testConstants (0ms)

ok 3 - RegressionTests::IntegerTest-testCreationFromBytes1 (0ms)

ok 4 - RegressionTests::IntegerTest-testDivision (0ms)

ok 5 - RegressionTests::IntegerTest-testEncodeDecode (0ms)

ok 6 - RegressionTests::IntegerTest-testFactorial (0ms)

ok 7 - RegressionTests::IntegerTest-testGCD (0ms)

ok 8 - RegressionTests::IntegerTest-testILC (0ms)

ok 9 - RegressionTests::IntegerTest-testInline1 (0ms)

ok 10 - RegressionTests::IntegerTest-testInteger1 (0ms)

ok 11 - RegressionTests::IntegerTest-testIntegerMisc (0ms)

...

ok 47 - RegressionTests::IntegerTest-test_gcdBug1 (0ms)

ok 48 - RegressionTests::IntegerTest-test_gcdBug2 (0ms)

ok 49 - RegressionTests::IntegerTest-test_gcdBug3 (0ms)

"

!

format_xml_jUnit

"

    sample output for one of the st/x regression-tests looks like:

   <?xml version='1.0' encoding='UTF-8' ?>

   <testsuite errors='1' failures='0' hostname='hazelnut.osuosl.org' name='net.cars.engine.BougieTest' tests='2' time='0.017' timestamp='2007-11-02T23:13:50'>

     <properties>

       <property name='java.vendor' value='IBM Corporation' />

       <property name='os.name' value='Linux' />

       <property name='sun.boot.class.path' value='/opt/ibm-jdk-bin-1.5.0.5a/jre/lib/vm.jar:/opt/ibm-jdk-bin-1.5.0.5a/jre/lib/core.jar:/opt/ibm-jdk-bin-1.5.0.5a/jre/lib/charsets.jar:/opt/ibm-jdk-bin-1.5.0.5a/jre/lib/graphics.jar:/opt/ibm-jdk-bin-1.5.0.5a/jre/lib/security.jar:/opt/ibm-jdk-bin-1.5.0.5a/jre/lib/ibmpkcs.jar:/opt/ibm-jdk-bin-1.5.0.5a/jre/lib/ibmorb.jar:/opt/ibm-jdk-bin-1.5.0.5a/jre/lib/ibmcfw.jar:/opt/ibm-jdk-bin-1.5.0.5a/jre/lib/ibmorbapi.jar:/opt/ibm-jdk-bin-1.5.0.5a/jre/lib/ibmjcefw.jar:/opt/ibm-jdk-bin-1.5.0.5a/jre/lib/ibmjgssprovider.jar:/opt/ibm-jdk-bin-1.5.0.5a/jre/lib/ibmjsseprovider2.jar:/opt/ibm-jdk-bin-1.5.0.5a/jre/lib/ibmjaaslm.jar:/opt/ibm-jdk-bin-1.5.0.5a/jre/lib/ibmcertpathprovider.jar:/opt/ibm-jdk-bin-1.5.0.5a/jre/lib/server.jar:/opt/ibm-jdk-bin-1.5.0.5a/jre/lib/xml.jar' />

       <property name='sun.java2d.fontpath' value='' />

       <property name='java.vm.specification.vendor' value='Sun Microsystems Inc.' />

       <property name='ant.home' value='/home/jancumps/project/continuus/ant/distro' />

      ...

     </properties>

     <testcase classname='net.cars.engine.BougieTest' name='sparkDry' time='0.0010' />

     <testcase classname='net.cars.engine.BougieTest' name='sparkHumid' time='0.0050'>

       <error message='humidity level too high' type='java.lang.RuntimeException'>java.lang.RuntimeException: humidity level too high

          at net.cars.engine.Bougie.spark(Unknown Source)

          at net.cars.engine.BougieTest.sparkHumid(BougieTest.java:36)

       </error>

     </testcase>

     <system-out><!![CDATA[]]></system-out>

     <system-err><!![CDATA[]]></system-err>

  </testsuite>

"

!

format_xml_perfPublisher

"

    sample output looks like:

<report name='GeneratedReport-0' categ='GeneratedReport'>

  <start>

    <date format='YYYYMMDD' val='20000101' />

    <time format='HHMMSS' val='195043' />

  </start>

  <test name='/file_0/test_0' executed='yes'>

    <description><!![CDATA[This is the description of the test number 0 member of the 0 file.]]></description>

    <targets>

      <target threaded='false'>PHP</target>

    </targets>

    <platform name='Plateform_0' remote='unknown' capspool='unknown'>

      <os>

        <type><!![CDATA[Linux-2.6.26-2-amd64-x86_64-with-glibc2.3.2]]></type>

        <name><!![CDATA[Linux]]></name>

        <version><!![CDATA[2.6.26-2-amd64]]></version>

        <distribution><!![CDATA[Linux-2.6.26-2-amd64-x86_64-with-debian-5.0.3]]></distribution>

      </os>

      <processor arch='x86_64'>

        <frequency unit='MHz' cpufreq='2667.000' />

      </processor>

      <hardware><!![CDATA[nVidia Corporation GeForce 8500 GT (rev a1)]]></hardware>

      <hardware><!![CDATA[nVidia Corporation GT200 [Tesla C1060 / Tesla S1070] (rev a1)]]></hardware>

      <compiler name='ifort' version='11.0 20090131' path='.' />

      <environment>{PUT HERE ALL YOUR LIBS AND THE DESCRIPTION OF YOUR TEST ENVIRONMENT}</environment>

    </platform>

    <commandline rank='0' time='20100128-195406.590832' duration='0.599782943726'>the_first_command_executed</commandline>

    <commandline rank='1' time='20100128-195406.590832' duration='0.599782943726'>the_second_command_executed</commandline>

    <result>

      <success passed='no' state='70' hasTimedOut='false' />

      <compiletime unit='s' mesure='1.456467636167308' isRelevant='false' />

      <performance unit='GFLOPs' mesure='39.621634393187904' isRelevant='false' />

      <executiontime unit='s' mesure='1.5300645576307736' isRelevant='false' />

      <errorlog><!![CDATA[EXEMPLE OF ERROR LOG]]></errorlog>

      <log name='NameOfLogNumber0'><!![CDATA[Here it's another of other log]]></log>

      <log name='NameOfLogNumber1'><!![CDATA[Here it's another of other log]]></log>

    </result>

  </test>

  <test name='/file_0/test_1' executed='yes'>

    ...

    </result>

  </test>

</report>

"

!

format_xml_pythonUnit

"

    sample output for one of the st/x regression-tests looks like:

<?xml version='1.0'?>

<unittest-results>

<test duration='0'

    status='success'

    ficture='RegressionTests::IntegerTest'

    name='testComparing'

    file='RegressionTests::IntegerTest.st'>

</test>

<test duration='0'

    status='success'

    ficture='RegressionTests::IntegerTest'

    name='testConstants'

    file='RegressionTests::IntegerTest.st'>

</test>

<test duration='0'

    status='success'

    ficture='RegressionTests::IntegerTest'

    name='testCreationFromBytes1'

    file='RegressionTests::IntegerTest.st'>

</test>

<test duration='0'

    status='success'

    ficture='RegressionTests::IntegerTest'

    name='testDivision'

    file='RegressionTests::IntegerTest.st'>

</test>

<test duration='0'

    status='success'

    ficture='RegressionTests::IntegerTest'

    name='testEncodeDecode'

    file='RegressionTests::IntegerTest.st'>

</test>

...

</unittest-results>

"

! !

!TestResultReporter class methodsFor:'queries'!

supportedFormats

   "return a list of formats and short-info-string, as per supported format symbol"

    ^ #(

        (#'xml_jUnit'           'a junit-like xml format')

        (#'xml_pythonUnit'      'a python unit-like xml format')

        (#'xml_perfPublisher'   'xml-based format for jenkins/hudson')

        (#'tap'                 'perl TAP unit test format')

    )

    "Created: / 30-07-2011 / 10:18:18 / cg"

! !

!TestResultReporter class methodsFor:'reporting'!

report: aTestResult format: format as: stringOrFilename

    self new report: aTestResult format: format as: stringOrFilename

!

report: aTestResult format: format on: stream

    self new report: aTestResult format: format on: stream

    "

     self report:(RegressionTests::IntegerTest runTests) format:#xml on:Transcript

    "

    "Modified (comment): / 30-07-2011 / 09:37:53 / cg"

! !

!TestResultReporter methodsFor:'reporting'!

report:formatSymbol

    "currently supported formatSymbols:

            xml_pythonUnit, xml_perfPublisher, tap"

    |reportFormatSelector|

    reportFormatSelector := self reportFormatSelector:formatSymbol.

    (self respondsTo: reportFormatSelector)

        ifTrue:[self perform: reportFormatSelector]

        ifFalse:[self error:'Unsupported format: ', formatSymbol].

    "Modified (comment): / 03-08-2011 / 12:57:54 / cg"

!

report: aTestResult format: format as: stringOrFilename

    | s |

    s := stringOrFilename asFilename writeStream.

    [ self report: aTestResult format: format on: s]

        ensure:[s close].

!

report: aTestResult format: aSymbol on: aStream

    result := aTestResult.

    stream := aStream.

    self report: aSymbol

!

reportFormatSelector:format

    ^ ('report' , format asString capitalized) asSymbol

! !

!TestResultReporter methodsFor:'reporting - tap'!

reportTap

    "TAP (perl unit test) report format"

    |idx reportWithStatus|

    "example:

        1..4

        ok 1 - Input file opened

        not ok 2 - First line of the input valid

        ok 3 - Read the rest of the file

        not ok 4 - Summarized correctly # TODO Not written yet

    "

    "/ what about not-executed tests - why only runCount ?

    stream nextPutLine: ('1..%1' bindWith:result runCount).

    reportWithStatus := 

        [:testOutComes :status |

            testOutComes do:[:each | self reportTapTest: each index: idx result: status. idx := idx + 1 ]

        ].

    idx := 1.

    reportWithStatus value:result passedOutcomes value:#success.

    reportWithStatus value:result failureOutcomes value:#failure.

    reportWithStatus value:result errorOutcomes value:#error.

    "Created: / 30-07-2011 / 10:12:31 / cg"

!

reportTapTest: test index:index result: testResult

    | testClassName executionTime  testDescription statusString|

    testClassName := self sunitNameOf: test class.

    "most tests do not know, and return nil here!!"

    executionTime := test executionTime ? 0.0.

    testDescription := '%1-%2 (%3ms)'

                            bindWith:testClassName

                            with:test selector

                            with:executionTime.

    statusString := (testResult == #success)

                        ifTrue:['ok']

                        ifFalse:['not ok'].

    stream nextPutLine:('%1 %2 - %3'

                            bindWith:statusString

                            with:index

                            with:testDescription).

    "Created: / 30-07-2011 / 10:28:06 / cg"

! !

!TestResultReporter methodsFor:'reporting - xml-jUnit'!

errorMessageFromExceptionDetailOf:testOutcome

    |detail|

    detail := testOutcome exceptionDetail.

    detail isNil ifTrue:[

        ^ 'no error message'.

    ].

    "Smalltalk/X dialect detection..."

    ((Smalltalk respondsTo: #isSmalltalkX) and:[Smalltalk isSmalltalkX]) ifTrue:[ 

        "exception specific description string"

        ^ detail at:#description ifAbsent:[ (detail at:#exception) description]

    ].

    ^ 'no error message (no dialect specific extractor)'.

    "Created: / 07-08-2011 / 12:45:45 / cg"

!

errorTypeFromExceptionDetailOf:testOutcome

    |detail|

    detail := testOutcome exceptionDetail.

    detail isNil ifTrue:[

        ^ 'unknown error type'.

    ].

    "Smalltalk/X dialect detection..."

    ((Smalltalk respondsTo: #isSmalltalkX) and:[Smalltalk isSmalltalkX]) ifTrue:[

        |exception|

        "class name of the exception"

        exception := detail at:#exception.

        exception isBehavior ifTrue:[

            ^ exception name.

        ].

        ^ exception printString.

    ].

    ^ 'unknown error type (no dialect specific extractor)'.

    "Created: / 07-08-2011 / 13:06:44 / cg"

!

reportTracebackFromExceptionDetailOf:testOutcome

    |detail|

    detail := testOutcome exceptionDetail.

    detail notNil ifTrue:[

        "Smalltalk/X dialect detection..."

        ((Smalltalk respondsTo: #isSmalltalkX) and:[Smalltalk isSmalltalkX]) ifTrue:[ 

            stream nextPutAll:(detail at:#backtrace).

        ].

    ].

    "Created: / 07-08-2011 / 12:46:19 / cg"

!

reportXml_jUnit

    "jUnit-like XML unittest report format"

    stream

        nextPutLine: '<?xml version="1.0"?>';

        nextPutAll: '<testsuite';

        nextPutAll:(' tests="%1"' bindWith:result runCount);

        nextPutAll:(' timestamp="%1"' bindWith:result timestamp printStringIso8601Format);

        nextPutAll:(' time="%1"' bindWith:result executionTime);

        nextPutAll:(' errors="%1"' bindWith:result errors size);

        nextPutAll:(' failures="%1"' bindWith:result failures size).

    result skipped size > 0 ifTrue:[

        stream