Added support for skipping tests.
To skip a test, simply add
BenchmarkSkipRequest signal
to setup or benchmark method. This may be used to skip
benchmarks that are not runnable for whatever reason.
--- a/s/BenchmarkExecutor.st Wed Dec 17 01:17:34 2014 +0000
+++ b/s/BenchmarkExecutor.st Mon Apr 20 12:53:16 2015 +0100
@@ -1,5 +1,7 @@
"{ Package: 'jv:calipel/s' }"
+"{ NameSpace: Smalltalk }"
+
Object subclass:#BenchmarkExecutor
instanceVariableNames:'instruments'
classVariableNames:''
@@ -255,39 +257,41 @@
"
Takes a benchmark instance and a set of parameter defines,
executes the benchmark and an outcome to the result. Returns that
- outcome
+ outcome or nil if benchmark has been skipped.
This is where real execution happens"
| measurements outcome |
- "First, warm it up"
- [
- self setUp:aBenchmarkInstance parameters: aCollection.
- self warmUp: aBenchmarkInstance.
- ] ensure:[
- self tearDown: aBenchmarkInstance
- ].
-
- measurements := (1 to: aBenchmarkResult runs) collect:[:i |
- [
+ [
+ "First, warm it up"
+ [
self setUp:aBenchmarkInstance parameters: aCollection.
- self benchmark: aBenchmarkInstance
+ self warmUp: aBenchmarkInstance.
] ensure:[
self tearDown: aBenchmarkInstance
].
+
+ measurements := (1 to: aBenchmarkResult runs) collect:[:i |
+ [
+ self setUp:aBenchmarkInstance parameters: aCollection.
+ self benchmark: aBenchmarkInstance
+ ] ensure:[
+ self tearDown: aBenchmarkInstance
+ ].
+ ].
+ aBenchmarkResult addOutcome:
+ (outcome := BenchmarkOutcome
+ benchmark: aBenchmarkInstance
+ parameters: aCollection
+ measurements: measurements).
+ ] on: BenchmarkSkipRequest do:[:ex |
+ outcome := nil.
].
-
- aBenchmarkResult addOutcome:
- (outcome := BenchmarkOutcome
- benchmark: aBenchmarkInstance
- parameters: aCollection
- measurements: measurements).
-
^ outcome
"Created: / 27-07-2013 / 12:32:36 / Jan Vrany <jan.vrany@fit.cvut.cz>"
- "Modified: / 24-11-2014 / 06:54:13 / Jan Vrany <jan.vrany@fit.cvut.cz>"
+ "Modified (comment): / 20-04-2015 / 12:44:15 / Jan Vrany <jan.vrany@fit.cvut.cz>"
!
setUp:aBenchmarkInstance parameters: aCollection
@@ -338,11 +342,15 @@
[
aBenchmarkInstance warmUp.
] on: Error do:[:ex|
- BenchmarkExecutionError new signal:'Error during warm-up: ', ex description.
+ (ex isKindOf: BenchmarkError) ifTrue:[
+ ex pass
+ ] ifFalse:[
+ BenchmarkExecutionError new signal:'Error during warm-up: ', ex description.
+ ]
]
"Created: / 24-06-2013 / 01:11:40 / Jan Vrany <jan.vrany@fit.cvut.cz>"
- "Modified: / 31-07-2013 / 01:04:06 / Jan Vrany <jan.vrany@fit.cvut.cz>"
+ "Modified: / 20-04-2015 / 12:45:29 / Jan Vrany <jan.vrany@fit.cvut.cz>"
! !
!BenchmarkExecutor methodsFor:'initialization'!
--- a/s/BenchmarkRunnerExecutor.st Wed Dec 17 01:17:34 2014 +0000
+++ b/s/BenchmarkRunnerExecutor.st Mon Apr 20 12:53:16 2015 +0100
@@ -67,20 +67,25 @@
transcript nextPutAll: '...'.
[
- outcome := super execute: aBenchmarkInstance result: aBenchmarkResult parameters: aCollection.
- transcript nextPutAll: 'OK ['.
- transcript nextPutAll: outcome time printString.
- transcript nextPutAll: ']'.
- aCollection notEmpty ifTrue:[
- transcript nextPutAll: ' {'.
- (aCollection asSortedCollection:[:a :b | a key name < b key name ]) do:[:paramAndValue|
- transcript nextPutAll: paramAndValue key name.
- transcript nextPutAll: '='.
- transcript nextPutAll: paramAndValue value storeString.
- ] separatedBy:[
- transcript nextPutAll: ', '.
+ outcome := super execute: aBenchmarkInstance result: aBenchmarkResult
+ parameters: aCollection.
+ outcome notNil ifTrue:[
+ transcript nextPutAll: 'OK ['.
+ transcript nextPutAll: outcome time printString.
+ transcript nextPutAll: ']'.
+ aCollection notEmpty ifTrue:[
+ transcript nextPutAll: ' {'.
+ (aCollection asSortedCollection:[:a :b | a key name < b key name ]) do:[:paramAndValue|
+ transcript nextPutAll: paramAndValue key name.
+ transcript nextPutAll: '='.
+ transcript nextPutAll: paramAndValue value storeString.
+ ] separatedBy:[
+ transcript nextPutAll: ', '.
+ ].
+ transcript nextPutAll: '}'.
].
- transcript nextPutAll: '}'.
+ ] ifFalse:[
+ transcript nextPutAll: 'SKIPPED'.
].
transcript nextPutAll: '
'.
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/s/BenchmarkSkipRequest.st Mon Apr 20 12:53:16 2015 +0100
@@ -0,0 +1,36 @@
+"{ Package: 'jv:calipel/s' }"
+
+"{ NameSpace: Smalltalk }"
+
+BenchmarkError subclass:#BenchmarkSkipRequest
+ instanceVariableNames:''
+ classVariableNames:''
+ poolDictionaries:''
+ category:'CalipeL-S-Core-Exceptions'
+!
+
+!BenchmarkSkipRequest class methodsFor:'documentation'!
+
+documentation
+"
+ When thrown during benchmark execution (including it's set up), the benchmark
+ is sinlently skipped. Use this to skip unsupported or otherwise not runnable benchmarks.
+ To skip a benchmark, simply add
+
+ BenchmarkSkipRequest signal
+
+ to set-up or benchmark method. You may want to do this conditionally
+ based on whether a benchmark is runnable or not.
+
+ [author:]
+ Jan Vrany <jan.vrany@fit.cvut.cz>
+
+ [instance variables:]
+
+ [class variables:]
+
+ [see also:]
+
+"
+! !
+
--- a/s/Make.proto Wed Dec 17 01:17:34 2014 +0000
+++ b/s/Make.proto Mon Apr 20 12:53:16 2015 +0100
@@ -106,7 +106,7 @@
# build all packages containing referenced classes for this package
-# they are nor needed to compile the package
+# they are not needed to compile the package (but later, to load it)
references:
@@ -144,6 +144,7 @@
$(OUTDIR)BenchmarkReportJSON.$(O) BenchmarkReportJSON.$(H): BenchmarkReportJSON.st $(INCLUDE_TOP)/jv/calipel/s/BenchmarkReport.$(H) $(INCLUDE_TOP)/stx/libbasic/Object.$(H) $(STCHDR)
$(OUTDIR)BenchmarkReportText.$(O) BenchmarkReportText.$(H): BenchmarkReportText.st $(INCLUDE_TOP)/jv/calipel/s/BenchmarkReport.$(H) $(INCLUDE_TOP)/stx/libbasic/Object.$(H) $(STCHDR)
$(OUTDIR)BenchmarkRunnerExecutor.$(O) BenchmarkRunnerExecutor.$(H): BenchmarkRunnerExecutor.st $(INCLUDE_TOP)/jv/calipel/s/BenchmarkExecutor.$(H) $(INCLUDE_TOP)/stx/libbasic/Object.$(H) $(STCHDR)
+$(OUTDIR)BenchmarkSkipRequest.$(O) BenchmarkSkipRequest.$(H): BenchmarkSkipRequest.st $(INCLUDE_TOP)/jv/calipel/s/BenchmarkError.$(H) $(INCLUDE_TOP)/stx/libbasic/Error.$(H) $(INCLUDE_TOP)/stx/libbasic/Exception.$(H) $(INCLUDE_TOP)/stx/libbasic/GenericException.$(H) $(INCLUDE_TOP)/stx/libbasic/Object.$(H) $(STCHDR)
# ENDMAKEDEPEND --- do not remove this line
--- a/s/Make.spec Wed Dec 17 01:17:34 2014 +0000
+++ b/s/Make.spec Mon Apr 20 12:53:16 2015 +0100
@@ -74,6 +74,7 @@
BenchmarkReportJSON \
BenchmarkReportText \
BenchmarkRunnerExecutor \
+ BenchmarkSkipRequest \
@@ -102,6 +103,7 @@
$(OUTDIR_SLASH)BenchmarkReportJSON.$(O) \
$(OUTDIR_SLASH)BenchmarkReportText.$(O) \
$(OUTDIR_SLASH)BenchmarkRunnerExecutor.$(O) \
+ $(OUTDIR_SLASH)BenchmarkSkipRequest.$(O) \
--- a/s/abbrev.stc Wed Dec 17 01:17:34 2014 +0000
+++ b/s/abbrev.stc Mon Apr 20 12:53:16 2015 +0100
@@ -24,3 +24,4 @@
BenchmarkReportJSON BenchmarkReportJSON jv:calipel/s 'CalipeL-S-Core-Reports' 0
BenchmarkReportText BenchmarkReportText jv:calipel/s 'CalipeL-S-Core-Reports' 0
BenchmarkRunnerExecutor BenchmarkRunnerExecutor jv:calipel/s 'CalipeL-S-Core-Runner' 0
+BenchmarkSkipRequest BenchmarkSkipRequest jv:calipel/s 'CalipeL-S-Core-Exceptions' 1
--- a/s/bc.mak Wed Dec 17 01:17:34 2014 +0000
+++ b/s/bc.mak Mon Apr 20 12:53:16 2015 +0100
@@ -91,6 +91,7 @@
$(OUTDIR)BenchmarkReportJSON.$(O) BenchmarkReportJSON.$(H): BenchmarkReportJSON.st $(INCLUDE_TOP)\jv\calipel\s\BenchmarkReport.$(H) $(INCLUDE_TOP)\stx\libbasic\Object.$(H) $(STCHDR)
$(OUTDIR)BenchmarkReportText.$(O) BenchmarkReportText.$(H): BenchmarkReportText.st $(INCLUDE_TOP)\jv\calipel\s\BenchmarkReport.$(H) $(INCLUDE_TOP)\stx\libbasic\Object.$(H) $(STCHDR)
$(OUTDIR)BenchmarkRunnerExecutor.$(O) BenchmarkRunnerExecutor.$(H): BenchmarkRunnerExecutor.st $(INCLUDE_TOP)\jv\calipel\s\BenchmarkExecutor.$(H) $(INCLUDE_TOP)\stx\libbasic\Object.$(H) $(STCHDR)
+$(OUTDIR)BenchmarkSkipRequest.$(O) BenchmarkSkipRequest.$(H): BenchmarkSkipRequest.st $(INCLUDE_TOP)\jv\calipel\s\BenchmarkError.$(H) $(INCLUDE_TOP)\stx\libbasic\Error.$(H) $(INCLUDE_TOP)\stx\libbasic\Exception.$(H) $(INCLUDE_TOP)\stx\libbasic\GenericException.$(H) $(INCLUDE_TOP)\stx\libbasic\Object.$(H) $(STCHDR)
# ENDMAKEDEPEND --- do not remove this line
--- a/s/bmake.bat Wed Dec 17 01:17:34 2014 +0000
+++ b/s/bmake.bat Mon Apr 20 12:53:16 2015 +0100
@@ -12,29 +12,29 @@
@echo "***********************************"
@echo "Buildung jv/calipel/s/stx
@echo "***********************************"
-@cd stx
-@call bmake %1 %2
-@cd ..
+@pushd stx
+@call bmake %1 %2 || exit /b "%errorlevel%"
+@popd
@echo "***********************************"
@echo "Buildung jv/calipel/s/tests
@echo "***********************************"
-@cd tests
-@call bmake %1 %2
-@cd ..
+@pushd tests
+@call bmake %1 %2 || exit /b "%errorlevel%"
+@popd
@echo "***********************************"
@echo "Buildung jv/calipel/s/benchmarks/micro
@echo "***********************************"
-@cd benchmarks\micro
-@call bmake %1 %2
-@cd ..\..
+@pushd benchmarks\micro
+@call bmake %1 %2 || exit /b "%errorlevel%"
+@popd
@echo "***********************************"
@echo "Buildung jv/calipel/s/benchmarks/game
@echo "***********************************"
-@cd benchmarks\game
-@call bmake %1 %2
-@cd ..\..
+@pushd benchmarks\game
+@call bmake %1 %2 || exit /b "%errorlevel%"
+@popd
--- a/s/jv_calipel_s.st Wed Dec 17 01:17:34 2014 +0000
+++ b/s/jv_calipel_s.st Mon Apr 20 12:53:16 2015 +0100
@@ -1,5 +1,7 @@
"{ Package: 'jv:calipel/s' }"
+"{ NameSpace: Smalltalk }"
+
LibraryDefinition subclass:#jv_calipel_s
instanceVariableNames:''
classVariableNames:''
@@ -48,12 +50,15 @@
referencedPreRequisites
"list packages which are a prerequisite, because they contain
classes which are referenced by my classes.
- We do not need these packages as a prerequisite for loading or compiling.
+ We do not need these packages as a prerequisite for compiling or loading,
+ however, a class from it may be referenced during execution and having it
+ unloaded then may lead to a runtime doesNotUnderstand error, unless the caller
+ includes explicit checks for the package being present.
This method is generated automatically,
by searching all classes (and their packages) which are referenced by my classes."
^ #(
- #'stx:libbasic3' "MessageTally - referenced by BenchmarkInstance>>spyIt"
+ #'stx:libbasic3' "MessageTally - referenced by BenchmarkExecutor>>spy:"
#'stx:libcompat' "DateAndTime - referenced by BenchmarkResult>>initializeTimestamp"
)
!
@@ -115,6 +120,7 @@
BenchmarkReportJSON
BenchmarkReportText
BenchmarkRunnerExecutor
+ BenchmarkSkipRequest
)
!
--- a/s/lccmake.bat Wed Dec 17 01:17:34 2014 +0000
+++ b/s/lccmake.bat Mon Apr 20 12:53:16 2015 +0100
@@ -8,29 +8,29 @@
@echo "***********************************"
@echo "Buildung jv/calipel/s/stx
@echo "***********************************"
-@cd stx
-@call lccmake %1 %2
-@cd ..
+@pushd stx
+@call lccmake %1 %2 || exit /b "%errorlevel%"
+@popd
@echo "***********************************"
@echo "Buildung jv/calipel/s/tests
@echo "***********************************"
-@cd tests
-@call lccmake %1 %2
-@cd ..
+@pushd tests
+@call lccmake %1 %2 || exit /b "%errorlevel%"
+@popd
@echo "***********************************"
@echo "Buildung jv/calipel/s/benchmarks/micro
@echo "***********************************"
-@cd benchmarks\micro
-@call lccmake %1 %2
-@cd ..\..
+@pushd benchmarks\micro
+@call lccmake %1 %2 || exit /b "%errorlevel%"
+@popd
@echo "***********************************"
@echo "Buildung jv/calipel/s/benchmarks/game
@echo "***********************************"
-@cd benchmarks\game
-@call lccmake %1 %2
-@cd ..\..
+@pushd benchmarks\game
+@call lccmake %1 %2 || exit /b "%errorlevel%"
+@popd
--- a/s/libInit.cc Wed Dec 17 01:17:34 2014 +0000
+++ b/s/libInit.cc Mon Apr 20 12:53:16 2015 +0100
@@ -50,6 +50,7 @@
_BenchmarkReportJSON_Init(pass,__pRT__,snd);
_BenchmarkReportText_Init(pass,__pRT__,snd);
_BenchmarkRunnerExecutor_Init(pass,__pRT__,snd);
+_BenchmarkSkipRequest_Init(pass,__pRT__,snd);
__END_PACKAGE__();
--- a/s/mingwmake.bat Wed Dec 17 01:17:34 2014 +0000
+++ b/s/mingwmake.bat Mon Apr 20 12:53:16 2015 +0100
@@ -16,29 +16,29 @@
@echo "***********************************"
@echo "Buildung jv/calipel/s/stx
@echo "***********************************"
-@cd stx
-@call mingwmake %1 %2
-@cd ..
+@pushd stx
+@call mingwmake %1 %2 || exit /b "%errorlevel%"
+@popd
@echo "***********************************"
@echo "Buildung jv/calipel/s/tests
@echo "***********************************"
-@cd tests
-@call mingwmake %1 %2
-@cd ..
+@pushd tests
+@call mingwmake %1 %2 || exit /b "%errorlevel%"
+@popd
@echo "***********************************"
@echo "Buildung jv/calipel/s/benchmarks/micro
@echo "***********************************"
-@cd benchmarks\micro
-@call mingwmake %1 %2
-@cd ..\..
+@pushd benchmarks\micro
+@call mingwmake %1 %2 || exit /b "%errorlevel%"
+@popd
@echo "***********************************"
@echo "Buildung jv/calipel/s/benchmarks/game
@echo "***********************************"
-@cd benchmarks\game
-@call mingwmake %1 %2
-@cd ..\..
+@pushd benchmarks\game
+@call mingwmake %1 %2 || exit /b "%errorlevel%"
+@popd
--- a/s/stx/Make.proto Wed Dec 17 01:17:34 2014 +0000
+++ b/s/stx/Make.proto Mon Apr 20 12:53:16 2015 +0100
@@ -107,7 +107,7 @@
# build all packages containing referenced classes for this package
-# they are nor needed to compile the package
+# they are not needed to compile the package (but later, to load it)
references:
--- a/s/stx/jv_calipel_s_stx.st Wed Dec 17 01:17:34 2014 +0000
+++ b/s/stx/jv_calipel_s_stx.st Mon Apr 20 12:53:16 2015 +0100
@@ -1,5 +1,7 @@
"{ Package: 'jv:calipel/s/stx' }"
+"{ NameSpace: Smalltalk }"
+
LibraryDefinition subclass:#jv_calipel_s_stx
instanceVariableNames:''
classVariableNames:''
@@ -35,7 +37,10 @@
referencedPreRequisites
"list packages which are a prerequisite, because they contain
classes which are referenced by my classes.
- We do not need these packages as a prerequisite for loading or compiling.
+ We do not need these packages as a prerequisite for compiling or loading,
+ however, a class from it may be referenced during execution and having it
+ unloaded then may lead to a runtime doesNotUnderstand error, unless the caller
+ includes explicit checks for the package being present.
This method is generated automatically,
by searching all classes (and their packages) which are referenced by my classes."
--- a/s/tests/BenchmarkInstanceTestsA.st Wed Dec 17 01:17:34 2014 +0000
+++ b/s/tests/BenchmarkInstanceTestsA.st Mon Apr 20 12:53:16 2015 +0100
@@ -1,5 +1,7 @@
"{ Package: 'jv:calipel/s/tests' }"
+"{ NameSpace: Smalltalk }"
+
TestCase subclass:#BenchmarkInstanceTestsA
instanceVariableNames:'log param1 param2 measurementValue'
classVariableNames:''
@@ -88,6 +90,32 @@
log add:#'fakeTearDown2'
"Created: / 09-03-2014 / 22:52:38 / Jan Vrany <jan.vrany@fit.cvut.cz>"
+!
+
+skipped01
+ <benchmark>
+
+ BenchmarkSkipRequest signal.
+ log add:#'skipped01'
+
+ "Created: / 20-04-2015 / 12:36:19 / Jan Vrany <jan.vrany@fit.cvut.cz>"
+!
+
+skipped02
+ <setup: #skippedSetup02>
+ <benchmark>
+
+
+ log add:#'skipped01'
+
+ "Created: / 20-04-2015 / 12:40:40 / Jan Vrany <jan.vrany@fit.cvut.cz>"
+!
+
+skippedSetup02
+ BenchmarkSkipRequest signal.
+ log add:#'skippedSetup02'
+
+ "Created: / 20-04-2015 / 12:40:54 / Jan Vrany <jan.vrany@fit.cvut.cz>"
! !
!BenchmarkInstanceTestsA methodsFor:'instrument mimicry'!
@@ -390,6 +418,34 @@
String streamContents: [:s | BenchmarkReport text write: r on: s ]
"Created: / 01-12-2014 / 02:57:47 / Jan Vrany <jan.vrany@fit.cvut.cz>"
+!
+
+test_08a
+ "Tests benchmark skip"
+
+ | b r |
+
+ measurementValue := BenchmarkMeasurementValueNotAvailable instance.
+ b := BenchmarkInstance new instance:self selector:#'skipped01'.
+ r := b run.
+
+ self assert: r outcomes isEmpty.
+
+ "Created: / 20-04-2015 / 12:38:01 / Jan Vrany <jan.vrany@fit.cvut.cz>"
+!
+
+test_08b
+ "Tests benchmark skip"
+
+ | b r |
+
+ measurementValue := BenchmarkMeasurementValueNotAvailable instance.
+ b := BenchmarkInstance new instance:self selector:#'skipped02'.
+ r := b run.
+
+ self assert: r outcomes isEmpty.
+
+ "Created: / 20-04-2015 / 12:41:04 / Jan Vrany <jan.vrany@fit.cvut.cz>"
! !
!BenchmarkInstanceTestsA class methodsFor:'documentation'!
--- a/s/tests/BenchmarkRunnerTests.st Wed Dec 17 01:17:34 2014 +0000
+++ b/s/tests/BenchmarkRunnerTests.st Mon Apr 20 12:53:16 2015 +0100
@@ -1,5 +1,7 @@
"{ Package: 'jv:calipel/s/tests' }"
+"{ NameSpace: Smalltalk }"
+
TestCase subclass:#BenchmarkRunnerTests
instanceVariableNames:''
classVariableNames:''
--- a/s/tests/Make.proto Wed Dec 17 01:17:34 2014 +0000
+++ b/s/tests/Make.proto Mon Apr 20 12:53:16 2015 +0100
@@ -111,7 +111,7 @@
# build all packages containing referenced classes for this package
-# they are nor needed to compile the package
+# they are not needed to compile the package (but later, to load it)
references:
--- a/s/tests/jv_calipel_s_tests.st Wed Dec 17 01:17:34 2014 +0000
+++ b/s/tests/jv_calipel_s_tests.st Mon Apr 20 12:53:16 2015 +0100
@@ -1,5 +1,7 @@
"{ Package: 'jv:calipel/s/tests' }"
+"{ NameSpace: Smalltalk }"
+
LibraryDefinition subclass:#jv_calipel_s_tests
instanceVariableNames:''
classVariableNames:''
@@ -35,7 +37,10 @@
referencedPreRequisites
"list packages which are a prerequisite, because they contain
classes which are referenced by my classes.
- We do not need these packages as a prerequisite for loading or compiling.
+ We do not need these packages as a prerequisite for compiling or loading,
+ however, a class from it may be referenced during execution and having it
+ unloaded then may lead to a runtime doesNotUnderstand error, unless the caller
+ includes explicit checks for the package being present.
This method is generated automatically,
by searching all classes (and their packages) which are referenced by my classes."
--- a/s/vcmake.bat Wed Dec 17 01:17:34 2014 +0000
+++ b/s/vcmake.bat Mon Apr 20 12:53:16 2015 +0100
@@ -20,29 +20,29 @@
@echo "***********************************"
@echo "Buildung jv/calipel/s/stx
@echo "***********************************"
-@cd stx
-@call vcmake %1 %2
-@cd ..
+@pushd stx
+@call vcmake %1 %2 || exit /b "%errorlevel%"
+@popd
@echo "***********************************"
@echo "Buildung jv/calipel/s/tests
@echo "***********************************"
-@cd tests
-@call vcmake %1 %2
-@cd ..
+@pushd tests
+@call vcmake %1 %2 || exit /b "%errorlevel%"
+@popd
@echo "***********************************"
@echo "Buildung jv/calipel/s/benchmarks/micro
@echo "***********************************"
-@cd benchmarks\micro
-@call vcmake %1 %2
-@cd ..\..
+@pushd benchmarks\micro
+@call vcmake %1 %2 || exit /b "%errorlevel%"
+@popd
@echo "***********************************"
@echo "Buildung jv/calipel/s/benchmarks/game
@echo "***********************************"
-@cd benchmarks\game
-@call vcmake %1 %2
-@cd ..\..
+@pushd benchmarks\game
+@call vcmake %1 %2 || exit /b "%errorlevel%"
+@popd