--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/benchmarks/Rakefile Sat Feb 16 19:08:45 2013 +0100
@@ -0,0 +1,257 @@
+#require 'pry'
+
+require 'yaml'
+
+
+# DO NOT EDIT following list to temporarily run one benchmark for testing.
+# You can run it by:
+#
+# rake <bench>
+#
+# for instance:
+#
+# rake Jasper
+#
+# alternatively, to run just stx:libjava version, run it by:
+#
+# ./benchmark-runner.st -b Jasper -n 1
+#
+TESTS = [
+ # Order matters!
+ # Micro...
+ [ "Ackerman" , 8 ] ,
+ [ "Ary" , 100000 ] ,
+ [ "Hash" , 10000 ] ,
+ [ "Strcat" , 5000000 ] ,
+ [ "Native1" , 1000000 ] ,
+
+ # Macro...
+ [ "Groovy" , 5 ] ,
+ [ "Saxon" , 10 ] ,
+ # [ "Jasper" , 1 ] ,
+
+ # Interop...
+ [ "MethodInvocation" , 500000000 ] ,
+
+ [ "PrimitiveArguments" , 100000000 ] ,
+ [ "ObjectArguments" , 100000000 ] ,
+ [ "WrappedArguments" , 100000000 ] ,
+
+ [ "OverloadedMethods2" , 100000000 ] ,
+ [ "OverloadedMethods" , 100000000 ] ,
+
+ # Broken
+ # [ "CrossLanguageInvocation" , 500000000 ] ,
+ # [ "Threadring" , 100000 ] ,
+
+
+ [] # terminator, to make commenting tests easy!!!
+ ]
+
+RESULTS_LAST="result.txt"
+RESULTS_LAST_CSV="result.csv"
+RESULTS_LOG="results-log.txt"
+
+# Defines how many times given benchmark is run. The minimal value
+# is then taken.
+BENCHMARK_RUNS=5
+
+task :default => [:all]
+
+task :all => [:compile, :run]
+
+task :build => [ :compile ]
+
+task :compile do
+ Dir.chdir 'java' do
+ system 'ant'
+ end
+end
+
+task 'echo-classpath' do
+ puts "export CLASSPATH=#{classpath()}"
+end
+
+task :clean do
+ Dir.chdir 'java' do
+ system 'ant clean'
+ end
+ rm_f RESULTS_LAST
+end
+
+task :run do
+ results = Hash.new
+ TESTS.each do | spec |
+ if spec.size == 2
+ benchmark(spec[0], spec[1], results, BENCHMARK_RUNS)
+ end
+ end
+
+ results = Hash[results.sort]
+
+ write_results_txt(STDOUT, results)
+
+ if File.exist? RESULTS_LAST
+ File.delete(RESULTS_LAST)
+ end
+ if File.exist? RESULTS_LAST_CSV
+ File.delete(RESULTS_LAST_CSV)
+ end
+
+ write_results_txt_to_file(RESULTS_LAST, results)
+ write_results_csv_to_file(RESULTS_LAST_CSV, results)
+ write_results_txt_to_file(RESULTS_LOG, results)
+end
+
+# Generate tasks to run individual benchmarks
+TESTS.each do | spec |
+ if (spec.size == 2)
+ task spec[0] do
+ results = Hash.new
+ benchmark(spec[0], spec[1], results, BENCHMARK_RUNS)
+ write_results_txt(STDOUT, results)
+ end
+ end
+end
+
+
+# Run given benchmark
+def benchmark(test, passes, results, runs)
+ puts "Benchmarking #{test} (#{runs} passes)"
+ times = Hash.new
+ puts "export CLASSPATH=#{classpath()}"
+ ENV['CLASSPATH'] = classpath()
+
+ [:jvm, :stx, :libjava, :stx2libjava].each do | platform |
+ times[platform] = measure(test, passes, platform, runs)
+ end
+
+ results[test] = times
+end
+
+# Return a class path arguments to be passed to java/javac
+# (including '-cp') or an empty string
+def classpath()
+ return '/usr/share/java/groovy-all.jar:../libs/libs/saxon-9.1.0.8.jar:/usr/share/java/saxonb-9.1.0.8.jar:/usr/share/java/itext-2.1.7.jar:/usr/share/java/jasperreports.jar:/usr/share/java/commons-digester.jar:/usr/share/java/commons-logging.jar:/usr/share/java/commons-collections3.jar:/usr/share/java/commons-beanutils.jar:java/bin:.'
+end
+
+
+def measure(test, passes, platform, runs)
+ min = 999999999
+ i = 1;
+ while (i <= runs)
+ puts "Pass #{i}"
+ t = measure_single(test, passes, platform)
+ if (t == 'N/A')
+ min = t
+ break;
+ end
+ if (min > t)
+ min = t
+ end
+ i = i + 1
+ end
+ return min
+end
+
+# Runs a given bench on given platform and return the time.
+# If bench fails (non-zero status value), raise an exception.
+def measure_single(test, passes, platform)
+ if (platform == :'jvm')
+ command = "java stx.libjava.benchmarks.#{test} #{passes}"
+ elsif (platform == :'stx')
+ command = "./benchmark-runner.sh --smalltalk -b #{test} -n #{passes} 2>&1"
+ elsif (platform == :'libjava')
+ command = "./benchmark-runner.sh --java -b #{test} -n #{passes} 2>&1"
+ elsif (platform == :'stx2libjava')
+ command = "./benchmark-runner.sh --smalltalk2java -b #{test} -n #{passes} 2>&1"
+ else
+ raise Exception.new("Unssuported platform: #{platform}")
+ end
+
+ puts " running: #{command}"
+ output = `#{command}`
+ if ($? != 0)
+ puts output
+ raise Exception.new("Command failed!")
+ end
+ execution_time = (output.scan /^EXECUTION TIME: N\/A/).last()
+ if execution_time
+ puts " => N/A"
+ return "N/A"
+ end
+
+ execution_time = (output.scan /^EXECUTION TIME: \d*[\.\d]\d*$/).last()
+ execution_time = execution_time.gsub /^EXECUTION TIME: /, ''
+ puts " => #{execution_time}"
+ return execution_time.to_i
+end
+
+
+def write_results_txt_to_file(filename, results)
+ File.open(filename, "a+") do |file|
+ write_results_txt(file, results)
+ end
+end
+
+
+def write_results_txt(file, results)
+ file.write("\n")
+ file.write(Time.now.to_s)
+ file.write("\n")
+ values = [[ "Test", "JVM", "STX-S", "STX-J" , "STX-J2S" ]]
+ TESTS.each do | pair |
+ if (pair.size == 2)
+ key = pair[0]
+ if (results.has_key? key)
+ values << [ key, results[key][:jvm], results[key][:stx], results[key][:libjava], results[key][:stx2libjava] ]
+ end
+ end
+ end
+ max_lengths = values[0].map { |val| val.length }
+ values.each do |row|
+ row.each_with_index do |elem, index|
+ elem_size = elem.size
+ max_lengths[index] = elem_size if elem_size > max_lengths[index]
+ end
+ end
+ values.each do |val|
+ format = max_lengths.map { |length| "%#{length}s" }.join(" " * 3)
+ file.write(format % val)
+ file.write("\n")
+ end
+ file.write("--\n")
+
+end
+
+def write_results_csv_to_file(filename, results)
+ File.open(filename, "a+") do |file|
+ write_results_csv(file, results)
+ end
+end
+
+
+def write_results_csv(file, results)
+ values = [[ "Test", "JVM", "STX-S", "STX-J" , "STX-J2S" ]]
+ TESTS.each do | pair |
+ if (pair.size == 2)
+ key = pair[0]
+ if (results.has_key? key)
+ values << [ key, results[key][:jvm], results[key][:stx], results[key][:libjava], results[key][:stx2libjava] ]
+ end
+ end
+ end
+ max_lengths = values[0].map { |val| val.length }
+ values.each do |row|
+ row.each_with_index do |elem, index|
+ elem_size = elem.size
+ max_lengths[index] = elem_size if elem_size > max_lengths[index]
+ end
+ end
+ values.each do |val|
+ format = max_lengths.map { |length| "%#{length}s , " }.join(" " * 3)
+ file.write(format % val)
+ file.write("\n")
+ end
+end
+