rakelib/test.rake
author Jan Vrany <jan.vrany@fit.cvut.cz>
Thu, 30 Aug 2018 10:06:05 +0100
changeset 254 70c9861ad62f
parent 250 86db38276922
parent 253 3747e4b3256b
permissions -rw-r--r--
Merge

TESTREPORT_CLASS = 'Builder::TestReport'
LINTREPORT_CLASS = 'Builder::LintReport'


# A helper class to keep a summary of a test report
class TestReportSummary
  SUMMARIES = []
  PATTERN = /tests="(?<run>\d+)".+failures="(?<failed>\d+)" errors="(?<errors>\d+)" skipped="(?<skipped>\d+)" time="(?<time>\d+\.\d+)"/

  attr_reader :pkg, :run, :failed, :errors, :skipped, :time

  def passed
    @run - @failed - @errors - @skipped
  end

  def outcome
    (@failed > 0 or @errors > 0) ? 'FAILED' : 'PASSED'
  end
  
  def execution_time
    hours   = (@time/3600).floor
    minutes = (@time/60).floor
    seconds = (@time%60).round
    "#{hours}h:#{minutes}m:#{seconds}s"
  end

  # Creates a new summary for given package and report file.
  def initialize(pkg_name, report_file)
    raise Exception.new("Report file does not exist! #{report_file}") unless File.exist? report_file
    @pkg = pkg_name
    matches = PATTERN.match(IO.read(report_file, 512))

    # Maybe the buffer is too small? Try to read up more
    # data....
    raise Exception.new(%q{Cannot "parse" report file!}) unless matches
    @run = matches['run'].to_i
    @failed = matches['failed'].to_i
    @errors = matches['errors'].to_i
    @skipped = matches['skipped'].to_i
    @time = matches['time'].to_f
  end
end

desc 'Run tests'
task :test => :'test:all'

desc 'Run tests (alias for target test)'
task :tests => :'test'


task :'setup:tasks' => :'setup:tasks:test'

def run_report(app, packages, report, global_opts = '', report_opts = '')
  #run_report_st = BUILD_DIR / 'stx' / 'goodies' / 'builder' / 'reports' / 'report-runner-old.st'
  run_report_st = BUILD_DIR / 'stx' / 'goodies' / 'builder' / 'reports' / 'report-runner.st'
  coveragerportformat_dot_st = BUILD_DIR / 'stx' / 'goodies' / 'builder' / 'reports' / 'Builder__CoverageReportFormat.st'

  report_dir = File.expand_path(REPORT_DIR)
  tmp_dir = File.expand_path(TMP_DIR)

  # Set STX_TMPDIR environment to make sure all temporary files created by
  # Smalltalk/X goes to a local tmp directory (which should be discarded
  # regularly). This helps to avoid garbage to heap up on Windows slaves
  # assuming workspaces is thrown away often.
  ENV['STX_TMPDIR'] = tmp_dir

  if app
    exe_dir = BUILD_DIR / app.directory
    win32? ? (exe = "#{app.executable}.com") : (exe = "./#{app.executable}")
  else
    exe_dir = BUILD_DIR / 'stx' / 'projects' / 'smalltalk'
    win32? ? (exe = 'stx.com') : (exe = './stx')
  end

  (mkdir_p tmp_dir) unless File.directory?(tmp_dir)
  chdir exe_dir do
    packages_args = ''
    packages.each {|p| packages_args += " -p #{p}"}

    File.exist?(coveragerportformat_dot_st) ? (runner_opts = "--abortOnSEGV --abortOnInternalError -I --execute #{run_report_st}") : (runner_opts = "-I -f #{run_report_st}")
    sh "#{exe} #{runner_opts} #{global_opts} -i \"#{BUILD_ID}\" -D \"#{report_dir}\" -r #{report} #{report_opts} #{packages_args}"
  end
  # No, do not remove tmp_dir here. If a test fails,
  # then temp dir may contain valuable files to debug/reproduce
  # (bad image files, outputs and so on). 
  # Depend on some other external process to wipe out temp directory
  # once not needed (full CI build cycle should do that)
  #
  # rm_rf tmp_dir
end

task :'setup:tasks:test' do
  $__testresults__ = []
  app = project.application

  # We shuffle the order of the packages by purpose.
  # Some tests are CPU bound while some are IO bound,
  # so this should help to distribute the load more evenly
  # in cases where multiple tests (`rake test` invocations)
  # are run in parallel (such as on CI)
  project.packages.shuffle.each do |pkg|
    if pkg.test
      task "test:package:#{pkg.name}:pre"
      task "test:package:#{pkg.name}:post"
      task "test:package:#{pkg.name}:main" => ['stx:goodies/builder/reports', REPORT_DIR] do
        report_file = File.expand_path(REPORT_DIR) / "#{pkg.name_components.join('_')}-#{BUILD_ID}-Test.xml"
        # Sigh, sigh. On CI server, sometimes it happen that tests are simply not run.
        # I was not able to debug the issue anyhow, any attempt to attach a debugger,
        # add a debug print or run it manually failed as the problem did not manifest.
        #
        # This is a feeble and horrible attempt to just cover up the problem by
        # trying multiple times. I don't want to spend more time chasing issues
        # like this. Sorry.
        #
        # Let's see if it helps.
        5.times do #for i in 1..5 do
          pkg.coverage ? run_report(app, [pkg.name], TESTREPORT_CLASS, '', '--coverage') : run_report(app, [pkg.name], TESTREPORT_CLASS)
          # Extract summary from XML report and keep it. Yeah, parsing XML
          # using regexps is a bad thing, but it's quick and lot less code!
          break if File.exist?(report_file)
        end
        report_summary = TestReportSummary.new(pkg.name, report_file)
        TestReportSummary::SUMMARIES << report_summary
      end

      task "test:package:#{pkg.name}" => [ "test:package:#{pkg.name}:pre",
                                           "test:package:#{pkg.name}:main",
                                           "test:package:#{pkg.name}:post" ]
      task :'test:packages' => "test:package:#{pkg.name}"
      task "#{pkg.name}:test" => "test:package:#{pkg.name}"
    end

    if pkg.lint
      task "lint:package:#{pkg.name}" => %W(stx:goodies/builder/reports REPORT_DIR) do
        #puts "LINT DISABLED (because of some bug in recent SmallLint - runs out of memory)"
        run_report(app, [pkg.name], LINTREPORT_CLASS)
      end
      task :'lint:packages' => "lint:package:#{pkg.name}"
    end
  end
end

task :'setup:tasks' => :'setup:tasks:test'
task :'test:setup' => :setup
task :'lint:setup' => :setup

namespace :test do
  task :all => %i(setup pre main post)
  task :pre
  task :post

  directory REPORT_DIR

  task :setup => :'setup:dependencies'

  task :main => %i(setup packages summary)

  task :packages

  task :summary do
    outcome = 'PASSED'
    # Detect if the test is run in ConEmu environment.  Running test suite in ConEmu
    # yields often incorrect results due to some code injection and other techniques 
    # used by ConEmu to fix bugs or other deficiencies in the MS code
    if (ENV['ConEmuPID'])
      puts
      puts 'WARNING: Test was executed within ConEmu environment the test results are UNRELIABLE!'
      puts 'Run the test suite directly in the cmd.exe or the powershell.exe.'
      puts
    end
    puts
    puts 'OVERALL SUMMARY'
    puts
    (TestReportSummary::SUMMARIES.sort { | a, b | a.pkg <=> b.pkg }).each do |test_summary|
      puts "%-20s %s - %d run, %d passed, %d skipped, %d failed, %d errors, execution time: %s" % [
        test_summary.pkg,
        test_summary.outcome,
        test_summary.run,
        test_summary.passed,
        test_summary.skipped,
        test_summary.failed,
        test_summary.errors,
        test_summary.execution_time
      ]
      outcome = 'FAILED' if test_summary.failed > 0 or test_summary.errors > 0
    end
    puts
    puts outcome
  end
end

desc 'Run static analysis on the code (SmallLint)'
task :lint => :'lint:all'


namespace :lint do
  task :all => %i(setup pre main post)
  task :pre
  task :post

  directory REPORT_DIR

  task :setup

  task :main => %i(setup packages)

  task :packages
end