320 skippedCount |
321 skippedCount |
321 |
322 |
322 ^nskipped |
323 ^nskipped |
323 |
324 |
324 "Created: / 31-01-2013 / 13:54:42 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
325 "Created: / 31-01-2013 / 13:54:42 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
|
326 ! |
|
327 |
|
328 timeoutScale |
|
329 "Return a timeout scaling factor used to adjust test-case defined timeout. |
|
330 This is required in order to stabilize tests on different computers and setups. |
|
331 Slower machines may need higher timeout otherwise tests may fail." |
|
332 |
|
333 "/ This scaling factor is computed automatically by running a (not so) simple |
|
334 "/ benchmark. To make things more complicated, we have to re-asses the scaling |
|
335 "/ factor time to time as machine performance may vary depending on a load - this |
|
336 "/ happens specially on CI setups where CI slaves are virtualized and running on |
|
337 "/ heavily-loaded systems. |
|
338 "/ |
|
339 "/ The benchmark itself consists of: |
|
340 "/ |
|
341 "/ * CPU benchmark - to handle systems with slow CPUs such as some |
|
342 "/ low-end Celerons on RPi-kind of thing) |
|
343 "/ * IO benchmark - to handle the case of slow IO on overloaded host |
|
344 "/ running many tests in parallel. |
|
345 "/ |
|
346 |
|
347 |
|
348 (timeoutScale isNil or: [timeoutScaleReassesmentTime < OperatingSystem getMillisecondTime ]) ifTrue:[ |
|
349 | time1 files time2 scale1 scale2 | |
|
350 |
|
351 "/ simple CPU benchmark |
|
352 time1 := Time millisecondsToRun:[3000 timesRepeat: [ 2000 factorial ]]. |
|
353 scale1 := time1 / 2500"mean value of an i5 64bit". |
|
354 |
|
355 files := (Smalltalk getPackageDirectoryForPackage: self class package) recursiveDirectoryContentsAsFilenames |
|
356 select: [:e | e isRegularFile and:[e suffix = 'st']]. |
|
357 time2 := Time millisecondsToRun:[ 10 timesRepeat: [ files shuffled do:[:e|e contents ] ] ]. |
|
358 scale2 := time2 / 2500"mean value of an 64bit linux ext4 on SATA SSD". |
|
359 |
|
360 timeoutScale := (scale1 max: scale2) max: 1. |
|
361 timeoutScaleReassesmentTime := OperatingSystem getMillisecondTime + (1000*60*3) "/ reasses every three minutes |
|
362 ]. |
|
363 ^ timeoutScale |
|
364 |
|
365 " |
|
366 Builder::TestReport::Result new timeoutScale; timeoutScale |
|
367 " |
|
368 |
|
369 "Created: / 24-08-2018 / 09:54:57 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
|
370 "Modified (comment): / 24-08-2018 / 11:20:15 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
325 ! ! |
371 ! ! |
326 |
372 |
327 !TestReport::Result methodsFor:'adding'! |
373 !TestReport::Result methodsFor:'adding'! |
328 |
374 |
329 addError: testcase detail: exception |
375 addError: testcase detail: exception |
384 |
430 |
385 initialize |
431 initialize |
386 super initialize. |
432 super initialize. |
387 |
433 |
388 npassed := nfailed := nerror := nskipped := 0. |
434 npassed := nfailed := nerror := nskipped := 0. |
389 TimeoutScale isNil ifTrue:[ |
435 timeoutScaleReassesmentTime := 0 |
390 | bench | |
|
391 |
|
392 bench := Time millisecondsToRun:[3000 timesRepeat: [ 2000 factorial ]]. |
|
393 TimeoutScale := (bench / 2500"mean value of an i5 32bit") max: 1. |
|
394 ]. |
|
395 |
436 |
396 "Created: / 31-01-2013 / 13:52:33 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
437 "Created: / 31-01-2013 / 13:52:33 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
397 "Modified: / 14-11-2016 / 23:37:40 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
438 "Modified: / 24-08-2018 / 10:58:55 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
398 ! ! |
439 ! ! |
399 |
440 |
400 !TestReport::Result methodsFor:'running'! |
441 !TestReport::Result methodsFor:'running'! |
401 |
442 |
402 performCase:aTestCase |
443 performCase:aTestCase |
496 lightForkCase:aTestCase debugged: debugged timeout: timeoutBase |
537 lightForkCase:aTestCase debugged: debugged timeout: timeoutBase |
497 | timeout thread sema stime etime timeouted error stack log logPos | |
538 | timeout thread sema stime etime timeouted error stack log logPos | |
498 |
539 |
499 Logger trace: 'Running %1>>%2' with: aTestCase nameForHDTestReport with: aTestCase selectorForHDTestReport. |
540 Logger trace: 'Running %1>>%2' with: aTestCase nameForHDTestReport with: aTestCase selectorForHDTestReport. |
500 Transcript show:'F'. |
541 Transcript show:'F'. |
501 timeout := (timeoutBase * TimeoutScale) rounded. |
542 timeout := (timeoutBase * self timeoutScale) rounded. |
502 sema := Semaphore new. |
543 sema := Semaphore new. |
503 stime := OperatingSystem getMillisecondTime. |
544 stime := OperatingSystem getMillisecondTime. |
504 log := false. |
545 log := false. |
505 logPos := format stream stream position. |
546 logPos := format stream stream position. |
506 timeouted := false. |
547 timeouted := false. |
518 Transcript show: 'K'. |
559 Transcript show: 'K'. |
519 ]. |
560 ]. |
520 |
561 |
521 etime := OperatingSystem getMillisecondTime. |
562 etime := OperatingSystem getMillisecondTime. |
522 timeouted ifTrue:[ |
563 timeouted ifTrue:[ |
523 error := TimeoutError new messageText: ('Timed out (effective %1ms, base %2ms, scale %3)' bindWith: timeout with: timeoutBase with:TimeoutScale) . |
564 error := TimeoutError new messageText: ('Timed out (effective %1ms, base %2ms, scale %3)' bindWith: timeout with: timeoutBase with: timeoutScale asFloat) . |
524 outcome isNil ifTrue:[ outcome := self createOutcome ]. |
565 outcome isNil ifTrue:[ outcome := self createOutcome ]. |
525 outcome result: TestResult stateError. |
566 outcome result: TestResult stateError. |
526 format |
567 format |
527 writeTestCase: aTestCase outcome: outcome |
568 writeTestCase: aTestCase outcome: outcome |
528 time: etime - stime |
569 time: etime - stime |
529 exception: error |
570 exception: error |
534 |
575 |
535 outcome := nil. |
576 outcome := nil. |
536 Transcript cr. |
577 Transcript cr. |
537 |
578 |
538 "Created: / 12-01-2012 / 17:42:12 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
579 "Created: / 12-01-2012 / 17:42:12 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
539 "Modified: / 16-11-2016 / 23:26:07 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
580 "Modified: / 24-08-2018 / 11:19:24 / Jan Vrany <jan.vrany@fit.cvut.cz>" |
540 ! |
581 ! |
541 |
582 |
542 unixForkCase:aTestCase debugged: debugged timeout: timeoutBase |
583 unixForkCase:aTestCase debugged: debugged timeout: timeoutBase |
543 | timeout pid status sema stime etime error stack suiteFailuresBefore suiteErrorsBefore log logPos | |
584 | timeout pid status sema stime etime error stack suiteFailuresBefore suiteErrorsBefore log logPos | |
544 |
585 |
545 Transcript show:'forking...'. |
586 Transcript show:'forking...'. |
546 timeout := (timeoutBase * TimeoutScale) rounded. |
587 timeout := (timeoutBase * self timeoutScale) rounded. |
547 sema := Semaphore new. |
588 sema := Semaphore new. |
548 stime := OperatingSystem getMillisecondTime. |
589 stime := OperatingSystem getMillisecondTime. |
549 log := false. |
590 log := false. |
550 logPos := format stream stream position. |
591 logPos := format stream stream position. |
551 Processor monitor: |
592 Processor monitor: |
591 super addPass: aTestCase. |
632 super addPass: aTestCase. |
592 ] ifTrue: |
633 ] ifTrue: |
593 [ log := true. |
634 [ log := true. |
594 status isNil |
635 status isNil |
595 ifTrue: |
636 ifTrue: |
596 [ error := TimeoutError new messageText: ('Timed out (effective %1ms, base %2ms, scale %3)' bindWith: timeout with: timeoutBase with:TimeoutScale) . |
637 [ error := TimeoutError new messageText: ('Timed out (effective %1ms, base %2ms, scale %3)' bindWith: timeout with: timeoutBase with:timeoutScale asFloat) . |
597 stack := 'Oops, timed out!! (timeout was ' , timeout printString , ' sec)'. ] |
638 stack := 'Oops, timed out!! (timeout was ' , timeout printString , ' sec)'. ] |
598 ifFalse: |
639 ifFalse: |
599 [ status status == #signal |
640 [ status status == #signal |
600 ifTrue: |
641 ifTrue: |
601 [ error := OSSignalInterrupt new parameter:status code. |
642 [ error := OSSignalInterrupt new parameter:status code. |