tests/test-util.lisp

   1 (defpackage :test-util
   2   (:use :cl :sb-ext)
   3   (:export #:with-test #:report-test-status #:*failures*
   4            #:really-invoke-debugger
   5            #:*break-on-failure* #:*break-on-expected-failure*
   6            #:make-kill-thread #:make-join-thread
   7            #:runtime))
   8
   9 (in-package :test-util)
  10
  11 (defvar *test-count* 0)
  12 (defvar *test-file* nil)
  13 (defvar *failures* nil)
  14 (defvar *break-on-failure* nil)
  15 (defvar *break-on-expected-failure* nil)
  16
  17 (defvar *threads-to-kill*)
  18 (defvar *threads-to-join*)
  19
  20 (eval-when (:compile-toplevel :load-toplevel :execute)
  21   (require :sb-posix))
  22
  23 ;;; run-program on Windows doesn't have an :environment parameter,
  24 ;;; set these globally
  25 (sb-posix:putenv (format nil "SBCL_MACHINE_TYPE=~A" (machine-type)))
  26 (sb-posix:putenv (format nil "SBCL_SOFTWARE_TYPE=~A" (software-type)))
  27
  28 #+sb-thread
  29 (defun make-kill-thread (&rest args)
  30   (let ((thread (apply #'sb-thread:make-thread args)))
  31     (when (boundp '*threads-to-kill*)
  32       (push thread *threads-to-kill*))
  33     thread))
  34
  35 #+sb-thread
  36 (defun make-join-thread (&rest args)
  37   (let ((thread (apply #'sb-thread:make-thread args)))
  38     (when (boundp '*threads-to-join*)
  39       (push thread *threads-to-join*))
  40     thread))
  41
  42 (defun log-msg (&rest args)
  43   (format *trace-output* "~&::: ")
  44   (apply #'format *trace-output* args)
  45   (terpri *trace-output*)
  46   (force-output *trace-output*))
  47
  48 (defmacro with-test ((&key fails-on broken-on skipped-on name)
  49                      &body body)
  50   (let ((block-name (gensym))
  51         #+sb-thread (threads (gensym "THREADS")))
  52     (flet ((name-ok (x y)
  53              (declare (ignore y))
  54              (typecase x
  55                (symbol (let ((package (symbol-package x)))
  56                          (or (null package)
  57                              (eql package (find-package "CL"))
  58                              (eql package (find-package "KEYWORD"))
  59                              (eql (mismatch "SB-" (package-name package)) 3))))
  60                (integer t))))
  61       (unless (tree-equal name name :test #'name-ok)
  62         (error "test name must be all-keywords: ~S" name)))
  63     `(progn
  64        (start-test)
  65        (cond
  66          ((broken-p ,broken-on)
  67           (fail-test :skipped-broken ',name "Test broken on this platform"))
  68          ((skipped-p ,skipped-on)
  69           (fail-test :skipped-disabled ',name "Test disabled for this combination of platform and features"))
  70          (t
  71           (let (#+sb-thread (,threads (sb-thread:list-all-threads))
  72                 (*threads-to-join* nil)
  73                 (*threads-to-kill* nil))
  74             (block ,block-name
  75               (handler-bind ((error (lambda (error)
  76                                       (if (expected-failure-p ,fails-on)
  77                                           (fail-test :expected-failure ',name error)
  78                                           (fail-test :unexpected-failure ',name error))
  79                                       (return-from ,block-name))))
  80                 (progn
  81                   (log-msg "Running ~S" ',name)
  82                   ,@body
  83                   #+sb-thread
  84                   (let ((any-leftover nil))
  85                     (dolist (thread *threads-to-join*)
  86                       (ignore-errors (sb-thread:join-thread thread)))
  87                     (dolist (thread *threads-to-kill*)
  88                       (ignore-errors (sb-thread:terminate-thread thread)))
  89                     (setf ,threads (union (union *threads-to-kill*
  90                                                  *threads-to-join*)
  91                                           ,threads))
  92                     #+(and sb-safepoint-strictly (not win32))
  93                     (dolist (thread (sb-thread:list-all-threads))
  94                       (when (typep thread 'sb-thread:signal-handling-thread)
  95                         (ignore-errors (sb-thread:join-thread thread))))
  96                     (dolist (thread (sb-thread:list-all-threads))
  97                       (unless (or (not (sb-thread:thread-alive-p thread))
  98                                   (eql thread sb-thread:*current-thread*)
  99                                   (member thread ,threads)
 100                                   (sb-thread:thread-emphemeral-p thread))
 101                         (setf any-leftover thread)
 102                         (ignore-errors (sb-thread:terminate-thread thread))))
 103                     (when any-leftover
 104                       (fail-test :leftover-thread ',name any-leftover)
 105                       (return-from ,block-name)))
 106                   (if (expected-failure-p ,fails-on)
 107                       (fail-test :unexpected-success ',name nil)
 108                       (log-msg "Success ~S" ',name)))))))))))
 109
 110 (defun report-test-status ()
 111   (with-standard-io-syntax
 112       (with-open-file (stream "test-status.lisp-expr"
 113                               :direction :output
 114                               :if-exists :supersede)
 115         (format stream "~s~%" *failures*))))
 116
 117 (defun start-test ()
 118   (unless (eq *test-file* *load-pathname*)
 119     (setf *test-file* *load-pathname*)
 120     (setf *test-count* 0))
 121   (incf *test-count*))
 122
 123 (defun really-invoke-debugger (condition)
 124   (with-simple-restart (continue "Continue")
 125     (let ((*invoke-debugger-hook* *invoke-debugger-hook*))
 126       (enable-debugger)
 127       (invoke-debugger condition))))
 128
 129 (defun fail-test (type test-name condition)
 130   (if (stringp condition)
 131       (log-msg "~@<~A ~S ~:_~A~:>"
 132                type test-name condition)
 133       (log-msg "~@<~A ~S ~:_due to ~S: ~4I~:_\"~A\"~:>"
 134                type test-name condition condition))
 135   (push (list type *test-file* (or test-name *test-count*))
 136         *failures*)
 137   (unless (stringp condition)
 138     (when (or (and *break-on-failure*
 139                    (not (eq type :expected-failure)))
 140               *break-on-expected-failure*)
 141       (really-invoke-debugger condition))))
 142
 143 (defun expected-failure-p (fails-on)
 144   (sb-impl::featurep fails-on))
 145
 146 (defun broken-p (broken-on)
 147   (sb-impl::featurep broken-on))
 148
 149 (defun skipped-p (skipped-on)
 150   (sb-impl::featurep skipped-on))
 151
 152 ;;; Repeat calling THUNK until its cumulated runtime, measured using
 153 ;;; GET-INTERNAL-RUN-TIME, is larger than PRECISION. Repeat this
 154 ;;; REPETITIONS many times and return the time one call to THUNK took
 155 ;;; in seconds as a float, according to the minimum of the cumulated
 156 ;;; runtimes over the repetitions.
 157 ;;; This allows to easily measure the runtime of expressions that take
 158 ;;; much less time than one internal time unit. Also, the results are
 159 ;;; unaffected, modulo quantization effects, by changes to
 160 ;;; INTERNAL-TIME-UNITS-PER-SECOND.
 161 ;;; Taking the minimum is intended to reduce the error introduced by
 162 ;;; garbage collections occurring at unpredictable times. The inner
 163 ;;; loop doubles the number of calls to THUNK each time before again
 164 ;;; measuring the time spent, so that the time measurement overhead
 165 ;;; doesn't distort the result if calling THUNK takes very little time.
 166 (defun runtime* (thunk repetitions precision)
 167   (loop repeat repetitions
 168         minimize
 169         (loop with start = (get-internal-run-time)
 170               with duration = 0
 171               for n = 1 then (* n 2)
 172               for total-runs = n then (+ total-runs n)
 173               do (dotimes (i n)
 174                    (funcall thunk))
 175                  (setf duration (- (get-internal-run-time) start))
 176               when (> duration precision)
 177               return (/ (float duration) (float total-runs)))
 178         into min-internal-time-units-per-call
 179         finally (return (/ min-internal-time-units-per-call
 180                            (float internal-time-units-per-second)))))
 181
 182 (defmacro runtime (form &key (repetitions 3) (precision 10))
 183   `(runtime* (lambda () ,form) ,repetitions ,precision))