3 (:export #:with-test #:report-test-status #:*failures*
4 #:really-invoke-debugger
5 #:*break-on-failure* #:*break-on-expected-failure*
6 #:make-kill-thread #:make-join-thread
9 (in-package :test-util)
11 (defvar *test-count* 0)
12 (defvar *test-file* nil)
13 (defvar *failures* nil)
14 (defvar *break-on-failure* nil)
15 (defvar *break-on-expected-failure* nil)
17 (defvar *threads-to-kill*)
18 (defvar *threads-to-join*)
20 (eval-when (:compile-toplevel :load-toplevel :execute)
23 ;;; run-program on Windows doesn't have an :environment parameter,
24 ;;; set these globally
25 (sb-posix:putenv (format nil "SBCL_MACHINE_TYPE=~A" (machine-type)))
26 (sb-posix:putenv (format nil "SBCL_SOFTWARE_TYPE=~A" (software-type)))
29 (defun make-kill-thread (&rest args)
30 (let ((thread (apply #'sb-thread:make-thread args)))
31 (when (boundp '*threads-to-kill*)
32 (push thread *threads-to-kill*))
36 (defun make-join-thread (&rest args)
37 (let ((thread (apply #'sb-thread:make-thread args)))
38 (when (boundp '*threads-to-join*)
39 (push thread *threads-to-join*))
42 (defun log-msg (&rest args)
43 (format *trace-output* "~&::: ")
44 (apply #'format *trace-output* args)
45 (terpri *trace-output*)
46 (force-output *trace-output*))
48 (defmacro with-test ((&key fails-on broken-on skipped-on name)
50 (let ((block-name (gensym))
51 #+sb-thread (threads (gensym "THREADS")))
55 (symbol (let ((package (symbol-package x)))
57 (eql package (find-package "CL"))
58 (eql package (find-package "KEYWORD"))
59 (eql (mismatch "SB-" (package-name package)) 3))))
61 (unless (tree-equal name name :test #'name-ok)
62 (error "test name must be all-keywords: ~S" name)))
66 ((broken-p ,broken-on)
67 (fail-test :skipped-broken ',name "Test broken on this platform"))
68 ((skipped-p ,skipped-on)
69 (fail-test :skipped-disabled ',name "Test disabled for this combination of platform and features"))
71 (let (#+sb-thread (,threads (sb-thread:list-all-threads))
72 (*threads-to-join* nil)
73 (*threads-to-kill* nil))
75 (handler-bind ((error (lambda (error)
76 (if (expected-failure-p ,fails-on)
77 (fail-test :expected-failure ',name error)
78 (fail-test :unexpected-failure ',name error))
79 (return-from ,block-name))))
81 (log-msg "Running ~S" ',name)
84 (let ((any-leftover nil))
85 (dolist (thread *threads-to-join*)
86 (ignore-errors (sb-thread:join-thread thread)))
87 (dolist (thread *threads-to-kill*)
88 (ignore-errors (sb-thread:terminate-thread thread)))
89 (setf ,threads (union (union *threads-to-kill*
92 #+(and sb-safepoint-strictly (not win32))
93 (dolist (thread (sb-thread:list-all-threads))
94 (when (typep thread 'sb-thread:signal-handling-thread)
95 (ignore-errors (sb-thread:join-thread thread))))
96 (dolist (thread (sb-thread:list-all-threads))
97 (unless (or (not (sb-thread:thread-alive-p thread))
98 (eql thread sb-thread:*current-thread*)
99 (member thread ,threads)
100 (sb-thread:thread-emphemeral-p thread))
101 (setf any-leftover thread)
102 (ignore-errors (sb-thread:terminate-thread thread))))
104 (fail-test :leftover-thread ',name any-leftover)
105 (return-from ,block-name)))
106 (if (expected-failure-p ,fails-on)
107 (fail-test :unexpected-success ',name nil)
108 (log-msg "Success ~S" ',name)))))))))))
110 (defun report-test-status ()
111 (with-standard-io-syntax
112 (with-open-file (stream "test-status.lisp-expr"
114 :if-exists :supersede)
115 (format stream "~s~%" *failures*))))
118 (unless (eq *test-file* *load-pathname*)
119 (setf *test-file* *load-pathname*)
120 (setf *test-count* 0))
123 (defun really-invoke-debugger (condition)
124 (with-simple-restart (continue "Continue")
125 (let ((*invoke-debugger-hook* *invoke-debugger-hook*))
127 (invoke-debugger condition))))
129 (defun fail-test (type test-name condition)
130 (if (stringp condition)
131 (log-msg "~@<~A ~S ~:_~A~:>"
132 type test-name condition)
133 (log-msg "~@<~A ~S ~:_due to ~S: ~4I~:_\"~A\"~:>"
134 type test-name condition condition))
135 (push (list type *test-file* (or test-name *test-count*))
137 (unless (stringp condition)
138 (when (or (and *break-on-failure*
139 (not (eq type :expected-failure)))
140 *break-on-expected-failure*)
141 (really-invoke-debugger condition))))
143 (defun expected-failure-p (fails-on)
144 (sb-impl::featurep fails-on))
146 (defun broken-p (broken-on)
147 (sb-impl::featurep broken-on))
149 (defun skipped-p (skipped-on)
150 (sb-impl::featurep skipped-on))
152 ;;; Repeat calling THUNK until its cumulated runtime, measured using
153 ;;; GET-INTERNAL-RUN-TIME, is larger than PRECISION. Repeat this
154 ;;; REPETITIONS many times and return the time one call to THUNK took
155 ;;; in seconds as a float, according to the minimum of the cumulated
156 ;;; runtimes over the repetitions.
157 ;;; This allows to easily measure the runtime of expressions that take
158 ;;; much less time than one internal time unit. Also, the results are
159 ;;; unaffected, modulo quantization effects, by changes to
160 ;;; INTERNAL-TIME-UNITS-PER-SECOND.
161 ;;; Taking the minimum is intended to reduce the error introduced by
162 ;;; garbage collections occurring at unpredictable times. The inner
163 ;;; loop doubles the number of calls to THUNK each time before again
164 ;;; measuring the time spent, so that the time measurement overhead
165 ;;; doesn't distort the result if calling THUNK takes very little time.
166 (defun runtime* (thunk repetitions precision)
167 (loop repeat repetitions
169 (loop with start = (get-internal-run-time)
171 for n = 1 then (* n 2)
172 for total-runs = n then (+ total-runs n)
175 (setf duration (- (get-internal-run-time) start))
176 when (> duration precision)
177 return (/ (float duration) (float total-runs)))
178 into min-internal-time-units-per-call
179 finally (return (/ min-internal-time-units-per-call
180 (float internal-time-units-per-second)))))
182 (defmacro runtime (form &key (repetitions 3) (precision 10))
183 `(runtime* (lambda () ,form) ,repetitions ,precision))