Really restore clisp cross-compilation.
[sbcl.git] / tests / test-util.lisp
1 (defpackage :test-util
2   (:use :cl :sb-ext)
3   (:export #:with-test #:report-test-status #:*failures*
4            #:really-invoke-debugger
5            #:*break-on-failure* #:*break-on-expected-failure*
6            #:make-kill-thread #:make-join-thread
7            #:runtime))
8
9 (in-package :test-util)
10
11 (defvar *test-count* 0)
12 (defvar *test-file* nil)
13 (defvar *failures* nil)
14 (defvar *break-on-failure* nil)
15 (defvar *break-on-expected-failure* nil)
16
17 (defvar *threads-to-kill*)
18 (defvar *threads-to-join*)
19
20 (eval-when (:compile-toplevel :load-toplevel :execute)
21   (require :sb-posix))
22
23 (sb-posix:putenv (format nil "SBCL_MACHINE_TYPE=~A" (machine-type)))
24 (sb-posix:putenv (format nil "SBCL_SOFTWARE_TYPE=~A" (software-type)))
25
26 #+sb-thread
27 (defun make-kill-thread (&rest args)
28   (let ((thread (apply #'sb-thread:make-thread args)))
29     (when (boundp '*threads-to-kill*)
30       (push thread *threads-to-kill*))
31     thread))
32
33 #+sb-thread
34 (defun make-join-thread (&rest args)
35   (let ((thread (apply #'sb-thread:make-thread args)))
36     (when (boundp '*threads-to-join*)
37       (push thread *threads-to-join*))
38     thread))
39
40 (defun log-msg (&rest args)
41   (format *trace-output* "~&::: ")
42   (apply #'format *trace-output* args)
43   (terpri *trace-output*)
44   (force-output *trace-output*))
45
46 (defmacro with-test ((&key fails-on broken-on skipped-on name)
47                      &body body)
48   (let ((block-name (gensym))
49         #+sb-thread (threads (gensym "THREADS")))
50     (flet ((name-ok (x y)
51              (declare (ignore y))
52              (typecase x
53                (symbol (let ((package (symbol-package x)))
54                          (or (null package)
55                              (eql package (find-package "CL"))
56                              (eql package (find-package "KEYWORD"))
57                              (eql (mismatch "SB-" (package-name package)) 3))))
58                (integer t))))
59       (unless (tree-equal name name :test #'name-ok)
60         (error "test name must be all-keywords: ~S" name)))
61     `(progn
62        (start-test)
63        (cond
64          ((broken-p ,broken-on)
65           (fail-test :skipped-broken ',name "Test broken on this platform"))
66          ((skipped-p ,skipped-on)
67           (fail-test :skipped-disabled ',name "Test disabled for this combination of platform and features"))
68          (t
69           (let (#+sb-thread (,threads (sb-thread:list-all-threads))
70                 (*threads-to-join* nil)
71                 (*threads-to-kill* nil))
72             (block ,block-name
73               (handler-bind ((error (lambda (error)
74                                       (if (expected-failure-p ,fails-on)
75                                           (fail-test :expected-failure ',name error)
76                                           (fail-test :unexpected-failure ',name error))
77                                       (return-from ,block-name))))
78                 (progn
79                   (log-msg "Running ~S" ',name)
80                   ,@body
81                   #+sb-thread
82                   (let ((any-leftover nil))
83                     (dolist (thread *threads-to-join*)
84                       (ignore-errors (sb-thread:join-thread thread)))
85                     (dolist (thread *threads-to-kill*)
86                       (ignore-errors (sb-thread:terminate-thread thread)))
87                     (setf ,threads (union (union *threads-to-kill*
88                                                  *threads-to-join*)
89                                           ,threads))
90                     #+(and sb-safepoint-strictly (not win32))
91                     (dolist (thread (sb-thread:list-all-threads))
92                       (when (typep thread 'sb-thread:signal-handling-thread)
93                         (ignore-errors (sb-thread:join-thread thread))))
94                     (dolist (thread (sb-thread:list-all-threads))
95                       (unless (or (not (sb-thread:thread-alive-p thread))
96                                   (eql thread sb-thread:*current-thread*)
97                                   (member thread ,threads)
98                                   (sb-thread:thread-ephemeral-p thread))
99                         (setf any-leftover thread)
100                         (ignore-errors (sb-thread:terminate-thread thread))))
101                     (when any-leftover
102                       (fail-test :leftover-thread ',name any-leftover)
103                       (return-from ,block-name)))
104                   (if (expected-failure-p ,fails-on)
105                       (fail-test :unexpected-success ',name nil)
106                       (log-msg "Success ~S" ',name)))))))))))
107
108 (defun report-test-status ()
109   (with-standard-io-syntax
110       (with-open-file (stream "test-status.lisp-expr"
111                               :direction :output
112                               :if-exists :supersede)
113         (format stream "~s~%" *failures*))))
114
115 (defun start-test ()
116   (unless (eq *test-file* *load-pathname*)
117     (setf *test-file* *load-pathname*)
118     (setf *test-count* 0))
119   (incf *test-count*))
120
121 (defun really-invoke-debugger (condition)
122   (with-simple-restart (continue "Continue")
123     (let ((*invoke-debugger-hook* *invoke-debugger-hook*))
124       (enable-debugger)
125       (invoke-debugger condition))))
126
127 (defun fail-test (type test-name condition)
128   (if (stringp condition)
129       (log-msg "~@<~A ~S ~:_~A~:>"
130                type test-name condition)
131       (log-msg "~@<~A ~S ~:_due to ~S: ~4I~:_\"~A\"~:>"
132                type test-name condition condition))
133   (push (list type *test-file* (or test-name *test-count*))
134         *failures*)
135   (unless (stringp condition)
136     (when (or (and *break-on-failure*
137                    (not (eq type :expected-failure)))
138               *break-on-expected-failure*)
139       (really-invoke-debugger condition))))
140
141 (defun expected-failure-p (fails-on)
142   (sb-impl::featurep fails-on))
143
144 (defun broken-p (broken-on)
145   (sb-impl::featurep broken-on))
146
147 (defun skipped-p (skipped-on)
148   (sb-impl::featurep skipped-on))
149
150 ;;; Repeat calling THUNK until its cumulated runtime, measured using
151 ;;; GET-INTERNAL-RUN-TIME, is larger than PRECISION. Repeat this
152 ;;; REPETITIONS many times and return the time one call to THUNK took
153 ;;; in seconds as a float, according to the minimum of the cumulated
154 ;;; runtimes over the repetitions.
155 ;;; This allows to easily measure the runtime of expressions that take
156 ;;; much less time than one internal time unit. Also, the results are
157 ;;; unaffected, modulo quantization effects, by changes to
158 ;;; INTERNAL-TIME-UNITS-PER-SECOND.
159 ;;; Taking the minimum is intended to reduce the error introduced by
160 ;;; garbage collections occurring at unpredictable times. The inner
161 ;;; loop doubles the number of calls to THUNK each time before again
162 ;;; measuring the time spent, so that the time measurement overhead
163 ;;; doesn't distort the result if calling THUNK takes very little time.
164 (defun runtime* (thunk repetitions precision)
165   (loop repeat repetitions
166         minimize
167         (loop with start = (get-internal-run-time)
168               with duration = 0
169               for n = 1 then (* n 2)
170               for total-runs = n then (+ total-runs n)
171               do (dotimes (i n)
172                    (funcall thunk))
173                  (setf duration (- (get-internal-run-time) start))
174               when (> duration precision)
175               return (/ (float duration) (float total-runs)))
176         into min-internal-time-units-per-call
177         finally (return (/ min-internal-time-units-per-call
178                            (float internal-time-units-per-second)))))
179
180 (defmacro runtime (form &key (repetitions 3) (precision 10))
181   `(runtime* (lambda () ,form) ,repetitions ,precision))