tests/run-tests.lisp

   1 #+#.(cl:if (cl:find-package "ASDF") '(or) '(and))
   2 (load (merge-pathnames "../contrib/asdf/asdf.fasl"))
   3
   4 #+#.(cl:if (cl:find-package "SB-POSIX") '(or) '(and))
   5 (let ((asdf:*central-registry*
   6        (cons "../contrib/systems/" asdf:*central-registry*)))
   7   (asdf:oos 'asdf:load-op 'sb-posix))
   8
   9 (load "test-util.lisp")
  10
  11 (defpackage :run-tests
  12     (:use :cl :test-util :sb-ext))
  13
  14 (load "assertoid.lisp")
  15
  16 (in-package run-tests)
  17
  18 (defvar *all-failures* nil)
  19 (defvar *break-on-error* nil)
  20 (defvar *accept-files* nil)
  21
  22 (defun run-all ()
  23   (dolist (arg (cdr *posix-argv*))
  24     (cond ((string= arg "--break-on-failure")
  25            (setf *break-on-error* t)
  26            (setf test-util:*break-on-failure* t))
  27           ((string= arg "--break-on-expected-failure")
  28            (setf test-util:*break-on-expected-failure* t))
  29           (t
  30            (push (truename (parse-namestring arg)) *accept-files*))))
  31   (pure-runner (pure-load-files) #'load-test)
  32   (pure-runner (pure-cload-files) #'cload-test)
  33   (impure-runner (impure-load-files) #'load-test)
  34   (impure-runner (impure-cload-files) #'cload-test)
  35   #-win32 (impure-runner (sh-files) #'sh-test)
  36   (report)
  37   (sb-ext:quit :unix-status (if (unexpected-failures)
  38                                 1
  39                                 104)))
  40
  41 (defun report ()
  42   (terpri)
  43   (format t "Finished running tests.~%")
  44   (cond (*all-failures*
  45          (format t "Status:~%")
  46          (dolist (fail (reverse *all-failures*))
  47            (cond ((eq (car fail) :unhandled-error)
  48                   (format t " ~20a ~a~%"
  49                           "Unhandled error"
  50                           (enough-namestring (second fail))))
  51                  ((eq (car fail) :invalid-exit-status)
  52                   (format t " ~20a ~a~%"
  53                           "Invalid exit status:"
  54                           (enough-namestring (second fail))))
  55                  (t
  56                   (format t " ~20a ~a / ~a~%"
  57                           (ecase (first fail)
  58                             (:expected-failure "Expected failure:")
  59                             (:unexpected-failure "Failure:")
  60                             (:unexpected-success "Unexpected success:"))
  61                           (enough-namestring (second fail))
  62                           (third fail))))))
  63         (t
  64          (format t "All tests succeeded~%"))))
  65
  66 (defun pure-runner (files test-fun)
  67   (format t "// Running pure tests (~a)~%" test-fun)
  68   (let ((*package* (find-package :cl-user))
  69         (*failures* nil))
  70     (setup-cl-user)
  71     (dolist (file files)
  72       (when (accept-test-file file)
  73         (format t "// Running ~a~%" file)
  74         (restart-case
  75             (handler-bind ((error (make-error-handler file)))
  76               (eval (funcall test-fun file)))
  77           (skip-file ()))))
  78     (append-failures)))
  79
  80 (defun run-in-child-sbcl (load-forms forms)
  81   ;; We used to fork() for POSIX platforms, and use this for Windows.
  82   ;; However, it seems better to use the same solution everywhere.
  83   (process-exit-code
  84    (#-win32 with-open-file #-win32 (devnull "/dev/null") #+win32 progn
  85      (sb-ext:run-program
  86       (first *POSIX-ARGV*)
  87       (append
  88        (list "--core" SB-INT:*CORE-STRING*
  89              "--noinform"
  90              "--no-sysinit"
  91              "--no-userinit"
  92              "--noprint"
  93              "--disable-debugger")
  94        (loop for form in (append load-forms forms)
  95              collect "--eval"
  96              collect (write-to-string form)))
  97       :output sb-sys:*stdout*
  98       :input #-win32 devnull #+win32 sb-sys:*stdin*))))
  99
 100 (defun run-impure-in-child-sbcl (test-file test-code)
 101   (run-in-child-sbcl
 102     `((load "test-util")
 103       (load "assertoid")
 104       (defpackage :run-tests
 105         (:use :cl :test-util :sb-ext)))
 106
 107     `((in-package :cl-user)
 108       (use-package :test-util)
 109       (use-package :assertoid)
 110       (setf test-util:*break-on-failure* ,test-util:*break-on-failure*)
 111       (setf test-util:*break-on-expected-failure*
 112             ,test-util:*break-on-expected-failure*)
 113       (let ((file ,test-file)
 114             (*break-on-error* ,run-tests::*break-on-error*))
 115         (format t "// Running ~a~%" file)
 116         (restart-case
 117             (handler-bind
 118                 ((error (lambda (condition)
 119                           (push (list :unhandled-error file)
 120                                 test-util::*failures*)
 121                           (cond (*break-on-error*
 122                                  (test-util:really-invoke-debugger condition))
 123                                 (t
 124                                  (format *error-output* "~&Unhandled ~a: ~a~%"
 125                                          (type-of condition) condition)
 126                                  (sb-debug:backtrace)))
 127                           (invoke-restart 'skip-file))))
 128               ,test-code)
 129           (skip-file ()
 130             (format t ">>>~a<<<~%" test-util::*failures*)))
 131         (test-util:report-test-status)
 132         (sb-ext:quit :unix-status 104)))))
 133
 134 (defun impure-runner (files test-fun)
 135   (format t "// Running impure tests (~a)~%" test-fun)
 136   (let ((*package* (find-package :cl-user)))
 137     (setup-cl-user)
 138     (dolist (file files)
 139       (when (accept-test-file file)
 140         (force-output)
 141         (let ((exit-code (run-impure-in-child-sbcl file
 142                                                    (funcall test-fun file))))
 143           (if (= exit-code 104)
 144               (with-open-file (stream "test-status.lisp-expr"
 145                                       :direction :input
 146                                       :if-does-not-exist :error)
 147                 (append-failures (read stream)))
 148               (push (list :invalid-exit-status file)
 149                     *all-failures*)))))))
 150
 151 (defun make-error-handler (file)
 152   (lambda (condition)
 153     (push (list :unhandled-error file) *failures*)
 154     (cond (*break-on-error*
 155            (test-util:really-invoke-debugger condition))
 156           (t
 157            (format *error-output* "~&Unhandled ~a: ~a~%"
 158                    (type-of condition) condition)
 159            (sb-debug:backtrace)))
 160     (invoke-restart 'skip-file)))
 161
 162 (defun append-failures (&optional (failures *failures*))
 163   (setf *all-failures* (append failures *all-failures*)))
 164
 165 (defun unexpected-failures ()
 166   (remove-if (lambda (x)
 167                (or (eq (car x) :expected-failure)
 168                    (eq (car x) :unexpected-success)))
 169              *all-failures*))
 170
 171 (defun setup-cl-user ()
 172   (use-package :test-util)
 173   (use-package :assertoid))
 174
 175 (defun load-test (file)
 176   `(load ,file))
 177
 178 (defun cload-test (file)
 179   `(let ((compile-name (compile-file-pathname ,file)))
 180      (unwind-protect
 181           (progn
 182             (compile-file ,file)
 183             (load compile-name))
 184        (ignore-errors
 185          (delete-file compile-name)))))
 186
 187 (defun sh-test (file)
 188   ;; What? No SB-POSIX:EXECV?
 189   `(let ((process (sb-ext:run-program "/bin/sh"
 190                                       (list (native-namestring ,file))
 191                                       :environment (test-util::test-env)
 192                                       :output *error-output*)))
 193      (sb-ext:quit :unix-status (process-exit-code process))))
 194
 195 (defun accept-test-file (file)
 196   (if *accept-files*
 197       (find (truename file) *accept-files* :test #'equalp)
 198       t))
 199
 200 (defun pure-load-files ()
 201   (directory "*.pure.lisp"))
 202
 203 (defun pure-cload-files ()
 204   (directory "*.pure-cload.lisp"))
 205
 206 (defun impure-load-files ()
 207   (directory "*.impure.lisp"))
 208
 209 (defun impure-cload-files ()
 210   (directory "*.impure-cload.lisp"))
 211
 212 (defun sh-files ()
 213   (directory "*.test.sh"))