3 (in-package :it.bese.FiveAM)
5 ;;;; * Analyzing the results
7 (defparameter *verbose-failures* nil
8 "T if we should print the expression failing, NIL otherwise.")
10 ;;;; Just as important as defining and runnig the tests is
11 ;;;; understanding the results. FiveAM provides the function EXPLAIN
12 ;;;; which prints a human readable summary (number passed, number
13 ;;;; failed, what failed and why, etc.) of a list of test results.
15 (defmethod explain ((exp detailed-text-explainer) results
16 &optional (stream *test-dribble*) (recursive-depth 0))
17 #| "Given a list of test results report write to stream detailed
18 human readable statistics regarding the results." |#
19 (multiple-value-bind (num-checks passed num-passed passed%
20 skipped num-skipped skipped%
21 failed num-failed failed%
22 unknown num-unknown unknown%)
23 (partition-results results)
24 (declare (ignore passed))
25 (flet ((output (&rest format-args)
26 (format stream "~&~vT" recursive-depth)
27 (apply #'format stream format-args)))
29 (when (zerop num-checks)
30 (output "Didn't run anything...huh?")
31 (return-from explain nil))
32 (output "Did ~D check~P.~%" num-checks num-checks)
33 (output " Pass: ~D (~2D%)~%" num-passed passed%)
34 (output " Skip: ~D (~2D%)~%" num-skipped skipped%)
35 (output " Fail: ~D (~2D%)~%" num-failed failed%)
37 (output " UNKNOWN RESULTS: ~D (~2D)~%" num-unknown unknown%))
40 (output "Failure Details:~%")
42 (output "--------------------------------~%")
43 (output "~A ~@{[~A]~}: ~%"
45 (description (test-case f)))
46 (output " ~A.~%" (reason f))
47 (when (for-all-test-failed-p f)
48 (output "Results collected with failure data:~%")
49 (explain exp (slot-value f 'result-list)
50 stream (+ 4 recursive-depth)))
51 (when (and *verbose-failures* (test-expr f))
52 (output " ~S~%" (test-expr f)))
53 (output "--------------------------------~%"))
56 (output "Skip Details:~%")
58 (output "~A ~@{[~A]~}: ~%"
60 (description (test-case f)))
61 (output " ~A.~%" (reason f)))
62 (terpri *test-dribble*)))))
64 (defmethod explain ((exp simple-text-explainer) results
65 &optional (stream *test-dribble*) (recursive-depth 0))
66 (multiple-value-bind (num-checks passed num-passed passed%
67 skipped num-skipped skipped%
68 failed num-failed failed%
69 unknown num-unknown unknown%)
70 (partition-results results)
71 (declare (ignore passed passed% skipped skipped% failed failed% unknown unknown%))
72 (format stream "~&~vTRan ~D checks, ~D passed" recursive-depth num-checks num-passed)
73 (when (plusp num-skipped)
74 (format stream ", ~D skipped " num-skipped))
75 (format stream " and ~D failed.~%" num-failed)
76 (when (plusp num-unknown)
77 (format stream "~vT~D UNKNOWN RESULTS.~%" recursive-depth num-unknown))))
79 (defun partition-results (results-list)
80 (let ((num-checks (length results-list)))
81 (destructuring-bind (passed skipped failed unknown)
82 (partitionx results-list
84 (typep res 'test-passed))
86 (typep res 'test-skipped))
88 (typep res 'test-failure))
90 (if (zerop num-checks)
98 passed (length passed) (floor (* 100 (/ (length passed) num-checks)))
99 skipped (length skipped) (floor (* 100 (/ (length skipped) num-checks)))
100 failed (length failed) (floor (* 100 (/ (length failed) num-checks)))
101 unknown (length unknown) (floor (* 100 (/ (length failed) num-checks))))))))
103 ;; Copyright (c) 2002-2003, Edward Marco Baringer
104 ;; All rights reserved.
106 ;; Redistribution and use in source and binary forms, with or without
107 ;; modification, are permitted provided that the following conditions are
110 ;; - Redistributions of source code must retain the above copyright
111 ;; notice, this list of conditions and the following disclaimer.
113 ;; - Redistributions in binary form must reproduce the above copyright
114 ;; notice, this list of conditions and the following disclaimer in the
115 ;; documentation and/or other materials provided with the distribution.
117 ;; - Neither the name of Edward Marco Baringer, nor BESE, nor the names
118 ;; of its contributors may be used to endorse or promote products
119 ;; derived from this software without specific prior written permission.
121 ;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
122 ;; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
123 ;; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
124 ;; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
125 ;; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
126 ;; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
127 ;; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
128 ;; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
129 ;; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
130 ;; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
131 ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE