Allow use of SB-RT's mechanism for expected test failures in contribs

... In RT, use the list of expected test failures not just for
    debugging output, but return as a primary result only whether
    there are unexpected failures, with additional result values for
    details on expected failures.

... Tweak make-target-contrib.sh so that the `test-passed' file, if
    non-empty, can indicate a `successful' build of the contrib with
    only known failures.

This mechanism is meant to follow in the footsteps of SBCL's main
test suite's mechanism for known failures, and is arguably not in
the original spirit of contribs as user contributions that must
build perfectly or fail to install entirely.  However, for parts of
contrib/ which are very commonly used as a part of SBCL, it is hoped
that this new mechanism will aid work toward bug fixes in those
contribs rather than de-emphasize it.

diff --git a/contrib/asdf-module.mk b/contrib/asdf-module.mk
index 9c32b23..dba81ea 100644 (file)
--- a/contrib/asdf-module.mk
+++ b/contrib/asdf-module.mk
@@ -32,7 +32,8 @@ all: $(EXTRA_ALL_TARGETS)
        $(SBCL) --eval '(defvar *system* "$(SYSTEM)")' --load ../asdf-stub.lisp --eval '(exit)'
 
 test: all
-       echo "(asdf:operate (quote asdf:load-op) :$(SYSTEM))" \
+       echo "(pushnew :sb-testing-contrib *features*)" \
+            "(asdf:operate (quote asdf:load-op) :$(SYSTEM))" \
             "(asdf:operate (quote asdf:test-op) :$(SYSTEM))" | \
          $(SBCL) --eval '(load "../asdf/asdf")'
 

diff --git a/contrib/sb-bsd-sockets/tests.lisp b/contrib/sb-bsd-sockets/tests.lisp
index 6f22017..827b7c0 100644 (file)
--- a/contrib/sb-bsd-sockets/tests.lisp
+++ b/contrib/sb-bsd-sockets/tests.lisp
@@ -3,6 +3,12 @@
 
 (in-package :sb-bsd-sockets-test)
 
+(defmacro deftest* ((name &key fails-on) form &rest results)
+  `(progn
+     (when (sb-impl::featurep ',fails-on)
+       (pushnew ',name sb-rt::*expected-failures*))
+     (deftest ,name ,form ,@results)))
+
 ;;; a real address
 (deftest make-inet-address
   (equalp (make-inet-address "127.0.0.1")  #(127 0 0 1))

diff --git a/contrib/sb-rt/rt.lisp b/contrib/sb-rt/rt.lisp
index 77a4560..8601ef7 100644 (file)
--- a/contrib/sb-rt/rt.lisp
+++ b/contrib/sb-rt/rt.lisp
@@ -248,7 +248,6 @@
                    ~:@(~{~<~%   ~1:;~S~>~
                          ~^, ~}~)."
                     (length new-failures)
-                    new-failures)))
-          ))
+                    new-failures)))))
       (finish-output s)
-      (null pending))))
+      (values (null new-failures) (null pending) pending))))

diff --git a/make-target-contrib.sh b/make-target-contrib.sh
index c171111..b609072 100644 (file)
--- a/make-target-contrib.sh
+++ b/make-target-contrib.sh
@@ -82,6 +82,24 @@ for i in $contribs_to_build; do
     fi | tee output/building-contrib.`basename $i` 
 done
 
+# Otherwise report expected failures:
+HEADER_HAS_BEEN_PRINTED=false
+for dir in contrib/*; do
+  f="$dir/test-passed"
+  if test -f "$f" && grep -i fail "$f" >/dev/null; then
+      if ! $HEADER_HAS_BEEN_PRINTED; then
+          cat <<EOF
+
+Note: Test suite failures which are expected for this combination of
+platform and features have been ignored:
+EOF
+          HEADER_HAS_BEEN_PRINTED=true
+      fi
+      echo "  $dir"
+      (unset IFS; while read line; do echo "    $line"; done <"$f")
+  fi
+done
+
 # Sometimes people used to see the "No tests failed." output from the last
 # DEFTEST in contrib self-tests and think that's all that is. So...
 HEADER_HAS_BEEN_PRINTED=false