;;;; This software is part of the SBCL system. See the README file for
;;;; more information.
;;;;
;;;; While most of SBCL is derived from the CMU CL system, the test
;;;; files (like this one) were written from scratch after the fork
;;;; from CMU CL.
;;;;
;;;; This software is in the public domain and is provided with
;;;; absolutely no warranty. See the COPYING and CREDITS files for
;;;; more information.

(in-package :cl-user)

(use-package :test-util)
(use-package :assertoid)

(defstruct foo)
(defstruct bar x y)

;;; SXHASH and PSXHASH should distribute hash values well over the
;;; space of possible values, so that collisions between the hash
;;; values of unequal objects should be very uncommon. (Except of
;;; course the hash values must collide when the objects are EQUAL or
;;; EQUALP respectively!)
(locally
  ;; In order to better test not-EQ-but-EQUAL and not-EQ-but-EQUALP,
  ;; we'd like to suppress some optimizations.
  (declare (notinline complex float coerce + - expt))
  (flet ((make-sxhash-subtests ()
           (list (cons 0 1)
                 (list 0 1)
                 (cons 1 0)
                 (cons (cons 1 0) (cons 0 0))
                 (cons (list 1 0) (list 0 0))
                 (list (cons 1 0) (list 0 0))
                 (list (cons 0 1) (list 0 0))
                 (list (cons 0 0) (cons 1 0))
                 (list (cons 0 0) (cons 0 1))

                 44     (float 44)     (coerce 44 'double-float)
                 -44    (float -44)    (coerce -44 'double-float)
                 0      (float 0)      (coerce 0 'double-float)
                 -0     (- (float 0))  (- (coerce 0 'double-float))
                 -121   (float -121)   (coerce -121 'double-float)
                 3/4    (float 3/4)    (coerce 3/4 'double-float)
                 -3/4   (float -3/4)   (coerce -3/4 'double-float)
                 45     (float 45)     (coerce 45 'double-float)
                 441/10 (float 441/10) (coerce (float 441/10) 'double-float)

                 (expt 2 33) (expt 2.0 33) (expt 2.0d0 33)
                 (- (expt 1/2 50)) (- (expt 0.5 50)) (- (expt 0.5d0 50))
                 (+ (expt 1/2 50)) (+ (expt 0.5 50)) (+ (expt 0.5d0 50))

                 (complex 1.0 2.0) (complex 1.0d0 2.0)
                 (complex 1.5 -3/2) (complex 1.5 -1.5d0)

                 #\x #\X #\*

                 (copy-seq "foo") (copy-seq "foobar") (copy-seq "foobarbaz")

                 (copy-seq #*)
                 (copy-seq #*0) (copy-seq #*1)
                 (copy-seq #*00) (copy-seq #*10)
                 (copy-seq #*01) (copy-seq #*11)
                 (copy-seq #*10010) (copy-seq #*100101) (bit-not #*01101)
                 (make-array 6 :fill-pointer 6
                             :element-type 'bit :initial-contents #*100101)

                 #'allocate-instance #'no-applicable-method))
         (make-psxhash-extra-subtests ()
           (list (copy-seq "")
                 (copy-seq #*)
                 (copy-seq #())
                 (copy-seq ())
                 (copy-seq '(()))
                 (copy-seq #(()))
                 (copy-seq '(#()))
                 (make-array 3 :fill-pointer 0)
                 (make-array 7 :fill-pointer 0 :element-type 'bit)
                 (make-array 8 :fill-pointer 0 :element-type 'character)
                 (vector (cons 1 0) (cons 0 0))
                 (vector (cons 0 1) (cons 0 0))
                 (vector (cons 0 0) (cons 1 0))
                 (vector (cons 0 0) (cons 0 1))
                 (vector (cons 1 0) (cons 0 0))
                 (vector (cons 0 1) (cons 0 0))
                 (vector (list 0 0) (cons 1 0))
                 (vector (list 0 0) (list 0 1))
                 (vector (vector 1 0) (list 0 0))
                 (vector (vector 0 1) (list 0 0))
                 (vector (vector 0 0) (list 1 0))
                 (vector (vector 0 0) (list 0 1))
                 (vector #*00 #*10)
                 (vector (vector 0 0) (list 0 1.0d0))
                 (vector (vector -0.0d0 0) (list 1.0 0))
                 (vector 1 0 1 0)
                 (vector 0 0 0)
                 (copy-seq #*1010)
                 (copy-seq #*000)
                 (replace (make-array 101
                                      :element-type 'bit
                                      :fill-pointer 4)
                          #*1010)
                 (replace (make-array 14
                                      :element-type '(unsigned-byte 8)
                                      :fill-pointer 3)
                          #*000)
                 (replace (make-array 14
                                      :element-type t
                                      :fill-pointer 3)
                          #*000)
                 (copy-seq "abc")
                 (copy-seq "ABC")
                 (copy-seq "aBc")
                 (copy-seq "abcc")
                 (copy-seq "1001")
                 'abc
                 (vector #\a #\b #\c)
                 (vector 'a 'b 'c)
                 (vector "A" 'b 'c)
                 (replace (make-array 14
                                      :element-type 'character
                                      :fill-pointer 3)
                          "aBc")
                 (replace (make-array 11
                                      :element-type 'character
                                      :fill-pointer 4)
                          "1001")
                 (replace (make-array 12
                                      :element-type 'bit
                                      :fill-pointer 4)
                          #*1001)
                 (replace (make-array 13
                                      :element-type t
                                      :fill-pointer 4)
                          "1001")
                 (replace (make-array 13
                                      :element-type t
                                      :fill-pointer 4)
                          #*1001)
                 ;; FIXME: What about multi-dimensional arrays, hmm?

                 (make-hash-table)
                 (make-hash-table :test 'equal)

                 (make-foo)
                 (make-bar)
                 (make-bar :x (list 1))
                 (make-bar :y (list 1))))
         (t->boolean (x) (if x t nil)))
    (let* (;; Note:
           ;;   * The APPEND noise here is to help more strenuously test
           ;;     not-EQ-but-EQUAL and not-EQ-but-EQUALP cases.
           ;;   * It seems not to be worth the hassle testing SXHASH on
           ;;     values whose structure isn't understood by EQUAL, since
           ;;     we get too many false positives "SXHASHes are equal even
           ;;     though values aren't EQUAL, what a crummy hash function!"
           ;;     FIXME: Or am I misunderstanding the intent of the
           ;;     the SXHASH specification? Perhaps SXHASH is supposed to
           ;;     descend into the structure of objects even when EQUAL
           ;;     doesn't, in order to avoid hashing together things which
           ;;     are guaranteed not to be EQUAL? The definition of SXHASH
           ;;     seems to leave this completely unspecified: should
           ;;     "well-distributed" depend on substructure that EQUAL
           ;;     ignores? For our internal hash tables, the stricter
           ;;     descend-into-the-structure behavior might improve
           ;;     performance even though it's not specified by ANSI. But
           ;;     is it reasonable for users to expect it? Hmm..
           (sxhash-tests (append (make-sxhash-subtests)
                                 (make-sxhash-subtests)))
           (psxhash-tests (append sxhash-tests
                                  (make-psxhash-extra-subtests)
                                  (make-psxhash-extra-subtests))))
      ;; Check that SXHASH compiler transforms give the same results
      ;; as the out-of-line version of SXHASH.
      (let* ((fundef `(lambda ()
                        (list ,@(mapcar (lambda (value)
                                          `(sxhash ',value))
                                        sxhash-tests))))
             (fun (compile nil fundef)))
        (assert (equal (funcall fun)
                       (mapcar #'sxhash sxhash-tests))))
      ;; Note: The tests for SXHASH-equality iff EQUAL and
      ;; PSXHASH-equality iff EQUALP could fail because of an unlucky
      ;; random collision. That's not very likely (since there are
      ;; (EXPT 2 29) possible hash values and only on the order of 100
      ;; test cases, so even with the birthday paradox a collision has
      ;; probability only (/ (EXPT 100 2) (EXPT 2 29)), but it's
      ;; probably worth checking if you are getting a mystifying error
      ;; from this test. (SXHASH values and PSXHASH values don't
      ;; change from run to run, so the random chance of bogus failure
      ;; happens once every time the code is changed in such a way
      ;; that the SXHASH distribution changes, not once every time the
      ;; tests are run.)
      (dolist (i sxhash-tests)
        (declare (notinline funcall))
        (unless (typep (funcall #'sxhash i) '(and fixnum unsigned-byte))
          (error "bad SXHASH behavior for ~S" i))
        (dolist (j sxhash-tests)
          (unless (or (eq (t->boolean (equal i j))
                          (t->boolean (= (sxhash i) (sxhash j))))
                      (and (typep i 'number)
                           (typep j 'number)
                           (= i j)
                           (subtypep (type-of i) (type-of j))
                           (subtypep (type-of j) (type-of i))))
            ;; (If you get a surprising failure here, maybe you were
            ;; just very unlucky; see the notes above.)
            (error "bad SXHASH behavior for ~S ~S" i j))))
      (dolist (i psxhash-tests)
        (unless (typep (sb-int:psxhash i) '(and fixnum unsigned-byte))
          (error "bad PSXHASH behavior for ~S" i))
        (dolist (j psxhash-tests)
          (unless (eq (t->boolean (equalp i j))
                      (t->boolean (= (sb-int:psxhash i) (sb-int:psxhash j))))
            ;; (If you get a surprising failure here, maybe you were
            ;; just very unlucky; see the notes above.)
            (error "bad PSXHASH behavior for ~S ~S" i j))))
      )))

;;; As of sbcl-0.6.12.10, writing hash tables readably should work.
;;; This isn't required by the ANSI standard, but it should be, since
;;; it's well-defined useful behavior which ANSI prohibits the users
;;; from implementing themselves. (ANSI says the users can't define
;;; their own their own PRINT-OBJECT (HASH-TABLE T) methods, and they
;;; can't even wiggle out of it by subclassing HASH-TABLE or STREAM.)
(let ((original-ht (make-hash-table :test 'equal :size 111))
      (original-keys '(1 10 11 400030002 -100000000)))
  (dolist (key original-keys)
    (setf (gethash key original-ht)
          (expt key 4)))
  (let* ((written-ht (with-output-to-string (s)
                       (write original-ht :stream s :readably t)))
         (read-ht (with-input-from-string (s written-ht)
                    (read s))))
    (assert (= (hash-table-count read-ht)
               (hash-table-count original-ht)
               (length original-keys)))
    (assert (eql (hash-table-test original-ht) (hash-table-test read-ht)))
    (assert (eql (hash-table-size original-ht) (hash-table-size read-ht)))
    (dolist (key original-keys)
      (assert (eql (gethash key read-ht)
                   (gethash key original-ht))))))

;;; NIL is both SYMBOL and LIST
(dolist (fun '(sxhash sb-impl::psxhash))
  (assert (= (eval `(,fun nil))
             (funcall fun nil)
             (funcall (compile nil `(lambda (x)
                                      (declare (symbol x))
                                      (,fun x)))
                      nil)
             (funcall (compile nil `(lambda (x)
                                      (declare (list x))
                                      (,fun x)))
                      nil)
             (funcall (compile nil `(lambda (x)
                                      (declare (null x))
                                      (,fun x)))
                      nil))))

;;; This test works reliably on non-conservative platforms and
;;; somewhat reliably on conservative platforms with threads.
(progn

(defparameter *ht* nil)

(defvar *cons-here*)

(declaim (notinline args))
(defun take (&rest args)
  (declare (ignore args)))

(defmacro alloc (&body body)
  "Execute BODY and try to reduce the chance of leaking a conservative root."
  #-sb-thread
  `(multiple-value-prog1
       (progn ,@body)
     (loop repeat 20000 do (setq *cons-here* (cons nil nil)))
     ;; KLUDGE: Clean the argument passing regs.
     (apply #'take (loop repeat 36 collect #'cons)))
  #+sb-thread
  (let ((values (gensym))
        (sem (gensym)))
    `(let ((,sem (sb-thread::make-semaphore))
           ,values)
       (make-join-thread (lambda ()
                           (setq ,values
                                 (multiple-value-list (progn ,@body)))
                           (sb-thread::signal-semaphore ,sem)))
       (sb-thread::wait-on-semaphore ,sem)
       (values-list ,values))))

(with-test (:name (:hash-table :weakness :eql :numbers) :skipped-on '(and :c-stack-is-control-stack (not :sb-thread)))
  (flet ((random-number ()
           (random 1000)))
    (loop for weakness in '(nil :key :value :key-and-value :key-or-value) do
          (let* ((ht (make-hash-table :weakness weakness))
                 (n (alloc (loop repeat 1000
                                 count (let ((key (random-number)))
                                         (if (gethash key ht)
                                             (setf (gethash key ht)
                                                   (random-number))))))))
            (gc :full t)
            (gc :full t)
            (assert (= n (hash-table-count ht)))))))

(defun add-removable-stuff (ht &key (n 100) (size 10))
  (flet ((unique-object ()
           (make-array size :fill-pointer 0)))
    (loop for i below n do
          (multiple-value-bind (key value)
              (ecase (hash-table-weakness ht)
                ((:key) (values (unique-object) i))
                ((:value) (values i (unique-object)))
                ((:key-and-value)
                 (if (zerop (random 2))
                     (values (unique-object) i)
                     (values i (unique-object))))
                ((:key-or-value)
                 (values (unique-object) (unique-object))))
            (setf (gethash key ht) value)))
    (values)))

(defun print-ht (ht &optional (stream t))
  (format stream "Weakness: ~S~%" (sb-impl::hash-table-weakness ht))
  (format stream "Table: ~S~%" (sb-impl::hash-table-table ht))
  (format stream "Next: ~S~%" (sb-impl::hash-table-next-vector ht))
  (format stream "Index: ~S~%" (sb-impl::hash-table-index-vector ht))
  (format stream "Hash: ~S~%" (sb-impl::hash-table-hash-vector ht))
  (force-output stream))

(with-test (:name (:hash-table :weakness :removal) :skipped-on '(and :c-stack-is-control-stack (not :sb-thread)))
  (loop for test in '(eq eql equal equalp) do
        (format t "test: ~A~%" test)
        (loop for weakness in '(:key :value :key-and-value :key-or-value)
              do
              (format t "weakness: ~A~%" weakness)
              (let ((ht (make-hash-table :test 'equal :weakness weakness)))
                (alloc (add-removable-stuff ht :n 117 :size 1))
                (loop for i upfrom 0
                      do (format t "~A. count: ~A~%" i (hash-table-count ht))
                      (force-output)
                      until (zerop (hash-table-count ht))
                      do
                      (when (= i 10)
                        (print-ht ht)
                        #-(or x86 x86-64)
                        (assert nil)
                        ;; With conservative gc the test may not be
                        ;; bullet-proof so it's not an outright
                        ;; failure but a warning.
                        #+(or x86 x86-64)
                        (progn
                          (warn "Weak hash removal test failed for weakness ~A"
                                weakness)
                          (return)))
                      (gc :full t))))))

(with-test (:name (:hash-table :weakness :string-interning) :skipped-on '(and :c-stack-is-control-stack (not :sb-thread)))
  (let ((ht (make-hash-table :test 'equal :weakness :key))
        (s "a"))
    (setf (gethash s ht) s)
    (assert (eq (gethash s ht) s))
    (assert (eq (gethash (copy-seq s) ht) s))))

;;; see if hash_vector is not written when there is none ...
(with-test (:name (:hash-table :weakness :eq) :skipped-on '(and :c-stack-is-control-stack (not :sb-thread)))
  (loop repeat 10 do
        (let ((index (random 2000)))
          (let ((first (+ most-positive-fixnum (mod (* index 31) 9)))
                (n 50000))
            (let ((hash-table (make-hash-table :weakness :key :test 'eq)))
              (dotimes (i n)
                (setf (gethash (+ first i) hash-table) i))
              hash-table)))))

;; used to crash in gc
(with-test (:name (:hash-table :weakness :keep) :skipped-on '(and :c-stack-is-control-stack (not :sb-thread)))
  (loop repeat 2 do
        (let ((h1 (make-hash-table :weakness :key :test #'equal))
              (keep ()))
          (loop for i from 0 to 1000
                for key = i
                for value = (make-array 10000 :fill-pointer 0)
                do
                (push value keep)
                (setf (gethash key h1) value))
          (sb-ext:gc :full t))))

)

;;; DEFINE-HASH-TABLE-TEST

(defstruct custom-hash-key name)
(defun custom-hash-test (x y)
  (equal (custom-hash-key-name x)
         (custom-hash-key-name y)))
(defun custom-hash-hash (x)
  (sxhash (custom-hash-key-name x)))
(define-hash-table-test custom-hash-test custom-hash-hash)
(with-test (:name define-hash-table-test.1)
  (let ((table (make-hash-table :test 'custom-hash-test)))
    (setf (gethash (make-custom-hash-key :name "foo") table) :foo)
    (setf (gethash (make-custom-hash-key :name "bar") table) :bar)
    (assert (eq :foo (gethash (make-custom-hash-key :name "foo") table)))
    (assert (eq :bar (gethash (make-custom-hash-key :name "bar") table)))
    (assert (eq 'custom-hash-test (hash-table-test table))))
  (let ((table (make-hash-table :test #'custom-hash-test)))
    (setf (gethash (make-custom-hash-key :name "foo") table) :foo)
    (setf (gethash (make-custom-hash-key :name "bar") table) :bar)
    (assert (eq :foo (gethash (make-custom-hash-key :name "foo") table)))
    (assert (eq :bar (gethash (make-custom-hash-key :name "bar") table)))
    (assert (eq 'custom-hash-test (hash-table-test table)))))


(defun head-eql (x y)
  (every #'eql (subseq x 0 3) (subseq y 0 3)))
(define-hash-table-test head-eql
    (lambda (x)
      (logand most-positive-fixnum
              (reduce #'+ (map 'list #'sxhash (subseq x 0 3))))))
(with-test (:name define-hash-table-test.2)
  (let ((table (make-hash-table :test 'head-eql)))
    (setf (gethash #(1 2 3 4) table) :|123|)
    (setf (gethash '(2 3 4 7) table) :|234|)
    (setf (gethash "foobar" table) :foo)
    (assert (eq :|123| (gethash '(1 2 3 ! 6) table)))
    (assert (eq :|234| (gethash #(2 3 4 0 2 1 a) table)))
    (assert (eq :foo (gethash '(#\f #\o #\o 1 2 3) table)))
    (assert (eq 'head-eql (hash-table-test table))))
  (let ((table (make-hash-table :test #'head-eql)))
    (setf (gethash #(1 2 3 4) table) :|123|)
    (setf (gethash '(2 3 4 7) table) :|234|)
    (setf (gethash "foobar" table) :foo)
    (assert (eq :|123| (gethash '(1 2 3 ! 6) table)))
    (assert (eq :|234| (gethash #(2 3 4 0 2 1 a) table)))
    (assert (eq :foo (gethash '(#\f #\o #\o 1 2 3) table)))
    (assert (eq 'head-eql (hash-table-test table)))))

(with-test (:name make-hash-table/hash-fun)
  (let ((table (make-hash-table
                :test #'=
                :hash-function (lambda (x)
                                 (sxhash (coerce (abs x) 'double-float))))))
    (incf (gethash 1 table 0))
    (incf (gethash 1.0f0 table))
    (incf (gethash 1.0d0 table))
    (incf (gethash (complex 1.0f0 0.0f0) table))
    (incf (gethash (complex 1.0d0 0.0d0) table))
    (assert (= 5 (gethash 1 table)))
    (assert (eq '= (hash-table-test table)))))

;;; success