X-Git-Url: http://repo.macrolet.net/gitweb/?a=blobdiff_plain;f=tests%2Fexternal-format.impure.lisp;h=aabdf03a6ff0bb0ffa4e7c7a819ce1ba25b9c47b;hb=31481ad7a664585715d60fbdeee153c5c5343400;hp=9673578e806c0f2a08e167ae8b82db835e87a32d;hpb=5cd0fc84df83d1b3321b7fc969843207721de429;p=sbcl.git

diff --git a/tests/external-format.impure.lisp b/tests/external-format.impure.lisp
index 9673578..aabdf03 100644
--- a/tests/external-format.impure.lisp
+++ b/tests/external-format.impure.lisp
@@ -10,7 +10,7 @@
 ;;;; While most of SBCL is derived from the CMU CL system, the test
 ;;;; files (like this one) were written from scratch after the fork
 ;;;; from CMU CL.
-;;;; 
+;;;;
 ;;;; This software is in the public domain and is provided with
 ;;;; absolutely no warranty. See the COPYING and CREDITS files for
 ;;;; more information.
@@ -19,25 +19,223 @@
   (let ((nxf (gensym)))
     `(dolist (,nxf sb-impl::*external-formats* ,result)
        (let ((,xf (first (first ,nxf))))
-	 ,@body))))
+         ,@body))))
 
 (do-external-formats (xf)
   (with-open-file (s "/dev/null" :direction :input :external-format xf)
     (assert (eq (read-char s nil s) s))))
 
+;;; Test standard character read-write equivalency over all external formats.
+(let ((standard-characters "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!$\"'(),_-./:;?+<=>#%&*@[\\]{|}`^~"))
+  (do-external-formats (xf)
+    (with-open-file (s "external-format-test.txt" :direction :output
+                     :if-exists :supersede :external-format xf)
+      (loop for character across standard-characters
+            do (write-char character s)))
+    (with-open-file (s "external-format-test.txt" :direction :input
+                     :external-format xf)
+      (loop for character across standard-characters
+            do (assert (eql (read-char s) character))))))
+
+(delete-file "external-format-test.txt")
+#-sb-unicode
+(progn
+  (test-util:report-test-status)
+  (sb-ext:quit :unix-status 104))
+
+;;; Test UTF-8 writing and reading of 1, 2, 3 and 4 octet characters with
+;;; all possible offsets. Tests for buffer edge bugs. fd-stream buffers are
+;;; 4096 wide.
+(dotimes (width-1 4)
+  (let ((character (code-char (elt '(1 #x81 #x801 #x10001) width-1))))
+    (dotimes (offset (+ width-1 1))
+      (with-open-file (s "external-format-test.txt" :direction :output
+                       :if-exists :supersede :external-format :utf-8)
+        (dotimes (n offset)
+          (write-char #\a s))
+        (dotimes (n 4097)
+          (write-char character s)))
+      (with-open-file (s "external-format-test.txt" :direction :input
+                       :external-format :utf-8)
+        (dotimes (n offset)
+          (assert (eql (read-char s) #\a)))
+        (dotimes (n 4097)
+          (assert (eql (read-char s) character)))
+        (assert (eql (read-char s nil s) s))))))
+
+;;; Test character decode restarts.
+(with-open-file (s "external-format-test.txt" :direction :output
+                 :if-exists :supersede :element-type '(unsigned-byte 8))
+  (write-byte 65 s)
+  (write-byte 66 s)
+  (write-byte #xe0 s)
+  (write-byte 67 s))
+(with-open-file (s "external-format-test.txt" :direction :input
+                 :external-format :utf-8)
+  (handler-bind
+      ((sb-int:character-decoding-error #'(lambda (decoding-error)
+                                            (declare (ignore decoding-error))
+                                            (invoke-restart
+                                             'sb-int:attempt-resync))))
+    (assert (equal (read-line s nil s) "ABC"))
+    (assert (equal (read-line s nil s) s))))
+(with-open-file (s "external-format-test.txt" :direction :input
+                 :external-format :utf-8)
+  (handler-bind
+      ((sb-int:character-decoding-error #'(lambda (decoding-error)
+                                            (declare (ignore decoding-error))
+                                            (invoke-restart
+                                             'sb-int:force-end-of-file))))
+    (assert (equal (read-line s nil s) "AB"))
+    (assert (equal (read-line s nil s) s))))
+
+;;; And again with more data to account for buffering (this was briefly)
+;;; broken in early 0.9.6.
+(with-open-file (s "external-format-test.txt" :direction :output
+                 :if-exists :supersede :element-type '(unsigned-byte 8))
+  (let ((a (make-array 50
+                       :element-type '(unsigned-byte 64)
+                       :initial-contents (map 'list #'char-code
+                                              "1234567890123456789012345678901234567890123456789."))))
+    (setf (aref a 49) (char-code #\Newline))
+    (dotimes (i 40)
+      (write-sequence a s))
+    (write-byte #xe0 s)
+    (dotimes (i 40)
+      (write-sequence a s))))
+(with-test (:name (:character-decode-large :attempt-resync))
+  (with-open-file (s "external-format-test.txt" :direction :input
+                     :external-format :utf-8)
+    (handler-bind
+        ((sb-int:character-decoding-error #'(lambda (decoding-error)
+                                              (declare (ignore decoding-error))
+                                              (invoke-restart
+                                               'sb-int:attempt-resync)))
+         ;; The failure mode is an infinite loop, add a timeout to detetct it.
+         (sb-ext:timeout (lambda () (error "Timeout"))))
+      (sb-ext:with-timeout 5
+        (dotimes (i 80)
+          (assert (equal (read-line s nil s)
+                         "1234567890123456789012345678901234567890123456789")))))))
+(with-test (:name (:character-decode-large :force-end-of-file)
+            :fails-on :sbcl)
+  (with-open-file (s "external-format-test.txt" :direction :input
+                     :external-format :utf-8)
+    (handler-bind
+        ((sb-int:character-decoding-error #'(lambda (decoding-error)
+                                              (declare (ignore decoding-error))
+                                              (invoke-restart
+                                               'sb-int:force-end-of-file)))
+         ;; The failure mode is an infinite loop, add a timeout to detetct it.
+         (sb-ext:timeout (lambda () (error "Timeout"))))
+      (sb-ext:with-timeout 5
+        (dotimes (i 80)
+          (assert (equal (read-line s nil s)
+                         "1234567890123456789012345678901234567890123456789")))
+        (assert (equal (read-line s nil s) s))))))
+
+;;; Test character encode restarts.
+(with-open-file (s "external-format-test.txt" :direction :output
+                 :if-exists :supersede :external-format :latin-1)
+  (handler-bind
+      ((sb-int:character-encoding-error #'(lambda (encoding-error)
+                                            (declare (ignore encoding-error))
+                                            (invoke-restart
+                                             'sb-impl::output-nothing))))
+    (write-char #\A s)
+    (write-char #\B s)
+    (write-char (code-char 322) s)
+    (write-char #\C s)))
+(with-open-file (s "external-format-test.txt" :direction :input
+                 :external-format :latin-1)
+  (assert (equal (read-line s nil s) "ABC"))
+  (assert (equal (read-line s nil s) s)))
+
+(with-open-file (s "external-format-test.txt" :direction :output
+                 :if-exists :supersede :external-format :latin-1)
+  (handler-bind
+      ((sb-int:character-encoding-error #'(lambda (encoding-error)
+                                            (declare (ignore encoding-error))
+                                            (invoke-restart
+                                             'sb-impl::output-nothing))))
+    (let ((string (make-array 4 :element-type 'character
+                              :initial-contents `(#\A #\B ,(code-char 322)
+                                                      #\C))))
+      (write-string string s))))
+(with-open-file (s "external-format-test.txt" :direction :input
+                 :external-format :latin-1)
+  (assert (equal (read-line s nil s) "ABC"))
+  (assert (equal (read-line s nil s) s)))
+
+;;; Test skipping character-decode-errors in comments.
 (let ((s (open "external-format-test.lisp" :direction :output
-	       :if-exists :supersede :external-format :latin-1)))
+               :if-exists :supersede :external-format :latin-1)))
   (unwind-protect
        (progn
-	 (write-string ";;; ABCD" s)
-	 (write-char (code-char 233) s)
-	 (terpri s)
-	 (close s)
-	 (compile-file "external-format-test.lisp" :external-format :utf-8))
+         (write-string ";;; ABCD" s)
+         (write-char (code-char 233) s)
+         (terpri s)
+         (close s)
+         (compile-file "external-format-test.lisp" :external-format :utf-8))
     (delete-file s)
     (let ((p (probe-file (compile-file-pathname "external-format-test.lisp"))))
       (when p
-	(delete-file p)))))
+        (delete-file p)))))
+
+
+;;;; KOI8-R external format
+(with-open-file (s "external-format-test.txt" :direction :output
+                 :if-exists :supersede :external-format :koi8-r)
+  (write-char (code-char #xB0) s)
+  (assert (eq
+           (handler-case
+               (progn
+                 (write-char (code-char #xBAAD) s)
+                 :bad)
+             (sb-int:character-encoding-error ()
+               :good))
+           :good)))
+(with-open-file (s "external-format-test.txt" :direction :input
+                 :element-type '(unsigned-byte 8))
+  (let ((byte (read-byte s)))
+    (assert (= (eval byte) #x9C))))
+(with-open-file (s "external-format-test.txt" :direction :input
+                 :external-format :koi8-r)
+  (let ((char (read-char s)))
+    (assert (= (char-code (eval char)) #xB0))))
+(delete-file "external-format-test.txt")
+
+;;; tests of FILE-STRING-LENGTH
+(let ((standard-characters "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!$\"'(),_-./:;?+<=>#%&*@[\\]{|}`^~"))
+  (do-external-formats (xf)
+    (with-open-file (s "external-format-test.txt" :direction :output
+                       :external-format xf)
+      (loop for x across standard-characters
+            for position = (file-position s)
+            for char-length = (file-string-length s x)
+            do (write-char x s)
+            do (assert (= (file-position s) (+ position char-length))))
+      (let ((position (file-position s))
+            (string-length (file-string-length s standard-characters)))
+        (write-string standard-characters s)
+        (assert (= (file-position s) (+ position string-length)))))
+    (delete-file "external-format-test.txt")))
 
-(sb-ext:quit :unix-status 104)
-    
\ No newline at end of file
+(let ((char-codes '(0 1 255 256 511 512 1023 1024 2047 2048 4095 4096
+                    8191 8192 16383 16384 32767 32768 65535 65536 131071
+                    131072 262143 262144)))
+  (with-open-file (s "external-format-test.txt" :direction :output
+                     :external-format :utf-8)
+    (dolist (code char-codes)
+      (let* ((char (code-char code))
+             (position (file-position s))
+             (char-length (file-string-length s char)))
+        (write-char char s)
+        (assert (= (file-position s) (+ position char-length)))))
+    (let* ((string (map 'string #'code-char char-codes))
+           (position (file-position s))
+           (string-length (file-string-length s string)))
+      (write-string string s)
+      (assert (= (file-position s) (+ position string-length))))))
+
+;;;; success
\ No newline at end of file