X-Git-Url: http://repo.macrolet.net/gitweb/?a=blobdiff_plain;f=src%2Fcompiler%2Fx86-64%2Ffloat.lisp;h=6f0a27f70721e180ad383d9cd41c5a517a2f86cf;hb=e2ae57e6839f264cd6c1b6bea66e7a373122db85;hp=6a15a523f37f8355f3d59f41eeecf4ea4806a66a;hpb=4ebdc81b1a9c6dbed6e98b112afc8dd32b17a2dd;p=sbcl.git diff --git a/src/compiler/x86-64/float.lisp b/src/compiler/x86-64/float.lisp index 6a15a52..6f0a27f 100644 --- a/src/compiler/x86-64/float.lisp +++ b/src/compiler/x86-64/float.lisp @@ -12,285 +12,154 @@ (in-package "SB!VM") (macrolet ((ea-for-xf-desc (tn slot) - `(make-ea - :dword :base ,tn - :disp (- (* ,slot n-word-bytes) - other-pointer-lowtag)))) - (defun ea-for-sf-desc (tn) - (ea-for-xf-desc tn single-float-value-slot)) + `(make-ea + :qword :base ,tn + :disp (- (* ,slot n-word-bytes) + other-pointer-lowtag)))) (defun ea-for-df-desc (tn) (ea-for-xf-desc tn double-float-value-slot)) ;; complex floats + (defun ea-for-csf-data-desc (tn) + (ea-for-xf-desc tn complex-single-float-data-slot)) (defun ea-for-csf-real-desc (tn) - (ea-for-xf-desc tn complex-single-float-real-slot)) + (ea-for-xf-desc tn complex-single-float-data-slot)) (defun ea-for-csf-imag-desc (tn) - (ea-for-xf-desc tn complex-single-float-imag-slot)) + (ea-for-xf-desc tn (+ complex-single-float-data-slot 1/2))) + + (defun ea-for-cdf-data-desc (tn) + (ea-for-xf-desc tn complex-double-float-real-slot)) (defun ea-for-cdf-real-desc (tn) (ea-for-xf-desc tn complex-double-float-real-slot)) (defun ea-for-cdf-imag-desc (tn) (ea-for-xf-desc tn complex-double-float-imag-slot))) (macrolet ((ea-for-xf-stack (tn kind) - `(make-ea - :dword :base rbp-tn - :disp (- (* (+ (tn-offset ,tn) - (ecase ,kind (:single 1) (:double 2) (:long 3))) - n-word-bytes))))) + (declare (ignore kind)) + `(make-ea + :qword :base rbp-tn + :disp (frame-byte-offset (tn-offset ,tn))))) (defun ea-for-sf-stack (tn) (ea-for-xf-stack tn :single)) (defun ea-for-df-stack (tn) (ea-for-xf-stack tn :double))) -;;; Telling the FPU to wait is required in order to make signals occur -;;; at the expected place, but naturally slows things down. -;;; -;;; NODE is the node whose compilation policy controls the decision -;;; whether to just blast through carelessly or carefully emit wait -;;; instructions and whatnot. -;;; -;;; NOTE-NEXT-INSTRUCTION, if supplied, is to be passed to -;;; #'NOTE-NEXT-INSTRUCTION. -(defun maybe-fp-wait (node &optional note-next-instruction) - (when (policy node (or (= debug 3) (> safety speed)))) - (when note-next-instruction - (note-next-instruction note-next-instruction :internal-error)) - (inst wait)) - ;;; complex float stack EAs (macrolet ((ea-for-cxf-stack (tn kind slot &optional base) - `(make-ea - :dword :base ,base - :disp (- (* (+ (tn-offset ,tn) - (* (ecase ,kind - (:single 1) - (:double 2) - (:long 3)) - (ecase ,slot (:real 1) (:imag 2)))) - n-word-bytes))))) + `(make-ea + :qword :base ,base + :disp (frame-byte-offset + (+ (tn-offset ,tn) + (cond ((= (tn-offset ,base) rsp-offset) + sp->fp-offset) + ((= (tn-offset ,base) rbp-offset) + 0) + (t (error "Unexpected offset."))) + (ecase ,kind + (:single + (ecase ,slot + (:real 0) + (:imag -1/2))) + (:double + (ecase ,slot + (:real 1) + (:imag 0))))))))) + (defun ea-for-csf-data-stack (tn &optional (base rbp-tn)) + (ea-for-cxf-stack tn :single :real base)) (defun ea-for-csf-real-stack (tn &optional (base rbp-tn)) (ea-for-cxf-stack tn :single :real base)) (defun ea-for-csf-imag-stack (tn &optional (base rbp-tn)) (ea-for-cxf-stack tn :single :imag base)) + + (defun ea-for-cdf-data-stack (tn &optional (base rbp-tn)) + (ea-for-cxf-stack tn :double :real base)) (defun ea-for-cdf-real-stack (tn &optional (base rbp-tn)) (ea-for-cxf-stack tn :double :real base)) (defun ea-for-cdf-imag-stack (tn &optional (base rbp-tn)) (ea-for-cxf-stack tn :double :imag base))) - -;;; Abstract out the copying of a FP register to the FP stack top, and -;;; provide two alternatives for its implementation. Note: it's not -;;; necessary to distinguish between a single or double register move -;;; here. -;;; -;;; Using a Pop then load. -(defun copy-fp-reg-to-fr0 (reg) - (aver (not (zerop (tn-offset reg)))) - (inst fstp fr0-tn) - (inst fld (make-random-tn :kind :normal - :sc (sc-or-lose 'double-reg) - :offset (1- (tn-offset reg))))) -;;; Using Fxch then Fst to restore the original reg contents. -#+nil -(defun copy-fp-reg-to-fr0 (reg) - (aver (not (zerop (tn-offset reg)))) - (inst fxch reg) - (inst fst reg)) - ;;;; move functions ;;; X is source, Y is destination. + +(define-move-fun (load-fp-zero 1) (vop x y) + ((fp-single-zero) (single-reg) + (fp-double-zero) (double-reg) + (fp-complex-single-zero) (complex-single-reg) + (fp-complex-double-zero) (complex-double-reg)) + (identity x) + (sc-case y + ((single-reg complex-single-reg) (inst xorps y y)) + ((double-reg complex-double-reg) (inst xorpd y y)))) + +(define-move-fun (load-fp-immediate 1) (vop x y) + ((fp-single-immediate) (single-reg) + (fp-double-immediate) (double-reg) + (fp-complex-single-immediate) (complex-single-reg) + (fp-complex-double-immediate) (complex-double-reg)) + (let ((x (register-inline-constant (tn-value x)))) + (sc-case y + (single-reg (inst movss y x)) + (double-reg (inst movsd y x)) + (complex-single-reg (inst movq y x)) + (complex-double-reg (inst movapd y x))))) + (define-move-fun (load-single 2) (vop x y) ((single-stack) (single-reg)) - (with-empty-tn@fp-top(y) - (inst fld (ea-for-sf-stack x)))) + (inst movss y (ea-for-sf-stack x))) (define-move-fun (store-single 2) (vop x y) ((single-reg) (single-stack)) - (cond ((zerop (tn-offset x)) - (inst fst (ea-for-sf-stack y))) - (t - (inst fxch x) - (inst fst (ea-for-sf-stack y)) - ;; This may not be necessary as ST0 is likely invalid now. - (inst fxch x)))) + (inst movss (ea-for-sf-stack y) x)) (define-move-fun (load-double 2) (vop x y) ((double-stack) (double-reg)) - (with-empty-tn@fp-top(y) - (inst fldd (ea-for-df-stack x)))) + (inst movsd y (ea-for-df-stack x))) (define-move-fun (store-double 2) (vop x y) ((double-reg) (double-stack)) - (cond ((zerop (tn-offset x)) - (inst fstd (ea-for-df-stack y))) - (t - (inst fxch x) - (inst fstd (ea-for-df-stack y)) - ;; This may not be necessary as ST0 is likely invalid now. - (inst fxch x)))) - - - -;;; The i387 has instructions to load some useful constants. This -;;; doesn't save much time but might cut down on memory access and -;;; reduce the size of the constant vector (CV). Intel claims they are -;;; stored in a more precise form on chip. Anyhow, might as well use -;;; the feature. It can be turned off by hacking the -;;; "immediate-constant-sc" in vm.lisp. -(eval-when (:compile-toplevel :execute) - (setf *read-default-float-format* 'double-float)) -(define-move-fun (load-fp-constant 2) (vop x y) - ((fp-constant) (single-reg double-reg)) - (let ((value (sb!c::constant-value (sb!c::tn-leaf x)))) - (with-empty-tn@fp-top(y) - (cond ((zerop value) - (inst fldz)) - ((= value 1e0) - (inst fld1)) - ((= value (coerce pi *read-default-float-format*)) - (inst fldpi)) - ((= value (log 10e0 2e0)) - (inst fldl2t)) - ((= value (log 2.718281828459045235360287471352662e0 2e0)) - (inst fldl2e)) - ((= value (log 2e0 10e0)) - (inst fldlg2)) - ((= value (log 2e0 2.718281828459045235360287471352662e0)) - (inst fldln2)) - (t (warn "ignoring bogus i387 constant ~A" value)))))) + (inst movsd (ea-for-df-stack y) x)) + (eval-when (:compile-toplevel :execute) (setf *read-default-float-format* 'single-float)) ;;;; complex float move functions -(defun complex-single-reg-real-tn (x) - (make-random-tn :kind :normal :sc (sc-or-lose 'single-reg) - :offset (tn-offset x))) -(defun complex-single-reg-imag-tn (x) - (make-random-tn :kind :normal :sc (sc-or-lose 'single-reg) - :offset (1+ (tn-offset x)))) - -(defun complex-double-reg-real-tn (x) - (make-random-tn :kind :normal :sc (sc-or-lose 'double-reg) - :offset (tn-offset x))) -(defun complex-double-reg-imag-tn (x) - (make-random-tn :kind :normal :sc (sc-or-lose 'double-reg) - :offset (1+ (tn-offset x)))) - ;;; X is source, Y is destination. (define-move-fun (load-complex-single 2) (vop x y) ((complex-single-stack) (complex-single-reg)) - (let ((real-tn (complex-single-reg-real-tn y))) - (with-empty-tn@fp-top (real-tn) - (inst fld (ea-for-csf-real-stack x)))) - (let ((imag-tn (complex-single-reg-imag-tn y))) - (with-empty-tn@fp-top (imag-tn) - (inst fld (ea-for-csf-imag-stack x))))) + (inst movq y (ea-for-csf-data-stack x))) (define-move-fun (store-complex-single 2) (vop x y) ((complex-single-reg) (complex-single-stack)) - (let ((real-tn (complex-single-reg-real-tn x))) - (cond ((zerop (tn-offset real-tn)) - (inst fst (ea-for-csf-real-stack y))) - (t - (inst fxch real-tn) - (inst fst (ea-for-csf-real-stack y)) - (inst fxch real-tn)))) - (let ((imag-tn (complex-single-reg-imag-tn x))) - (inst fxch imag-tn) - (inst fst (ea-for-csf-imag-stack y)) - (inst fxch imag-tn))) + (inst movq (ea-for-csf-data-stack y) x)) (define-move-fun (load-complex-double 2) (vop x y) ((complex-double-stack) (complex-double-reg)) - (let ((real-tn (complex-double-reg-real-tn y))) - (with-empty-tn@fp-top(real-tn) - (inst fldd (ea-for-cdf-real-stack x)))) - (let ((imag-tn (complex-double-reg-imag-tn y))) - (with-empty-tn@fp-top(imag-tn) - (inst fldd (ea-for-cdf-imag-stack x))))) + (inst movupd y (ea-for-cdf-data-stack x))) (define-move-fun (store-complex-double 2) (vop x y) ((complex-double-reg) (complex-double-stack)) - (let ((real-tn (complex-double-reg-real-tn x))) - (cond ((zerop (tn-offset real-tn)) - (inst fstd (ea-for-cdf-real-stack y))) - (t - (inst fxch real-tn) - (inst fstd (ea-for-cdf-real-stack y)) - (inst fxch real-tn)))) - (let ((imag-tn (complex-double-reg-imag-tn x))) - (inst fxch imag-tn) - (inst fstd (ea-for-cdf-imag-stack y)) - (inst fxch imag-tn))) - + (inst movupd (ea-for-cdf-data-stack y) x)) ;;;; move VOPs ;;; float register to register moves -(define-vop (float-move) - (:args (x)) - (:results (y)) - (:note "float move") - (:generator 0 - (unless (location= x y) - (cond ((zerop (tn-offset y)) - (copy-fp-reg-to-fr0 x)) - ((zerop (tn-offset x)) - (inst fstd y)) - (t - (inst fxch x) - (inst fstd y) - (inst fxch x)))))) - -(define-vop (single-move float-move) - (:args (x :scs (single-reg) :target y :load-if (not (location= x y)))) - (:results (y :scs (single-reg) :load-if (not (location= x y))))) -(define-move-vop single-move :move (single-reg) (single-reg)) - -(define-vop (double-move float-move) - (:args (x :scs (double-reg) :target y :load-if (not (location= x y)))) - (:results (y :scs (double-reg) :load-if (not (location= x y))))) -(define-move-vop double-move :move (double-reg) (double-reg)) - -;;; complex float register to register moves -(define-vop (complex-float-move) - (:args (x :target y :load-if (not (location= x y)))) - (:results (y :load-if (not (location= x y)))) - (:note "complex float move") - (:generator 0 - (unless (location= x y) - ;; Note the complex-float-regs are aligned to every second - ;; float register so there is not need to worry about overlap. - (let ((x-real (complex-double-reg-real-tn x)) - (y-real (complex-double-reg-real-tn y))) - (cond ((zerop (tn-offset y-real)) - (copy-fp-reg-to-fr0 x-real)) - ((zerop (tn-offset x-real)) - (inst fstd y-real)) - (t - (inst fxch x-real) - (inst fstd y-real) - (inst fxch x-real)))) - (let ((x-imag (complex-double-reg-imag-tn x)) - (y-imag (complex-double-reg-imag-tn y))) - (inst fxch x-imag) - (inst fstd y-imag) - (inst fxch x-imag))))) - -(define-vop (complex-single-move complex-float-move) - (:args (x :scs (complex-single-reg) :target y - :load-if (not (location= x y)))) - (:results (y :scs (complex-single-reg) :load-if (not (location= x y))))) -(define-move-vop complex-single-move :move - (complex-single-reg) (complex-single-reg)) - -(define-vop (complex-double-move complex-float-move) - (:args (x :scs (complex-double-reg) - :target y :load-if (not (location= x y)))) - (:results (y :scs (complex-double-reg) :load-if (not (location= x y))))) -(define-move-vop complex-double-move :move - (complex-double-reg) (complex-double-reg)) +(macrolet ((frob (vop sc) + `(progn + (define-vop (,vop) + (:args (x :scs (,sc) + :target y + :load-if (not (location= x y)))) + (:results (y :scs (,sc) + :load-if (not (location= x y)))) + (:note "float move") + (:generator 0 + (move y x))) + (define-move-vop ,vop :move (,sc) (,sc))))) + (frob single-move single-reg) + (frob double-move double-reg) + (frob complex-single-move complex-single-reg) + (frob complex-double-move complex-double-reg)) ;;; Move from float to a descriptor reg. allocating a new float @@ -298,14 +167,12 @@ (define-vop (move-from-single) (:args (x :scs (single-reg) :to :save)) (:results (y :scs (descriptor-reg))) - (:node-var node) (:note "float to pointer coercion") - (:generator 13 - (with-fixed-allocation (y - single-float-widetag - single-float-size node) - (with-tn@fp-top(x) - (inst fst (ea-for-sf-desc y)))))) + (:generator 4 + (inst movd y x) + (inst shl y 32) + (inst or y single-float-widetag))) + (define-move-vop move-from-single :move (single-reg) (descriptor-reg)) @@ -316,43 +183,38 @@ (:note "float to pointer coercion") (:generator 13 (with-fixed-allocation (y - double-float-widetag - double-float-size - node) - (with-tn@fp-top(x) - (inst fstd (ea-for-df-desc y)))))) + double-float-widetag + double-float-size + node) + (inst movsd (ea-for-df-desc y) x)))) (define-move-vop move-from-double :move (double-reg) (descriptor-reg)) -(define-vop (move-from-fp-constant) - (:args (x :scs (fp-constant))) - (:results (y :scs (descriptor-reg))) - (:generator 2 - (ecase (sb!c::constant-value (sb!c::tn-leaf x)) - (0f0 (load-symbol-value y *fp-constant-0f0*)) - (1f0 (load-symbol-value y *fp-constant-1f0*)) - (0d0 (load-symbol-value y *fp-constant-0d0*)) - (1d0 (load-symbol-value y *fp-constant-1d0*))))) -(define-move-vop move-from-fp-constant :move - (fp-constant) (descriptor-reg)) - ;;; Move from a descriptor to a float register. (define-vop (move-to-single) - (:args (x :scs (descriptor-reg))) - (:results (y :scs (single-reg))) + (:args (x :scs (descriptor-reg) :target tmp)) + (:temporary (:sc unsigned-reg) tmp) + (:results (y :scs (single-reg single-stack))) (:note "pointer to float coercion") (:generator 2 - (with-empty-tn@fp-top(y) - (inst fld (ea-for-sf-desc x))))) -(define-move-vop move-to-single :move (descriptor-reg) (single-reg)) + (move tmp x) + (inst shr tmp 32) + (sc-case y + (single-reg + (inst movd y tmp)) + (single-stack + (let ((slot (make-ea :dword :base rbp-tn + :disp (frame-byte-offset (tn-offset y))))) + (inst mov slot (reg-in-size tmp :dword))))))) + +(define-move-vop move-to-single :move (descriptor-reg) (single-reg single-stack)) (define-vop (move-to-double) (:args (x :scs (descriptor-reg))) (:results (y :scs (double-reg))) (:note "pointer to float coercion") (:generator 2 - (with-empty-tn@fp-top(y) - (inst fldd (ea-for-df-desc x))))) + (inst movsd y (ea-for-df-desc x)))) (define-move-vop move-to-double :move (descriptor-reg) (double-reg)) @@ -365,15 +227,10 @@ (:note "complex float to pointer coercion") (:generator 13 (with-fixed-allocation (y - complex-single-float-widetag - complex-single-float-size - node) - (let ((real-tn (complex-single-reg-real-tn x))) - (with-tn@fp-top(real-tn) - (inst fst (ea-for-csf-real-desc y)))) - (let ((imag-tn (complex-single-reg-imag-tn x))) - (with-tn@fp-top(imag-tn) - (inst fst (ea-for-csf-imag-desc y))))))) + complex-single-float-widetag + complex-single-float-size + node) + (inst movq (ea-for-csf-data-desc y) x)))) (define-move-vop move-from-complex-single :move (complex-single-reg) (descriptor-reg)) @@ -384,39 +241,29 @@ (:note "complex float to pointer coercion") (:generator 13 (with-fixed-allocation (y - complex-double-float-widetag - complex-double-float-size - node) - (let ((real-tn (complex-double-reg-real-tn x))) - (with-tn@fp-top(real-tn) - (inst fstd (ea-for-cdf-real-desc y)))) - (let ((imag-tn (complex-double-reg-imag-tn x))) - (with-tn@fp-top(imag-tn) - (inst fstd (ea-for-cdf-imag-desc y))))))) + complex-double-float-widetag + complex-double-float-size + node) + (inst movapd (ea-for-cdf-data-desc y) x)))) (define-move-vop move-from-complex-double :move (complex-double-reg) (descriptor-reg)) ;;; Move from a descriptor to a complex float register. (macrolet ((frob (name sc format) - `(progn - (define-vop (,name) - (:args (x :scs (descriptor-reg))) - (:results (y :scs (,sc))) - (:note "pointer to complex float coercion") - (:generator 2 - (let ((real-tn (complex-double-reg-real-tn y))) - (with-empty-tn@fp-top(real-tn) - ,@(ecase format - (:single '((inst fld (ea-for-csf-real-desc x)))) - (:double '((inst fldd (ea-for-cdf-real-desc x))))))) - (let ((imag-tn (complex-double-reg-imag-tn y))) - (with-empty-tn@fp-top(imag-tn) - ,@(ecase format - (:single '((inst fld (ea-for-csf-imag-desc x)))) - (:double '((inst fldd (ea-for-cdf-imag-desc x))))))))) - (define-move-vop ,name :move (descriptor-reg) (,sc))))) - (frob move-to-complex-single complex-single-reg :single) - (frob move-to-complex-double complex-double-reg :double)) + `(progn + (define-vop (,name) + (:args (x :scs (descriptor-reg))) + (:results (y :scs (,sc))) + (:note "pointer to complex float coercion") + (:generator 2 + ,(ecase format + (:single + '(inst movq y (ea-for-csf-data-desc x))) + (:double + '(inst movapd y (ea-for-cdf-data-desc x)))))) + (define-move-vop ,name :move (descriptor-reg) (,sc))))) + (frob move-to-complex-single complex-single-reg :single) + (frob move-to-complex-double complex-double-reg :double)) ;;;; the move argument vops ;;;; @@ -425,112 +272,60 @@ ;;; the general MOVE-ARG VOP (macrolet ((frob (name sc stack-sc format) - `(progn - (define-vop (,name) - (:args (x :scs (,sc) :target y) - (fp :scs (any-reg) - :load-if (not (sc-is y ,sc)))) - (:results (y)) - (:note "float argument move") - (:generator ,(case format (:single 2) (:double 3) (:long 4)) - (sc-case y - (,sc - (unless (location= x y) - (cond ((zerop (tn-offset y)) - (copy-fp-reg-to-fr0 x)) - ((zerop (tn-offset x)) - (inst fstd y)) - (t - (inst fxch x) - (inst fstd y) - (inst fxch x))))) - (,stack-sc - (if (= (tn-offset fp) esp-offset) - (let* ((offset (* (tn-offset y) n-word-bytes)) - (ea (make-ea :dword :base fp :disp offset))) - (with-tn@fp-top(x) - ,@(ecase format - (:single '((inst fst ea))) - (:double '((inst fstd ea)))))) - (let ((ea (make-ea - :dword :base fp - :disp (- (* (+ (tn-offset y) - ,(case format - (:single 1) - (:double 2) - (:long 3))) - n-word-bytes))))) - (with-tn@fp-top(x) - ,@(ecase format - (:single '((inst fst ea))) - (:double '((inst fstd ea))))))))))) - (define-move-vop ,name :move-arg - (,sc descriptor-reg) (,sc))))) + `(progn + (define-vop (,name) + (:args (x :scs (,sc) :target y) + (fp :scs (any-reg) + :load-if (not (sc-is y ,sc)))) + (:results (y)) + (:note "float argument move") + (:generator ,(case format (:single 2) (:double 3) ) + (sc-case y + (,sc + (move y x)) + (,stack-sc + (if (= (tn-offset fp) esp-offset) + (let* ((offset (* (tn-offset y) n-word-bytes)) + (ea (make-ea :dword :base fp :disp offset))) + ,@(ecase format + (:single '((inst movss ea x))) + (:double '((inst movsd ea x))))) + (let ((ea (make-ea + :dword :base fp + :disp (frame-byte-offset (tn-offset y))))) + ,@(ecase format + (:single '((inst movss ea x))) + (:double '((inst movsd ea x)))))))))) + (define-move-vop ,name :move-arg + (,sc descriptor-reg) (,sc))))) (frob move-single-float-arg single-reg single-stack :single) (frob move-double-float-arg double-reg double-stack :double)) ;;;; complex float MOVE-ARG VOP (macrolet ((frob (name sc stack-sc format) - `(progn - (define-vop (,name) - (:args (x :scs (,sc) :target y) - (fp :scs (any-reg) - :load-if (not (sc-is y ,sc)))) - (:results (y)) - (:note "complex float argument move") - (:generator ,(ecase format (:single 2) (:double 3) (:long 4)) - (sc-case y - (,sc - (unless (location= x y) - (let ((x-real (complex-double-reg-real-tn x)) - (y-real (complex-double-reg-real-tn y))) - (cond ((zerop (tn-offset y-real)) - (copy-fp-reg-to-fr0 x-real)) - ((zerop (tn-offset x-real)) - (inst fstd y-real)) - (t - (inst fxch x-real) - (inst fstd y-real) - (inst fxch x-real)))) - (let ((x-imag (complex-double-reg-imag-tn x)) - (y-imag (complex-double-reg-imag-tn y))) - (inst fxch x-imag) - (inst fstd y-imag) - (inst fxch x-imag)))) - (,stack-sc - (let ((real-tn (complex-double-reg-real-tn x))) - (cond ((zerop (tn-offset real-tn)) - ,@(ecase format - (:single - '((inst fst - (ea-for-csf-real-stack y fp)))) - (:double - '((inst fstd - (ea-for-cdf-real-stack y fp)))))) - (t - (inst fxch real-tn) - ,@(ecase format - (:single - '((inst fst - (ea-for-csf-real-stack y fp)))) - (:double - '((inst fstd - (ea-for-cdf-real-stack y fp))))) - (inst fxch real-tn)))) - (let ((imag-tn (complex-double-reg-imag-tn x))) - (inst fxch imag-tn) - ,@(ecase format - (:single - '((inst fst (ea-for-csf-imag-stack y fp)))) - (:double - '((inst fstd (ea-for-cdf-imag-stack y fp))))) - (inst fxch imag-tn)))))) - (define-move-vop ,name :move-arg - (,sc descriptor-reg) (,sc))))) + `(progn + (define-vop (,name) + (:args (x :scs (,sc) :target y) + (fp :scs (any-reg) + :load-if (not (sc-is y ,sc)))) + (:results (y)) + (:note "complex float argument move") + (:generator ,(ecase format (:single 2) (:double 3)) + (sc-case y + (,sc + (move y x)) + (,stack-sc + ,(ecase format + (:single + '(inst movq (ea-for-csf-data-stack y fp) x)) + (:double + '(inst movupd (ea-for-cdf-data-stack y fp) x))))))) + (define-move-vop ,name :move-arg + (,sc descriptor-reg) (,sc))))) (frob move-complex-single-float-arg - complex-single-reg complex-single-stack :single) + complex-single-reg complex-single-stack :single) (frob move-complex-double-float-arg - complex-double-reg complex-double-stack :double)) + complex-double-reg complex-double-stack :double)) (define-move-vop move-arg :move-arg (single-reg double-reg @@ -540,867 +335,788 @@ ;;;; arithmetic VOPs -;;; dtc: the floating point arithmetic vops -;;; -;;; Note: Although these can accept x and y on the stack or pointed to -;;; from a descriptor register, they will work with register loading -;;; without these. Same deal with the result - it need only be a -;;; register. When load-tns are needed they will probably be in ST0 -;;; and the code below should be able to correctly handle all cases. -;;; -;;; However it seems to produce better code if all arg. and result -;;; options are used; on the P86 there is no extra cost in using a -;;; memory operand to the FP instructions - not so on the PPro. -;;; -;;; It may also be useful to handle constant args? -;;; -;;; 22-Jul-97: descriptor args lose in some simple cases when -;;; a function result computed in a loop. Then Python insists -;;; on consing the intermediate values! For example -#| -(defun test(a n) - (declare (type (simple-array double-float (*)) a) - (fixnum n)) - (let ((sum 0d0)) - (declare (type double-float sum)) - (dotimes (i n) - (incf sum (* (aref a i)(aref a i)))) - sum)) -|# -;;; So, disabling descriptor args until this can be fixed elsewhere. -(macrolet - ((frob (op fop-sti fopr-sti - fop fopr sname scost - fopd foprd dname dcost - lname lcost) - #!-long-float (declare (ignore lcost lname)) - `(progn - (define-vop (,sname) - (:translate ,op) - (:args (x :scs (single-reg single-stack #+nil descriptor-reg) - :to :eval) - (y :scs (single-reg single-stack #+nil descriptor-reg) - :to :eval)) - (:temporary (:sc single-reg :offset fr0-offset - :from :eval :to :result) fr0) - (:results (r :scs (single-reg single-stack))) - (:arg-types single-float single-float) - (:result-types single-float) - (:policy :fast-safe) - (:note "inline float arithmetic") - (:vop-var vop) - (:save-p :compute-only) - (:node-var node) - (:generator ,scost - ;; Handle a few special cases - (cond - ;; x, y, and r are the same register. - ((and (sc-is x single-reg) (location= x r) (location= y r)) - (cond ((zerop (tn-offset r)) - (inst ,fop fr0)) - (t - (inst fxch r) - (inst ,fop fr0) - ;; XX the source register will not be valid. - (note-next-instruction vop :internal-error) - (inst fxch r)))) - - ;; x and r are the same register. - ((and (sc-is x single-reg) (location= x r)) - (cond ((zerop (tn-offset r)) - (sc-case y - (single-reg - ;; ST(0) = ST(0) op ST(y) - (inst ,fop y)) - (single-stack - ;; ST(0) = ST(0) op Mem - (inst ,fop (ea-for-sf-stack y))) - (descriptor-reg - (inst ,fop (ea-for-sf-desc y))))) - (t - ;; y to ST0 - (sc-case y - (single-reg - (unless (zerop (tn-offset y)) - (copy-fp-reg-to-fr0 y))) - ((single-stack descriptor-reg) - (inst fstp fr0) - (if (sc-is y single-stack) - (inst fld (ea-for-sf-stack y)) - (inst fld (ea-for-sf-desc y))))) - ;; ST(i) = ST(i) op ST0 - (inst ,fop-sti r))) - (maybe-fp-wait node vop)) - ;; y and r are the same register. - ((and (sc-is y single-reg) (location= y r)) - (cond ((zerop (tn-offset r)) - (sc-case x - (single-reg - ;; ST(0) = ST(x) op ST(0) - (inst ,fopr x)) - (single-stack - ;; ST(0) = Mem op ST(0) - (inst ,fopr (ea-for-sf-stack x))) - (descriptor-reg - (inst ,fopr (ea-for-sf-desc x))))) - (t - ;; x to ST0 - (sc-case x - (single-reg - (unless (zerop (tn-offset x)) - (copy-fp-reg-to-fr0 x))) - ((single-stack descriptor-reg) - (inst fstp fr0) - (if (sc-is x single-stack) - (inst fld (ea-for-sf-stack x)) - (inst fld (ea-for-sf-desc x))))) - ;; ST(i) = ST(0) op ST(i) - (inst ,fopr-sti r))) - (maybe-fp-wait node vop)) - ;; the default case - (t - ;; Get the result to ST0. - - ;; Special handling is needed if x or y are in ST0, and - ;; simpler code is generated. - (cond - ;; x is in ST0 - ((and (sc-is x single-reg) (zerop (tn-offset x))) - ;; ST0 = ST0 op y - (sc-case y - (single-reg - (inst ,fop y)) - (single-stack - (inst ,fop (ea-for-sf-stack y))) - (descriptor-reg - (inst ,fop (ea-for-sf-desc y))))) - ;; y is in ST0 - ((and (sc-is y single-reg) (zerop (tn-offset y))) - ;; ST0 = x op ST0 - (sc-case x - (single-reg - (inst ,fopr x)) - (single-stack - (inst ,fopr (ea-for-sf-stack x))) - (descriptor-reg - (inst ,fopr (ea-for-sf-desc x))))) - (t - ;; x to ST0 - (sc-case x - (single-reg - (copy-fp-reg-to-fr0 x)) - (single-stack - (inst fstp fr0) - (inst fld (ea-for-sf-stack x))) - (descriptor-reg - (inst fstp fr0) - (inst fld (ea-for-sf-desc x)))) - ;; ST0 = ST0 op y - (sc-case y - (single-reg - (inst ,fop y)) - (single-stack - (inst ,fop (ea-for-sf-stack y))) - (descriptor-reg - (inst ,fop (ea-for-sf-desc y)))))) - - (note-next-instruction vop :internal-error) - - ;; Finally save the result. - (sc-case r - (single-reg - (cond ((zerop (tn-offset r)) - (maybe-fp-wait node)) - (t - (inst fst r)))) - (single-stack - (inst fst (ea-for-sf-stack r)))))))) - - (define-vop (,dname) - (:translate ,op) - (:args (x :scs (double-reg double-stack #+nil descriptor-reg) - :to :eval) - (y :scs (double-reg double-stack #+nil descriptor-reg) - :to :eval)) - (:temporary (:sc double-reg :offset fr0-offset - :from :eval :to :result) fr0) - (:results (r :scs (double-reg double-stack))) - (:arg-types double-float double-float) - (:result-types double-float) - (:policy :fast-safe) - (:note "inline float arithmetic") - (:vop-var vop) - (:save-p :compute-only) - (:node-var node) - (:generator ,dcost - ;; Handle a few special cases. - (cond - ;; x, y, and r are the same register. - ((and (sc-is x double-reg) (location= x r) (location= y r)) - (cond ((zerop (tn-offset r)) - (inst ,fop fr0)) - (t - (inst fxch x) - (inst ,fopd fr0) - ;; XX the source register will not be valid. - (note-next-instruction vop :internal-error) - (inst fxch r)))) - - ;; x and r are the same register. - ((and (sc-is x double-reg) (location= x r)) - (cond ((zerop (tn-offset r)) - (sc-case y - (double-reg - ;; ST(0) = ST(0) op ST(y) - (inst ,fopd y)) - (double-stack - ;; ST(0) = ST(0) op Mem - (inst ,fopd (ea-for-df-stack y))) - (descriptor-reg - (inst ,fopd (ea-for-df-desc y))))) - (t - ;; y to ST0 - (sc-case y - (double-reg - (unless (zerop (tn-offset y)) - (copy-fp-reg-to-fr0 y))) - ((double-stack descriptor-reg) - (inst fstp fr0) - (if (sc-is y double-stack) - (inst fldd (ea-for-df-stack y)) - (inst fldd (ea-for-df-desc y))))) - ;; ST(i) = ST(i) op ST0 - (inst ,fop-sti r))) - (maybe-fp-wait node vop)) - ;; y and r are the same register. - ((and (sc-is y double-reg) (location= y r)) - (cond ((zerop (tn-offset r)) - (sc-case x - (double-reg - ;; ST(0) = ST(x) op ST(0) - (inst ,foprd x)) - (double-stack - ;; ST(0) = Mem op ST(0) - (inst ,foprd (ea-for-df-stack x))) - (descriptor-reg - (inst ,foprd (ea-for-df-desc x))))) - (t - ;; x to ST0 - (sc-case x - (double-reg - (unless (zerop (tn-offset x)) - (copy-fp-reg-to-fr0 x))) - ((double-stack descriptor-reg) - (inst fstp fr0) - (if (sc-is x double-stack) - (inst fldd (ea-for-df-stack x)) - (inst fldd (ea-for-df-desc x))))) - ;; ST(i) = ST(0) op ST(i) - (inst ,fopr-sti r))) - (maybe-fp-wait node vop)) - ;; the default case - (t - ;; Get the result to ST0. - - ;; Special handling is needed if x or y are in ST0, and - ;; simpler code is generated. - (cond - ;; x is in ST0 - ((and (sc-is x double-reg) (zerop (tn-offset x))) - ;; ST0 = ST0 op y - (sc-case y - (double-reg - (inst ,fopd y)) - (double-stack - (inst ,fopd (ea-for-df-stack y))) - (descriptor-reg - (inst ,fopd (ea-for-df-desc y))))) - ;; y is in ST0 - ((and (sc-is y double-reg) (zerop (tn-offset y))) - ;; ST0 = x op ST0 - (sc-case x - (double-reg - (inst ,foprd x)) - (double-stack - (inst ,foprd (ea-for-df-stack x))) - (descriptor-reg - (inst ,foprd (ea-for-df-desc x))))) - (t - ;; x to ST0 - (sc-case x - (double-reg - (copy-fp-reg-to-fr0 x)) - (double-stack - (inst fstp fr0) - (inst fldd (ea-for-df-stack x))) - (descriptor-reg - (inst fstp fr0) - (inst fldd (ea-for-df-desc x)))) - ;; ST0 = ST0 op y - (sc-case y - (double-reg - (inst ,fopd y)) - (double-stack - (inst ,fopd (ea-for-df-stack y))) - (descriptor-reg - (inst ,fopd (ea-for-df-desc y)))))) - - (note-next-instruction vop :internal-error) - - ;; Finally save the result. - (sc-case r - (double-reg - (cond ((zerop (tn-offset r)) - (maybe-fp-wait node)) - (t - (inst fst r)))) - (double-stack - (inst fstd (ea-for-df-stack r)))))))) - ))) - - (frob + fadd-sti fadd-sti - fadd fadd +/single-float 2 - faddd faddd +/double-float 2 - +/long-float 2) - (frob - fsub-sti fsubr-sti - fsub fsubr -/single-float 2 - fsubd fsubrd -/double-float 2 - -/long-float 2) - (frob * fmul-sti fmul-sti - fmul fmul */single-float 3 - fmuld fmuld */double-float 3 - */long-float 3) - (frob / fdiv-sti fdivr-sti - fdiv fdivr //single-float 12 - fdivd fdivrd //double-float 12 - //long-float 12)) +(define-vop (float-op) + (:args (x) (y)) + (:results (r)) + (:policy :fast-safe) + (:note "inline float arithmetic") + (:vop-var vop) + (:save-p :compute-only)) + +(macrolet ((frob (name comm-name sc constant-sc ptype) + `(progn + (define-vop (,name float-op) + (:args (x :scs (,sc ,constant-sc) + :target r + :load-if (not (sc-is x ,constant-sc))) + (y :scs (,sc ,constant-sc) + :load-if (not (sc-is y ,constant-sc)))) + (:results (r :scs (,sc))) + (:arg-types ,ptype ,ptype) + (:result-types ,ptype)) + (define-vop (,comm-name float-op) + (:args (x :scs (,sc ,constant-sc) + :target r + :load-if (not (sc-is x ,constant-sc))) + (y :scs (,sc ,constant-sc) + :target r + :load-if (not (sc-is y ,constant-sc)))) + (:results (r :scs (,sc))) + (:arg-types ,ptype ,ptype) + (:result-types ,ptype))))) + (frob single-float-op single-float-comm-op + single-reg fp-single-immediate single-float) + (frob double-float-op double-float-comm-op + double-reg fp-double-immediate double-float) + (frob complex-single-float-op complex-single-float-comm-op + complex-single-reg fp-complex-single-immediate + complex-single-float) + (frob complex-double-float-op complex-double-float-comm-op + complex-double-reg fp-complex-double-immediate + complex-double-float)) + +(macrolet ((generate (opinst commutative constant-sc load-inst) + `(flet ((get-constant (tn) + (register-inline-constant + ,@(and (eq constant-sc 'fp-single-immediate) + '(:aligned)) + (tn-value tn)))) + (declare (ignorable #'get-constant)) + (cond + ((location= x r) + (when (sc-is y ,constant-sc) + (setf y (get-constant y))) + (inst ,opinst x y)) + ((and ,commutative (location= y r)) + (when (sc-is x ,constant-sc) + (setf x (get-constant x))) + (inst ,opinst y x)) + ((not (location= r y)) + (if (sc-is x ,constant-sc) + (inst ,load-inst r (get-constant x)) + (move r x)) + (when (sc-is y ,constant-sc) + (setf y (get-constant y))) + (inst ,opinst r y)) + (t + (if (sc-is x ,constant-sc) + (inst ,load-inst tmp (get-constant x)) + (move tmp x)) + (inst ,opinst tmp y) + (move r tmp))))) + (frob (op sinst sname scost dinst dname dcost commutative + &optional csinst csname cscost cdinst cdname cdcost) + `(progn + (define-vop (,sname ,(if commutative + 'single-float-comm-op + 'single-float-op)) + (:translate ,op) + (:temporary (:sc single-reg) tmp) + (:generator ,scost + (generate ,sinst ,commutative fp-single-immediate movss))) + (define-vop (,dname ,(if commutative + 'double-float-comm-op + 'double-float-op)) + (:translate ,op) + (:temporary (:sc double-reg) tmp) + (:generator ,dcost + (generate ,dinst ,commutative fp-double-immediate movsd))) + ,(when csinst + `(define-vop (,csname + ,(if commutative + 'complex-single-float-comm-op + 'complex-single-float-op)) + (:translate ,op) + (:temporary (:sc complex-single-reg) tmp) + (:generator ,cscost + (generate ,csinst ,commutative + fp-complex-single-immediate movq)))) + ,(when cdinst + `(define-vop (,cdname + ,(if commutative + 'complex-double-float-comm-op + 'complex-double-float-op)) + (:translate ,op) + (:temporary (:sc complex-double-reg) tmp) + (:generator ,cdcost + (generate ,cdinst ,commutative + fp-complex-double-immediate movapd))))))) + (frob + addss +/single-float 2 addsd +/double-float 2 t + addps +/complex-single-float 3 addpd +/complex-double-float 3) + (frob - subss -/single-float 2 subsd -/double-float 2 nil + subps -/complex-single-float 3 subpd -/complex-double-float 3) + (frob * mulss */single-float 4 mulsd */double-float 5 t) + (frob / divss //single-float 12 divsd //double-float 19 nil)) + +(macrolet ((frob (op cost commutativep + duplicate-inst op-inst real-move-inst complex-move-inst + real-sc real-constant-sc real-type + complex-sc complex-constant-sc complex-type + real-complex-name complex-real-name) + (cond ((not duplicate-inst) ; simple case + `(flet ((load-into (r x) + (sc-case x + (,real-constant-sc + (inst ,real-move-inst r + (register-inline-constant (tn-value x)))) + (,complex-constant-sc + (inst ,complex-move-inst r + (register-inline-constant (tn-value x)))) + (t (move r x))))) + ,(when real-complex-name + `(define-vop (,real-complex-name float-op) + (:translate ,op) + (:args (x :scs (,real-sc ,real-constant-sc) + :target r + :load-if (not (sc-is x ,real-constant-sc))) + (y :scs (,complex-sc ,complex-constant-sc) + ,@(when commutativep '(:target r)) + :load-if (not (sc-is y ,complex-constant-sc)))) + (:arg-types ,real-type ,complex-type) + (:results (r :scs (,complex-sc) + ,@(unless commutativep '(:from (:argument 0))))) + (:result-types ,complex-type) + (:generator ,cost + ,(when commutativep + `(when (location= y r) + (rotatef x y))) + (load-into r x) + (when (sc-is y ,real-constant-sc ,complex-constant-sc) + (setf y (register-inline-constant + :aligned (tn-value y)))) + (inst ,op-inst r y)))) + + ,(when complex-real-name + `(define-vop (,complex-real-name float-op) + (:translate ,op) + (:args (x :scs (,complex-sc ,complex-constant-sc) + :target r + :load-if (not (sc-is x ,complex-constant-sc))) + (y :scs (,real-sc ,real-constant-sc) + ,@(when commutativep '(:target r)) + :load-if (not (sc-is y ,real-constant-sc)))) + (:arg-types ,complex-type ,real-type) + (:results (r :scs (,complex-sc) + ,@(unless commutativep '(:from (:argument 0))))) + (:result-types ,complex-type) + (:generator ,cost + ,(when commutativep + `(when (location= y r) + (rotatef x y))) + (load-into r x) + (when (sc-is y ,real-constant-sc ,complex-constant-sc) + (setf y (register-inline-constant + :aligned (tn-value y)))) + (inst ,op-inst r y)))))) + (commutativep ; must duplicate, but commutative + `(progn + ,(when real-complex-name + `(define-vop (,real-complex-name float-op) + (:translate ,op) + (:args (x :scs (,real-sc ,real-constant-sc) + :target dup + :load-if (not (sc-is x ,real-constant-sc))) + (y :scs (,complex-sc ,complex-constant-sc) + :target r + :to :result + :load-if (not (sc-is y ,complex-constant-sc)))) + (:arg-types ,real-type ,complex-type) + (:temporary (:sc ,complex-sc :target r + :from (:argument 0) + :to :result) + dup) + (:results (r :scs (,complex-sc))) + (:result-types ,complex-type) + (:generator ,cost + (if (sc-is x ,real-constant-sc) + (inst ,complex-move-inst dup + (register-inline-constant + (complex (tn-value x) (tn-value x)))) + (let ((real x)) + ,duplicate-inst)) + ;; safe: dup /= y + (when (location= dup r) + (rotatef dup y)) + (if (sc-is y ,complex-constant-sc) + (inst ,complex-move-inst r + (register-inline-constant (tn-value y))) + (move r y)) + (when (sc-is dup ,complex-constant-sc) + (setf dup (register-inline-constant + :aligned (tn-value dup)))) + (inst ,op-inst r dup)))) + + ,(when complex-real-name + `(define-vop (,complex-real-name float-op) + (:translate ,op) + (:args (x :scs (,complex-sc ,complex-constant-sc) + :target r + :to :result + :load-if (not (sc-is x ,complex-constant-sc))) + (y :scs (,real-sc ,real-constant-sc) + :target dup + :load-if (not (sc-is y ,real-constant-sc)))) + (:arg-types ,complex-type ,real-type) + (:temporary (:sc ,complex-sc :target r + :from (:argument 1) + :to :result) + dup) + (:results (r :scs (,complex-sc))) + (:result-types ,complex-type) + (:generator ,cost + (if (sc-is y ,real-constant-sc) + (inst ,complex-move-inst dup + (register-inline-constant + (complex (tn-value y) (tn-value y)))) + (let ((real y)) + ,duplicate-inst)) + (when (location= dup r) + (rotatef x dup)) + (if (sc-is x ,complex-constant-sc) + (inst ,complex-move-inst r + (register-inline-constant (tn-value x))) + (move r x)) + (when (sc-is dup ,complex-constant-sc) + (setf dup (register-inline-constant + :aligned (tn-value dup)))) + (inst ,op-inst r dup)))))) + (t ; duplicate, not commutative + `(progn + ,(when real-complex-name + `(define-vop (,real-complex-name float-op) + (:translate ,op) + (:args (x :scs (,real-sc ,real-constant-sc) + :target r + :load-if (not (sc-is x ,real-constant-sc))) + (y :scs (,complex-sc ,complex-constant-sc) + :to :result + :load-if (not (sc-is y ,complex-constant-sc)))) + (:arg-types ,real-type ,complex-type) + (:results (r :scs (,complex-sc) :from (:argument 0))) + (:result-types ,complex-type) + (:generator ,cost + (if (sc-is x ,real-constant-sc) + (inst ,complex-move-inst dup + (register-inline-constant + (complex (tn-value x) (tn-value x)))) + (let ((real x) + (dup r)) + ,duplicate-inst)) + (when (sc-is y ,complex-constant-sc) + (setf y (register-inline-constant + :aligned (tn-value y)))) + (inst ,op-inst r y)))) + + ,(when complex-real-name + `(define-vop (,complex-real-name float-op) + (:translate ,op) + (:args (x :scs (,complex-sc) + :target r + :to :eval) + (y :scs (,real-sc ,real-constant-sc) + :target dup + :load-if (not (sc-is y ,complex-constant-sc)))) + (:arg-types ,complex-type ,real-type) + (:temporary (:sc ,complex-sc :from (:argument 1)) + dup) + (:results (r :scs (,complex-sc) :from :eval)) + (:result-types ,complex-type) + (:generator ,cost + (if (sc-is y ,real-constant-sc) + (setf dup (register-inline-constant + :aligned (complex (tn-value y) + (tn-value y)))) + (let ((real y)) + ,duplicate-inst)) + (move r x) + (inst ,op-inst r dup)))))))) + (def-real-complex-op (op commutativep duplicatep + single-inst single-real-complex-name single-complex-real-name single-cost + double-inst double-real-complex-name double-complex-real-name double-cost) + `(progn + (frob ,op ,single-cost ,commutativep + ,(and duplicatep + `(progn + (move dup real) + (inst unpcklps dup dup))) + ,single-inst movss movq + single-reg fp-single-immediate single-float + complex-single-reg fp-complex-single-immediate complex-single-float + ,single-real-complex-name ,single-complex-real-name) + (frob ,op ,double-cost ,commutativep + ,(and duplicatep + `(progn + (move dup real) + (inst unpcklpd dup dup))) + ,double-inst movsd movapd + double-reg fp-double-immediate double-float + complex-double-reg fp-complex-double-immediate complex-double-float + ,double-real-complex-name ,double-complex-real-name)))) + (def-real-complex-op + t nil + addps +/real-complex-single-float +/complex-real-single-float 3 + addpd +/real-complex-double-float +/complex-real-double-float 4) + (def-real-complex-op - nil nil + subps -/real-complex-single-float -/complex-real-single-float 3 + subpd -/real-complex-double-float -/complex-real-double-float 4) + (def-real-complex-op * t t + mulps */real-complex-single-float */complex-real-single-float 4 + mulpd */real-complex-double-float */complex-real-double-float 5) + (def-real-complex-op / nil t + nil nil nil nil + divpd nil //complex-real-double-float 19)) + +(define-vop (//complex-real-single-float float-op) + (:translate /) + (:args (x :scs (complex-single-reg fp-complex-single-immediate fp-complex-single-zero) + :to (:result 0) + :target r + :load-if (not (sc-is x fp-complex-single-immediate fp-complex-single-zero))) + (y :scs (single-reg fp-single-immediate fp-single-zero) + :target dup + :load-if (not (sc-is y fp-single-immediate fp-single-zero)))) + (:arg-types complex-single-float single-float) + (:temporary (:sc complex-single-reg :from (:argument 1)) dup) + (:results (r :scs (complex-single-reg))) + (:result-types complex-single-float) + (:generator 12 + (flet ((duplicate (x) + (let ((word (ldb (byte 64 0) + (logior (ash (single-float-bits (imagpart x)) 32) + (ldb (byte 32 0) + (single-float-bits (realpart x))))))) + (register-inline-constant :oword (logior (ash word 64) word))))) + (sc-case y + (fp-single-immediate + (setf dup (duplicate (complex (tn-value y) (tn-value y))))) + (fp-single-zero + (inst xorps dup dup)) + (t (move dup y) + (inst shufps dup dup #b00000000))) + (sc-case x + (fp-complex-single-immediate + (inst movaps r (duplicate (tn-value x)))) + (fp-complex-single-zero + (inst xorps r r)) + (t + (move r x) + (inst unpcklpd r r))) + (inst divps r dup) + (inst movq r r)))) + +;; Complex multiplication +;; r := rx * ry - ix * iy +;; i := rx * iy + ix * ry +;; +;; Transpose for SIMDness +;; rx*ry rx*iy +;; -ix*iy +ix*ry +;; +;; [rx rx] * [ry iy] +;;+ [ix ix] * [-iy ry] +;; [r i] + +(macrolet ((define-complex-* (name cost type sc tmp-p &body body) + `(define-vop (,name float-op) + (:translate *) + (:args (x :scs (,sc) :target r) + (y :scs (,sc) :target copy-y)) + (:arg-types ,type ,type) + (:temporary (:sc ,sc) imag) + (:temporary (:sc ,sc :from :eval) copy-y) + ,@(when tmp-p + `((:temporary (:sc ,sc) xmm))) + (:results (r :scs (,sc) :from :eval)) + (:result-types ,type) + (:generator ,cost + (when (or (location= x copy-y) + (location= y r)) + (rotatef x y)) + ,@body)))) + (define-complex-* */complex-single-float 20 + complex-single-float complex-single-reg t + (inst xorps xmm xmm) + (move r x) + (inst unpcklps r r) + (move imag r) + (inst unpckhpd imag xmm) + (inst unpcklpd r xmm) + (move copy-y y) ; y == r only if y == x == r + (setf y copy-y) + + (inst mulps r y) + + (inst shufps y y #b11110001) + (inst xorps y (register-inline-constant :oword (ash 1 31))) + + (inst mulps imag y) + (inst addps r imag)) + (define-complex-* */complex-double-float 25 + complex-double-float complex-double-reg nil + (move imag x) + (move r x) + (move copy-y y) + (setf y copy-y) + (inst unpcklpd r r) + (inst unpckhpd imag imag) + + (inst mulpd r y) + + (inst shufpd y y #b01) + (inst xorpd y (register-inline-constant :oword (ash 1 63))) + + (inst mulpd imag y) + (inst addpd r imag))) + +(define-vop (fsqrt) + (:args (x :scs (double-reg))) + (:results (y :scs (double-reg))) + (:translate %sqrt) + (:policy :fast-safe) + (:arg-types double-float) + (:result-types double-float) + (:note "inline float arithmetic") + (:vop-var vop) + (:save-p :compute-only) + (:generator 1 + (note-this-location vop :internal-error) + (inst sqrtsd y x))) -(macrolet ((frob (name inst translate sc type) - `(define-vop (,name) - (:args (x :scs (,sc) :target fr0)) - (:results (y :scs (,sc))) - (:translate ,translate) - (:policy :fast-safe) - (:arg-types ,type) - (:result-types ,type) - (:temporary (:sc double-reg :offset fr0-offset - :from :argument :to :result) fr0) - (:ignore fr0) - (:note "inline float arithmetic") - (:vop-var vop) - (:save-p :compute-only) - (:generator 1 - (note-this-location vop :internal-error) - (unless (zerop (tn-offset x)) - (inst fxch x) ; x to top of stack - (unless (location= x y) - (inst fst x))) ; Maybe save it. - (inst ,inst) ; Clobber st0. - (unless (zerop (tn-offset y)) - (inst fst y)))))) - - (frob abs/single-float fabs abs single-reg single-float) - (frob abs/double-float fabs abs double-reg double-float) - - (frob %negate/single-float fchs %negate single-reg single-float) - (frob %negate/double-float fchs %negate double-reg double-float)) +(macrolet ((frob ((name translate sc type) &body body) + `(define-vop (,name) + (:args (x :scs (,sc) :target y)) + (:results (y :scs (,sc))) + (:translate ,translate) + (:policy :fast-safe) + (:arg-types ,type) + (:result-types ,type) + (:note "inline float arithmetic") + (:vop-var vop) + (:save-p :compute-only) + (:generator 1 + (note-this-location vop :internal-error) + ;; we should be able to do this better. what we + ;; really would like to do is use the target as the + ;; temp whenever it's not also the source + (move y x) + ,@body)))) + (frob (%negate/double-float %negate double-reg double-float) + (inst xorpd y (register-inline-constant :oword (ash 1 63)))) + (frob (%negate/complex-double-float %negate complex-double-reg complex-double-float) + (inst xorpd y (register-inline-constant + :oword (logior (ash 1 127) (ash 1 63))))) + (frob (conjugate/complex-double-float conjugate complex-double-reg complex-double-float) + (inst xorpd y (register-inline-constant :oword (ash 1 127)))) + (frob (%negate/single-float %negate single-reg single-float) + (inst xorps y (register-inline-constant :oword (ash 1 31)))) + (frob (%negate/complex-single-float %negate complex-single-reg complex-single-float) + (inst xorps y (register-inline-constant + :oword (logior (ash 1 31) (ash 1 63))))) + (frob (conjugate/complex-single-float conjugate complex-single-reg complex-single-float) + (inst xorpd y (register-inline-constant :oword (ash 1 63)))) + (frob (abs/double-float abs double-reg double-float) + (inst andpd y (register-inline-constant :oword (ldb (byte 63 0) -1)))) + (frob (abs/single-float abs single-reg single-float) + (inst andps y (register-inline-constant :oword (ldb (byte 31 0) -1))))) + ;;;; comparison -(define-vop (=/float) - (:args (x) (y)) - (:temporary (:sc word-reg :offset eax-offset :from :eval) temp) - (:conditional) - (:info target not-p) +(define-vop (float-compare) (:policy :fast-safe) (:vop-var vop) (:save-p :compute-only) - (:note "inline float comparison") - (:ignore temp) - (:generator 3 - (note-this-location vop :internal-error) - (cond - ;; x is in ST0; y is in any reg. - ((zerop (tn-offset x)) - (inst fucom y)) - ;; y is in ST0; x is in another reg. - ((zerop (tn-offset y)) - (inst fucom x)) - ;; x and y are the same register, not ST0 - ((location= x y) - (inst fxch x) - (inst fucom fr0-tn) - (inst fxch x)) - ;; x and y are different registers, neither ST0. - (t - (inst fxch x) - (inst fucom y) - (inst fxch x))) - (inst fnstsw) ; status word to ax - (inst and ah-tn #x45) ; C3 C2 C0 - (inst cmp ah-tn #x40) - (inst jmp (if not-p :ne :e) target))) - -(define-vop (=/single-float =/float) - (:translate =) + (:note "inline float comparison")) + +;;; EQL +(macrolet ((define-float-eql (name cost sc constant-sc type) + `(define-vop (,name float-compare) + (:translate eql) + (:args (x :scs (,sc ,constant-sc) + :target mask + :load-if (not (sc-is x ,constant-sc))) + (y :scs (,sc ,constant-sc) + :target mask + :load-if (not (sc-is y ,constant-sc)))) + (:arg-types ,type ,type) + (:temporary (:sc ,sc :from :eval) mask) + (:temporary (:sc any-reg) bits) + (:conditional :e) + (:generator ,cost + (when (or (location= y mask) + (not (xmm-register-p x))) + (rotatef x y)) + (aver (xmm-register-p x)) + (move mask x) + (when (sc-is y ,constant-sc) + (setf y (register-inline-constant :aligned (tn-value y)))) + (inst pcmpeqd mask y) + (inst movmskps bits mask) + (inst cmp bits #b1111))))) + (define-float-eql eql/single-float 4 + single-reg fp-single-immediate single-float) + (define-float-eql eql/double-float 4 + double-reg fp-double-immediate double-float) + (define-float-eql eql/complex-single-float 5 + complex-single-reg fp-complex-single-immediate complex-single-float) + (define-float-eql eql/complex-double-float 5 + complex-double-reg fp-complex-double-immediate complex-double-float)) + +;;; comiss and comisd can cope with one or other arg in memory: we +;;; could (should, indeed) extend these to cope with descriptor args +;;; and stack args + +(define-vop (single-float-compare float-compare) (:args (x :scs (single-reg)) - (y :scs (single-reg))) + (y :scs (single-reg single-stack fp-single-immediate) + :load-if (not (sc-is y single-stack fp-single-immediate)))) (:arg-types single-float single-float)) - -(define-vop (=/double-float =/float) - (:translate =) +(define-vop (double-float-compare float-compare) (:args (x :scs (double-reg)) - (y :scs (double-reg))) + (y :scs (double-reg double-stack descriptor-reg fp-double-immediate) + :load-if (not (sc-is y double-stack descriptor-reg fp-double-immediate)))) (:arg-types double-float double-float)) -(define-vop (single-float) - (:translate >) - (:args (x :scs (single-reg single-stack descriptor-reg)) - (y :scs (single-reg single-stack descriptor-reg))) - (:arg-types single-float single-float) - (:temporary (:sc single-reg :offset fr0-offset :from :eval) fr0) - (:temporary (:sc word-reg :offset eax-offset :from :eval) temp) - (:conditional) - (:info target not-p) - (:policy :fast-safe) - (:note "inline float comparison") - (:ignore temp) - (:generator 3 - ;; Handle a few special cases. - (cond - ;; y is ST0. - ((and (sc-is y single-reg) (zerop (tn-offset y))) - (sc-case x - (single-reg - (inst fcom x)) - ((single-stack descriptor-reg) - (if (sc-is x single-stack) - (inst fcom (ea-for-sf-stack x)) - (inst fcom (ea-for-sf-desc x))))) - (inst fnstsw) ; status word to ax - (inst and ah-tn #x45) - (inst cmp ah-tn #x01)) - - ;; general case when y is not in ST0 - (t - ;; x to ST0 - (sc-case x - (single-reg - (unless (zerop (tn-offset x)) - (copy-fp-reg-to-fr0 x))) - ((single-stack descriptor-reg) - (inst fstp fr0) - (if (sc-is x single-stack) - (inst fld (ea-for-sf-stack x)) - (inst fld (ea-for-sf-desc x))))) - (sc-case y - (single-reg - (inst fcom y)) - ((single-stack descriptor-reg) - (if (sc-is y single-stack) - (inst fcom (ea-for-sf-stack y)) - (inst fcom (ea-for-sf-desc y))))) - (inst fnstsw) ; status word to ax - (inst and ah-tn #x45))) - (inst jmp (if not-p :ne :e) target))) - -(define-vop (>double-float) - (:translate >) - (:args (x :scs (double-reg double-stack descriptor-reg)) - (y :scs (double-reg double-stack descriptor-reg))) - (:arg-types double-float double-float) - (:temporary (:sc double-reg :offset fr0-offset :from :eval) fr0) - (:temporary (:sc word-reg :offset eax-offset :from :eval) temp) - (:conditional) - (:info target not-p) - (:policy :fast-safe) - (:note "inline float comparison") - (:ignore temp) - (:generator 3 - ;; Handle a few special cases. - (cond - ;; y is ST0. - ((and (sc-is y double-reg) (zerop (tn-offset y))) - (sc-case x - (double-reg - (inst fcomd x)) - ((double-stack descriptor-reg) - (if (sc-is x double-stack) - (inst fcomd (ea-for-df-stack x)) - (inst fcomd (ea-for-df-desc x))))) - (inst fnstsw) ; status word to ax - (inst and ah-tn #x45) - (inst cmp ah-tn #x01)) - - ;; general case when y is not in ST0 - (t - ;; x to ST0 - (sc-case x - (double-reg - (unless (zerop (tn-offset x)) - (copy-fp-reg-to-fr0 x))) - ((double-stack descriptor-reg) - (inst fstp fr0) - (if (sc-is x double-stack) - (inst fldd (ea-for-df-stack x)) - (inst fldd (ea-for-df-desc x))))) - (sc-case y - (double-reg - (inst fcomd y)) - ((double-stack descriptor-reg) - (if (sc-is y double-stack) - (inst fcomd (ea-for-df-stack y)) - (inst fcomd (ea-for-df-desc y))))) - (inst fnstsw) ; status word to ax - (inst and ah-tn #x45))) - (inst jmp (if not-p :ne :e) target))) - -;;; Comparisons with 0 can use the FTST instruction. - -(define-vop (float-test) - (:args (x)) - (:temporary (:sc word-reg :offset eax-offset :from :eval) temp) - (:conditional) - (:info target not-p y) - (:variant-vars code) - (:policy :fast-safe) - (:vop-var vop) - (:save-p :compute-only) - (:note "inline float comparison") - (:ignore temp y) - (:generator 2 - (note-this-location vop :internal-error) - (cond - ;; x is in ST0 - ((zerop (tn-offset x)) - (inst ftst)) - ;; x not ST0 - (t - (inst fxch x) - (inst ftst) - (inst fxch x))) - (inst fnstsw) ; status word to ax - (inst and ah-tn #x45) ; C3 C2 C0 - (unless (zerop code) - (inst cmp ah-tn code)) - (inst jmp (if not-p :ne :e) target))) - -(define-vop (=0/single-float float-test) +(define-vop (=/single-float single-float-compare) (:translate =) - (:args (x :scs (single-reg))) - (:arg-types single-float (:constant (single-float 0f0 0f0))) - (:variant #x40)) -(define-vop (=0/double-float float-test) + (:args (x :scs (single-reg single-stack fp-single-immediate) + :target xmm + :load-if (not (sc-is x single-stack fp-single-immediate))) + (y :scs (single-reg single-stack fp-single-immediate) + :target xmm + :load-if (not (sc-is y single-stack fp-single-immediate)))) + (:temporary (:sc single-reg :from :eval) xmm) + (:info) + (:conditional not :p :ne) + (:vop-var vop) + (:generator 3 + (when (or (location= y xmm) + (and (not (xmm-register-p x)) + (xmm-register-p y))) + (rotatef x y)) + (sc-case x + (single-reg (setf xmm x)) + (single-stack (inst movss xmm (ea-for-sf-stack x))) + (fp-single-immediate + (inst movss xmm (register-inline-constant (tn-value x))))) + (sc-case y + (single-stack + (setf y (ea-for-sf-stack y))) + (fp-single-immediate + (setf y (register-inline-constant (tn-value y)))) + (t)) + (note-this-location vop :internal-error) + (inst comiss xmm y) + ;; if PF&CF, there was a NaN involved => not equal + ;; otherwise, ZF => equal + )) + +(define-vop (=/double-float double-float-compare) (:translate =) - (:args (x :scs (double-reg))) - (:arg-types double-float (:constant (double-float 0d0 0d0))) - (:variant #x40)) - -(define-vop (<0/single-float float-test) - (:translate <) - (:args (x :scs (single-reg))) - (:arg-types single-float (:constant (single-float 0f0 0f0))) - (:variant #x01)) -(define-vop (<0/double-float float-test) - (:translate <) - (:args (x :scs (double-reg))) - (:arg-types double-float (:constant (double-float 0d0 0d0))) - (:variant #x01)) - -(define-vop (>0/single-float float-test) - (:translate >) - (:args (x :scs (single-reg))) - (:arg-types single-float (:constant (single-float 0f0 0f0))) - (:variant #x00)) -(define-vop (>0/double-float float-test) - (:translate >) - (:args (x :scs (double-reg))) - (:arg-types double-float (:constant (double-float 0d0 0d0))) - (:variant #x00)) + (:args (x :scs (double-reg double-stack fp-double-immediate descriptor-reg) + :target xmm + :load-if (not (sc-is x double-stack fp-double-immediate descriptor-reg))) + (y :scs (double-reg double-stack fp-double-immediate descriptor-reg) + :target xmm + :load-if (not (sc-is y double-stack fp-double-immediate descriptor-reg)))) + (:temporary (:sc double-reg :from :eval) xmm) + (:info) + (:conditional not :p :ne) + (:vop-var vop) + (:generator 3 + (when (or (location= y xmm) + (and (not (xmm-register-p x)) + (xmm-register-p y))) + (rotatef x y)) + (sc-case x + (double-reg + (setf xmm x)) + (double-stack + (inst movsd xmm (ea-for-df-stack x))) + (fp-double-immediate + (inst movsd xmm (register-inline-constant (tn-value x)))) + (descriptor-reg + (inst movsd xmm (ea-for-df-desc x)))) + (sc-case y + (double-stack + (setf y (ea-for-df-stack y))) + (fp-double-immediate + (setf y (register-inline-constant (tn-value y)))) + (descriptor-reg + (setf y (ea-for-df-desc y))) + (t)) + (note-this-location vop :internal-error) + (inst comisd xmm y))) + +(macrolet ((define-complex-float-= (complex-complex-name complex-real-name real-complex-name + real-sc real-constant-sc real-type + complex-sc complex-constant-sc complex-type + real-move-inst complex-move-inst + cmp-inst mask-inst mask) + `(progn + (define-vop (,complex-complex-name float-compare) + (:translate =) + (:args (x :scs (,complex-sc ,complex-constant-sc) + :target cmp + :load-if (not (sc-is x ,complex-constant-sc))) + (y :scs (,complex-sc ,complex-constant-sc) + :target cmp + :load-if (not (sc-is y ,complex-constant-sc)))) + (:arg-types ,complex-type ,complex-type) + (:temporary (:sc ,complex-sc :from :eval) cmp) + (:temporary (:sc unsigned-reg) bits) + (:info) + (:conditional :e) + (:generator 3 + (when (location= y cmp) + (rotatef x y)) + (sc-case x + (,real-constant-sc + (inst ,real-move-inst cmp (register-inline-constant + (tn-value x)))) + (,complex-constant-sc + (inst ,complex-move-inst cmp (register-inline-constant + (tn-value x)))) + (t + (move cmp x))) + (when (sc-is y ,real-constant-sc ,complex-constant-sc) + (setf y (register-inline-constant :aligned (tn-value y)))) + (note-this-location vop :internal-error) + (inst ,cmp-inst :eq cmp y) + (inst ,mask-inst bits cmp) + (inst cmp bits ,mask))) + (define-vop (,complex-real-name ,complex-complex-name) + (:args (x :scs (,complex-sc ,complex-constant-sc) + :target cmp + :load-if (not (sc-is x ,complex-constant-sc))) + (y :scs (,real-sc ,real-constant-sc) + :target cmp + :load-if (not (sc-is y ,real-constant-sc)))) + (:arg-types ,complex-type ,real-type)) + (define-vop (,real-complex-name ,complex-complex-name) + (:args (x :scs (,real-sc ,real-constant-sc) + :target cmp + :load-if (not (sc-is x ,real-constant-sc))) + (y :scs (,complex-sc ,complex-constant-sc) + :target cmp + :load-if (not (sc-is y ,complex-constant-sc)))) + (:arg-types ,real-type ,complex-type))))) + (define-complex-float-= =/complex-single-float =/complex-real-single-float =/real-complex-single-float + single-reg fp-single-immediate single-float + complex-single-reg fp-complex-single-immediate complex-single-float + movss movq cmpps movmskps #b1111) + (define-complex-float-= =/complex-double-float =/complex-real-double-float =/real-complex-double-float + double-reg fp-double-immediate double-float + complex-double-reg fp-complex-double-immediate complex-double-float + movsd movapd cmppd movmskpd #b11)) + +(macrolet ((define- (op single-name double-name &rest flags) + `(progn + (define-vop (,double-name double-float-compare) + (:translate ,op) + (:info) + (:conditional ,@flags) + (:generator 3 + (sc-case y + (double-stack + (setf y (ea-for-df-stack y))) + (descriptor-reg + (setf y (ea-for-df-desc y))) + (fp-double-immediate + (setf y (register-inline-constant (tn-value y)))) + (t)) + (inst comisd x y))) + (define-vop (,single-name single-float-compare) + (:translate ,op) + (:info) + (:conditional ,@flags) + (:generator 3 + (sc-case y + (single-stack + (setf y (ea-for-sf-stack y))) + (fp-single-immediate + (setf y (register-inline-constant (tn-value y)))) + (t)) + (inst comiss x y)))))) + (define- < > >single-float >double-float not :p :na)) ;;;; conversion -(macrolet ((frob (name translate to-sc to-type) - `(define-vop (,name) - (:args (x :scs (signed-stack signed-reg) :target temp)) - (:temporary (:sc signed-stack) temp) - (:results (y :scs (,to-sc))) - (:arg-types signed-num) - (:result-types ,to-type) - (:policy :fast-safe) - (:note "inline float coercion") - (:translate ,translate) - (:vop-var vop) - (:save-p :compute-only) - (:generator 5 - (sc-case x - (signed-reg - (inst mov temp x) - (with-empty-tn@fp-top(y) - (note-this-location vop :internal-error) - (inst fild temp))) - (signed-stack - (with-empty-tn@fp-top(y) - (note-this-location vop :internal-error) - (inst fild x)))))))) - (frob %single-float/signed %single-float single-reg single-float) - (frob %double-float/signed %double-float double-reg double-float)) - -(macrolet ((frob (name translate to-sc to-type) - `(define-vop (,name) - (:args (x :scs (unsigned-reg))) - (:results (y :scs (,to-sc))) - (:arg-types unsigned-num) - (:result-types ,to-type) - (:policy :fast-safe) - (:note "inline float coercion") - (:translate ,translate) - (:vop-var vop) - (:save-p :compute-only) - (:generator 6 - (inst push 0) - (inst push x) - (with-empty-tn@fp-top(y) - (note-this-location vop :internal-error) - (inst fildl (make-ea :dword :base rsp-tn))) - (inst add rsp-tn 16))))) - (frob %single-float/unsigned %single-float single-reg single-float) - (frob %double-float/unsigned %double-float double-reg double-float)) - -;;; These should be no-ops but the compiler might want to move some -;;; things around. -(macrolet ((frob (name translate from-sc from-type to-sc to-type) - `(define-vop (,name) - (:args (x :scs (,from-sc) :target y)) - (:results (y :scs (,to-sc))) - (:arg-types ,from-type) - (:result-types ,to-type) - (:policy :fast-safe) - (:note "inline float coercion") - (:translate ,translate) - (:vop-var vop) - (:save-p :compute-only) - (:generator 2 - (note-this-location vop :internal-error) - (unless (location= x y) - (cond - ((zerop (tn-offset x)) - ;; x is in ST0, y is in another reg. not ST0 - (inst fst y)) - ((zerop (tn-offset y)) - ;; y is in ST0, x is in another reg. not ST0 - (copy-fp-reg-to-fr0 x)) - (t - ;; Neither x or y are in ST0, and they are not in - ;; the same reg. - (inst fxch x) - (inst fst y) - (inst fxch x)))))))) - - (frob %single-float/double-float %single-float double-reg - double-float single-reg single-float) - - (frob %double-float/single-float %double-float single-reg single-float - double-reg double-float)) - -(macrolet ((frob (trans from-sc from-type round-p) - `(define-vop (,(symbolicate trans "/" from-type)) - (:args (x :scs (,from-sc))) - (:temporary (:sc signed-stack) stack-temp) - ,@(unless round-p - '((:temporary (:sc unsigned-stack) scw) - (:temporary (:sc any-reg) rcw))) - (:results (y :scs (signed-reg))) - (:arg-types ,from-type) - (:result-types signed-num) - (:translate ,trans) - (:policy :fast-safe) - (:note "inline float truncate") - (:vop-var vop) - (:save-p :compute-only) - (:generator 5 - ,@(unless round-p - '((note-this-location vop :internal-error) - ;; Catch any pending FPE exceptions. - (inst wait))) - (,(if round-p 'progn 'pseudo-atomic) - ;; Normal mode (for now) is "round to best". - (with-tn@fp-top (x) - ,@(unless round-p - '((inst fnstcw scw) ; save current control word - (move rcw scw) ; into 16-bit register - (inst or rcw (ash #b11 10)) ; CHOP - (move stack-temp rcw) - (inst fldcw stack-temp))) - (sc-case y - (signed-stack - (inst fist y)) - (signed-reg - (inst fist stack-temp) - (inst mov y stack-temp))) - ,@(unless round-p - '((inst fldcw scw))))))))) - (frob %unary-truncate single-reg single-float nil) - (frob %unary-truncate double-reg double-float nil) - - (frob %unary-round single-reg single-float t) - (frob %unary-round double-reg double-float t)) - -(macrolet ((frob (trans from-sc from-type round-p) - `(define-vop (,(symbolicate trans "/" from-type "=>UNSIGNED")) - (:args (x :scs (,from-sc) :target fr0)) - (:temporary (:sc double-reg :offset fr0-offset - :from :argument :to :result) fr0) - ,@(unless round-p - '((:temporary (:sc unsigned-stack) stack-temp) - (:temporary (:sc unsigned-stack) scw) - (:temporary (:sc any-reg) rcw))) - (:results (y :scs (unsigned-reg))) - (:arg-types ,from-type) - (:result-types unsigned-num) - (:translate ,trans) - (:policy :fast-safe) - (:note "inline float truncate") - (:vop-var vop) - (:save-p :compute-only) - (:generator 5 - ,@(unless round-p - '((note-this-location vop :internal-error) - ;; Catch any pending FPE exceptions. - (inst wait))) - ;; Normal mode (for now) is "round to best". - (unless (zerop (tn-offset x)) - (copy-fp-reg-to-fr0 x)) - ,@(unless round-p - '((inst fnstcw scw) ; save current control word - (move rcw scw) ; into 16-bit register - (inst or rcw (ash #b11 10)) ; CHOP - (move stack-temp rcw) - (inst fldcw stack-temp))) - (inst sub rsp-tn 8) - (inst fistpl (make-ea :dword :base rsp-tn)) - (inst pop y) - (inst fld fr0) ; copy fr0 to at least restore stack. - (inst add rsp-tn 8) - ,@(unless round-p - '((inst fldcw scw))))))) - (frob %unary-truncate single-reg single-float nil) - (frob %unary-truncate double-reg double-float nil) - (frob %unary-round single-reg single-float t) - (frob %unary-round double-reg double-float t)) +(macrolet ((frob (name translate inst to-sc to-type) + `(define-vop (,name) + (:args (x :scs (signed-stack signed-reg))) + (:results (y :scs (,to-sc))) + (:arg-types signed-num) + (:result-types ,to-type) + (:policy :fast-safe) + (:note "inline float coercion") + (:translate ,translate) + (:vop-var vop) + (:save-p :compute-only) + (:generator 5 + (note-this-location vop :internal-error) + (inst ,inst y x))))) + (frob %single-float/signed %single-float cvtsi2ss single-reg single-float) + (frob %double-float/signed %double-float cvtsi2sd double-reg double-float)) + +(macrolet ((frob (name translate inst from-scs from-type ea-func to-sc to-type) + `(define-vop (,name) + (:args (x :scs ,from-scs :target y)) + (:results (y :scs (,to-sc))) + (:arg-types ,from-type) + (:result-types ,to-type) + (:policy :fast-safe) + (:note "inline float coercion") + (:translate ,translate) + (:vop-var vop) + (:save-p :compute-only) + (:generator 2 + (note-this-location vop :internal-error) + (inst ,inst y (sc-case x + (,(first from-scs) x) + (,(second from-scs) (,ea-func x)))))))) + (frob %single-float/double-float %single-float cvtsd2ss + (double-reg double-stack) double-float ea-for-df-stack + single-reg single-float) + + (frob %double-float/single-float %double-float cvtss2sd + (single-reg single-stack) single-float ea-for-sf-stack + double-reg double-float)) + +(macrolet ((frob (trans inst from-scs from-type ea-func) + `(define-vop (,(symbolicate trans "/" from-type)) + (:args (x :scs ,from-scs)) + (:results (y :scs (signed-reg))) + (:arg-types ,from-type) + (:result-types signed-num) + (:translate ,trans) + (:policy :fast-safe) + (:note "inline float truncate") + (:vop-var vop) + (:save-p :compute-only) + (:generator 5 + (inst ,inst y (sc-case x + (,(first from-scs) x) + (,(second from-scs) (,ea-func x)))))))) + (frob %unary-truncate/single-float cvttss2si + (single-reg single-stack) single-float ea-for-sf-stack) + (frob %unary-truncate/double-float cvttsd2si + (double-reg double-stack) double-float ea-for-df-stack) + + (frob %unary-round cvtss2si + (single-reg single-stack) single-float ea-for-sf-stack) + (frob %unary-round cvtsd2si + (double-reg double-stack) double-float ea-for-df-stack)) (define-vop (make-single-float) (:args (bits :scs (signed-reg) :target res - :load-if (not (or (and (sc-is bits signed-stack) - (sc-is res single-reg)) - (and (sc-is bits signed-stack) - (sc-is res single-stack) - (location= bits res)))))) + :load-if (not (or (and (sc-is bits signed-stack) + (sc-is res single-reg)) + (and (sc-is bits signed-stack) + (sc-is res single-stack) + (location= bits res)))))) (:results (res :scs (single-reg single-stack))) - (:temporary (:sc signed-stack) stack-temp) (:arg-types signed-num) (:result-types single-float) (:translate make-single-float) @@ -1409,43 +1125,37 @@ (:generator 4 (sc-case res (single-stack - (sc-case bits - (signed-reg - (inst mov res bits)) - (signed-stack - (aver (location= bits res))))) + (sc-case bits + (signed-reg + (inst mov res bits)) + (signed-stack + (aver (location= bits res))))) (single-reg - (sc-case bits - (signed-reg - ;; source must be in memory - (inst mov stack-temp bits) - (with-empty-tn@fp-top(res) - (inst fld stack-temp))) - (signed-stack - (with-empty-tn@fp-top(res) - (inst fld bits)))))))) + (sc-case bits + (signed-reg + (inst movd res bits)) + (signed-stack + (inst movd res bits))))))) (define-vop (make-double-float) (:args (hi-bits :scs (signed-reg)) - (lo-bits :scs (unsigned-reg))) + (lo-bits :scs (unsigned-reg))) (:results (res :scs (double-reg))) - (:temporary (:sc double-stack) temp) + (:temporary (:sc unsigned-reg) temp) (:arg-types signed-num unsigned-num) (:result-types double-float) (:translate make-double-float) (:policy :fast-safe) (:vop-var vop) (:generator 2 - (let ((offset (1+ (tn-offset temp)))) - (storew hi-bits rbp-tn (- offset)) - (storew lo-bits rbp-tn (- (1+ offset))) - (with-empty-tn@fp-top(res) - (inst fldd (make-ea :dword :base rbp-tn - :disp (- (* (1+ offset) n-word-bytes)))))))) + (move temp hi-bits) + (inst shl temp 32) + (inst or temp lo-bits) + (inst movd res temp))) (define-vop (single-float-bits) (:args (float :scs (single-reg descriptor-reg) - :load-if (not (sc-is float single-stack)))) + :load-if (not (sc-is float single-stack)))) (:results (bits :scs (signed-reg))) (:temporary (:sc signed-stack :from :argument :to :result) stack-temp) (:arg-types single-float) @@ -1457,27 +1167,27 @@ (sc-case bits (signed-reg (sc-case float - (single-reg - (with-tn@fp-top(float) - (inst fst stack-temp) - (inst mov bits stack-temp))) - (single-stack - (inst mov bits float)) - (descriptor-reg - (loadw - bits float single-float-value-slot - other-pointer-lowtag)))) + (single-reg + (inst movss stack-temp float) + (move bits stack-temp)) + (single-stack + (move bits float)) + (descriptor-reg + (move bits float) + (inst shr bits 32)))) (signed-stack (sc-case float - (single-reg - (with-tn@fp-top(float) - (inst fst bits)))))))) + (single-reg + (inst movss bits float))))) + ;; Sign-extend + (inst shl bits 32) + (inst sar bits 32))) (define-vop (double-float-high-bits) (:args (float :scs (double-reg descriptor-reg) - :load-if (not (sc-is float double-stack)))) + :load-if (not (sc-is float double-stack)))) (:results (hi-bits :scs (signed-reg))) - (:temporary (:sc double-stack) temp) + (:temporary (:sc signed-stack :from :argument :to :result) temp) (:arg-types double-float) (:result-types signed-num) (:translate double-float-high-bits) @@ -1486,23 +1196,20 @@ (:generator 5 (sc-case float (double-reg - (with-tn@fp-top(float) - (let ((where (make-ea :dword :base rbp-tn - :disp (- (* (+ 2 (tn-offset temp)) - n-word-bytes))))) - (inst fstd where))) - (loadw hi-bits rbp-tn (- (1+ (tn-offset temp))))) + (inst movsd temp float) + (move hi-bits temp)) (double-stack - (loadw hi-bits rbp-tn (- (1+ (tn-offset float))))) + (loadw hi-bits ebp-tn (frame-word-offset (tn-offset float)))) (descriptor-reg - (loadw hi-bits float (1+ double-float-value-slot) - other-pointer-lowtag))))) + (loadw hi-bits float double-float-value-slot + other-pointer-lowtag))) + (inst sar hi-bits 32))) (define-vop (double-float-low-bits) (:args (float :scs (double-reg descriptor-reg) - :load-if (not (sc-is float double-stack)))) + :load-if (not (sc-is float double-stack)))) (:results (lo-bits :scs (unsigned-reg))) - (:temporary (:sc double-stack) temp) + (:temporary (:sc signed-stack :from :argument :to :result) temp) (:arg-types double-float) (:result-types unsigned-num) (:translate double-float-low-bits) @@ -1511,1294 +1218,116 @@ (:generator 5 (sc-case float (double-reg - (with-tn@fp-top(float) - (let ((where (make-ea :dword :base rbp-tn - :disp (- (* (+ 2 (tn-offset temp)) - n-word-bytes))))) - (inst fstd where))) - (loadw lo-bits rbp-tn (- (+ 2 (tn-offset temp))))) + (inst movsd temp float) + (move lo-bits temp)) (double-stack - (loadw lo-bits rbp-tn (- (+ 2 (tn-offset float))))) + (loadw lo-bits ebp-tn (frame-word-offset (tn-offset float)))) (descriptor-reg - (loadw lo-bits float double-float-value-slot - other-pointer-lowtag))))) - - -;;;; float mode hackery + (loadw lo-bits float double-float-value-slot + other-pointer-lowtag))) + (inst shl lo-bits 32) + (inst shr lo-bits 32))) -(sb!xc:deftype float-modes () '(unsigned-byte 64)) ; really only 16 -(defknown floating-point-modes () float-modes (flushable)) -(defknown ((setf floating-point-modes)) (float-modes) - float-modes) - -(def!constant npx-env-size (* 7 n-word-bytes)) -(def!constant npx-cw-offset 0) -(def!constant npx-sw-offset 4) - -(define-vop (floating-point-modes) - (:results (res :scs (unsigned-reg))) - (:result-types unsigned-num) - (:translate floating-point-modes) - (:policy :fast-safe) - (:temporary (:sc unsigned-reg :offset eax-offset :target res - :to :result) eax) - (:generator 8 - (inst sub rsp-tn npx-env-size) ; Make space on stack. - (inst wait) ; Catch any pending FPE exceptions - (inst fstenv (make-ea :dword :base rsp-tn)) ; masks all exceptions - (inst fldenv (make-ea :dword :base rsp-tn)) ; Restore previous state. - ;; Move current status to high word. - (inst movzxd eax (make-ea :dword :base rsp-tn :disp (- npx-sw-offset 2))) - ;; Move exception mask to low word. - (inst mov ax-tn (make-ea :word :base rsp-tn :disp npx-cw-offset)) - (inst add rsp-tn npx-env-size) ; Pop stack. - (inst xor eax #x3f) ; Flip exception mask to trap enable bits. - (move res eax))) - -;;; XXX BROKEN -(define-vop (set-floating-point-modes) - (:args (new :scs (unsigned-reg) :to :result :target res)) - (:results (res :scs (unsigned-reg))) - (:arg-types unsigned-num) - (:result-types unsigned-num) - (:translate (setf floating-point-modes)) - (:policy :fast-safe) - (:temporary (:sc unsigned-reg :offset eax-offset - :from :eval :to :result) eax) - (:generator 3 - (inst sub rsp-tn npx-env-size) ; Make space on stack. - (inst wait) ; Catch any pending FPE exceptions. - (inst fstenv (make-ea :dword :base rsp-tn)) - (inst mov eax new) - (inst xor eax #x3f) ; Turn trap enable bits into exception mask. - (inst mov (make-ea :word :base rsp-tn :disp npx-cw-offset) ax-tn) - (inst shr eax 16) ; position status word - (inst mov (make-ea :word :base rsp-tn :disp npx-sw-offset) ax-tn) - (inst fldenv (make-ea :dword :base rsp-tn)) - (inst add rsp-tn npx-env-size) ; Pop stack. - (move res new))) - - -(progn - -;;; Let's use some of the 80387 special functions. -;;; -;;; These defs will not take effect unless code/irrat.lisp is modified -;;; to remove the inlined alien routine def. - -(macrolet ((frob (func trans op) - `(define-vop (,func) - (:args (x :scs (double-reg) :target fr0)) - (:temporary (:sc double-reg :offset fr0-offset - :from :argument :to :result) fr0) - (:ignore fr0) - (:results (y :scs (double-reg))) - (:arg-types double-float) - (:result-types double-float) - (:translate ,trans) - (:policy :fast-safe) - (:note "inline NPX function") - (:vop-var vop) - (:save-p :compute-only) - (:node-var node) - (:generator 5 - (note-this-location vop :internal-error) - (unless (zerop (tn-offset x)) - (inst fxch x) ; x to top of stack - (unless (location= x y) - (inst fst x))) ; maybe save it - (inst ,op) ; clobber st0 - (cond ((zerop (tn-offset y)) - (maybe-fp-wait node)) - (t - (inst fst y))))))) - - ;; Quick versions of fsin and fcos that require the argument to be - ;; within range 2^63. - (frob fsin-quick %sin-quick fsin) - (frob fcos-quick %cos-quick fcos) - (frob fsqrt %sqrt fsqrt)) - -;;; Quick version of ftan that requires the argument to be within -;;; range 2^63. -(define-vop (ftan-quick) - (:translate %tan-quick) - (:args (x :scs (double-reg) :target fr0)) - (:temporary (:sc double-reg :offset fr0-offset - :from :argument :to :result) fr0) - (:temporary (:sc double-reg :offset fr1-offset - :from :argument :to :result) fr1) - (:results (y :scs (double-reg))) - (:arg-types double-float) - (:result-types double-float) - (:policy :fast-safe) - (:note "inline tan function") - (:vop-var vop) - (:save-p :compute-only) - (:generator 5 - (note-this-location vop :internal-error) - (case (tn-offset x) - (0 - (inst fstp fr1)) - (1 - (inst fstp fr0)) - (t - (inst fstp fr0) - (inst fstp fr0) - (inst fldd (make-random-tn :kind :normal - :sc (sc-or-lose 'double-reg) - :offset (- (tn-offset x) 2))))) - (inst fptan) - ;; Result is in fr1 - (case (tn-offset y) - (0 - (inst fxch fr1)) - (1) - (t - (inst fxch fr1) - (inst fstd y))))) - -;;; These versions of fsin, fcos, and ftan try to use argument -;;; reduction but to do this accurately requires greater precision and -;;; it is hopelessly inaccurate. -#+nil -(macrolet ((frob (func trans op) - `(define-vop (,func) - (:translate ,trans) - (:args (x :scs (double-reg) :target fr0)) - (:temporary (:sc unsigned-reg :offset eax-offset - :from :eval :to :result) eax) - (:temporary (:sc unsigned-reg :offset fr0-offset - :from :argument :to :result) fr0) - (:temporary (:sc unsigned-reg :offset fr1-offset - :from :argument :to :result) fr1) - (:results (y :scs (double-reg))) - (:arg-types double-float) - (:result-types double-float) - (:policy :fast-safe) - (:note "inline sin/cos function") - (:vop-var vop) - (:save-p :compute-only) - (:ignore eax) - (:generator 5 - (note-this-location vop :internal-error) - (unless (zerop (tn-offset x)) - (inst fxch x) ; x to top of stack - (unless (location= x y) - (inst fst x))) ; maybe save it - (inst ,op) - (inst fnstsw) ; status word to ax - (inst and ah-tn #x04) ; C2 - (inst jmp :z DONE) - ;; Else x was out of range so reduce it; ST0 is unchanged. - (inst fstp fr1) ; Load 2*PI - (inst fldpi) - (inst fadd fr0) - (inst fxch fr1) - LOOP - (inst fprem1) - (inst fnstsw) ; status word to ax - (inst and ah-tn #x04) ; C2 - (inst jmp :nz LOOP) - (inst ,op) - DONE - (unless (zerop (tn-offset y)) - (inst fstd y)))))) - (frob fsin %sin fsin) - (frob fcos %cos fcos)) - - - -;;; These versions of fsin, fcos, and ftan simply load a 0.0 result if -;;; the argument is out of range 2^63 and would thus be hopelessly -;;; inaccurate. -(macrolet ((frob (func trans op) - `(define-vop (,func) - (:translate ,trans) - (:args (x :scs (double-reg) :target fr0)) - (:temporary (:sc double-reg :offset fr0-offset - :from :argument :to :result) fr0) - (:temporary (:sc unsigned-reg :offset eax-offset - :from :argument :to :result) eax) - (:results (y :scs (double-reg))) - (:arg-types double-float) - (:result-types double-float) - (:policy :fast-safe) - (:note "inline sin/cos function") - (:vop-var vop) - (:save-p :compute-only) - (:ignore eax) - (:generator 5 - (note-this-location vop :internal-error) - (unless (zerop (tn-offset x)) - (inst fxch x) ; x to top of stack - (unless (location= x y) - (inst fst x))) ; maybe save it - (inst ,op) - (inst fnstsw) ; status word to ax - (inst and ah-tn #x04) ; C2 - (inst jmp :z DONE) - ;; Else x was out of range so reduce it; ST0 is unchanged. - (inst fstp fr0) ; Load 0.0 - (inst fldz) - DONE - (unless (zerop (tn-offset y)) - (inst fstd y)))))) - (frob fsin %sin fsin) - (frob fcos %cos fcos)) - -(define-vop (ftan) - (:translate %tan) - (:args (x :scs (double-reg) :target fr0)) - (:temporary (:sc double-reg :offset fr0-offset - :from :argument :to :result) fr0) - (:temporary (:sc double-reg :offset fr1-offset - :from :argument :to :result) fr1) - (:temporary (:sc unsigned-reg :offset eax-offset - :from :argument :to :result) eax) - (:results (y :scs (double-reg))) - (:arg-types double-float) - (:result-types double-float) - (:ignore eax) - (:policy :fast-safe) - (:note "inline tan function") - (:vop-var vop) - (:save-p :compute-only) - (:ignore eax) - (:generator 5 - (note-this-location vop :internal-error) - (case (tn-offset x) - (0 - (inst fstp fr1)) - (1 - (inst fstp fr0)) - (t - (inst fstp fr0) - (inst fstp fr0) - (inst fldd (make-random-tn :kind :normal - :sc (sc-or-lose 'double-reg) - :offset (- (tn-offset x) 2))))) - (inst fptan) - (inst fnstsw) ; status word to ax - (inst and ah-tn #x04) ; C2 - (inst jmp :z DONE) - ;; Else x was out of range so reduce it; ST0 is unchanged. - (inst fldz) ; Load 0.0 - (inst fxch fr1) - DONE - ;; Result is in fr1 - (case (tn-offset y) - (0 - (inst fxch fr1)) - (1) - (t - (inst fxch fr1) - (inst fstd y))))) - -#+nil -(define-vop (fexp) - (:translate %exp) - (:args (x :scs (double-reg double-stack descriptor-reg) :target fr0)) - (:temporary (:sc double-reg :offset fr0-offset - :from :argument :to :result) fr0) - (:temporary (:sc double-reg :offset fr1-offset - :from :argument :to :result) fr1) - (:temporary (:sc double-reg :offset fr2-offset - :from :argument :to :result) fr2) - (:results (y :scs (double-reg))) - (:arg-types double-float) - (:result-types double-float) - (:policy :fast-safe) - (:note "inline exp function") - (:vop-var vop) - (:save-p :compute-only) - (:generator 5 - (note-this-location vop :internal-error) - (sc-case x - (double-reg - (cond ((zerop (tn-offset x)) - ;; x is in fr0 - (inst fstp fr1) - (inst fldl2e) - (inst fmul fr1)) - (t - ;; x is in a FP reg, not fr0 - (inst fstp fr0) - (inst fldl2e) - (inst fmul x)))) - ((double-stack descriptor-reg) - (inst fstp fr0) - (inst fldl2e) - (if (sc-is x double-stack) - (inst fmuld (ea-for-df-stack x)) - (inst fmuld (ea-for-df-desc x))))) - ;; Now fr0=x log2(e) - (inst fst fr1) - (inst frndint) - (inst fst fr2) - (inst fsubp-sti fr1) - (inst f2xm1) - (inst fld1) - (inst faddp-sti fr1) - (inst fscale) - (inst fld fr0) - (case (tn-offset y) - ((0 1)) - (t (inst fstd y))))) - -;;; Modified exp that handles the following special cases: -;;; exp(+Inf) is +Inf; exp(-Inf) is 0; exp(NaN) is NaN. -(define-vop (fexp) - (:translate %exp) - (:args (x :scs (double-reg) :target fr0)) - (:temporary (:sc word-reg :offset eax-offset :from :eval :to :result) temp) - (:temporary (:sc double-reg :offset fr0-offset - :from :argument :to :result) fr0) - (:temporary (:sc double-reg :offset fr1-offset - :from :argument :to :result) fr1) - (:temporary (:sc double-reg :offset fr2-offset - :from :argument :to :result) fr2) - (:results (y :scs (double-reg))) - (:arg-types double-float) - (:result-types double-float) - (:policy :fast-safe) - (:note "inline exp function") - (:vop-var vop) - (:save-p :compute-only) - (:ignore temp) - (:generator 5 - (note-this-location vop :internal-error) - (unless (zerop (tn-offset x)) - (inst fxch x) ; x to top of stack - (unless (location= x y) - (inst fst x))) ; maybe save it - ;; Check for Inf or NaN - (inst fxam) - (inst fnstsw) - (inst sahf) - (inst jmp :nc NOINFNAN) ; Neither Inf or NaN. - (inst jmp :np NOINFNAN) ; NaN gives NaN? Continue. - (inst and ah-tn #x02) ; Test sign of Inf. - (inst jmp :z DONE) ; +Inf gives +Inf. - (inst fstp fr0) ; -Inf gives 0 - (inst fldz) - (inst jmp-short DONE) - NOINFNAN - (inst fstp fr1) - (inst fldl2e) - (inst fmul fr1) - ;; Now fr0=x log2(e) - (inst fst fr1) - (inst frndint) - (inst fst fr2) - (inst fsubp-sti fr1) - (inst f2xm1) - (inst fld1) - (inst faddp-sti fr1) - (inst fscale) - (inst fld fr0) - DONE - (unless (zerop (tn-offset y)) - (inst fstd y)))) - -;;; Expm1 = exp(x) - 1. -;;; Handles the following special cases: -;;; expm1(+Inf) is +Inf; expm1(-Inf) is -1.0; expm1(NaN) is NaN. -(define-vop (fexpm1) - (:translate %expm1) - (:args (x :scs (double-reg) :target fr0)) - (:temporary (:sc word-reg :offset eax-offset :from :eval :to :result) temp) - (:temporary (:sc double-reg :offset fr0-offset - :from :argument :to :result) fr0) - (:temporary (:sc double-reg :offset fr1-offset - :from :argument :to :result) fr1) - (:temporary (:sc double-reg :offset fr2-offset - :from :argument :to :result) fr2) - (:results (y :scs (double-reg))) - (:arg-types double-float) - (:result-types double-float) - (:policy :fast-safe) - (:note "inline expm1 function") - (:vop-var vop) - (:save-p :compute-only) - (:ignore temp) - (:generator 5 - (note-this-location vop :internal-error) - (unless (zerop (tn-offset x)) - (inst fxch x) ; x to top of stack - (unless (location= x y) - (inst fst x))) ; maybe save it - ;; Check for Inf or NaN - (inst fxam) - (inst fnstsw) - (inst sahf) - (inst jmp :nc NOINFNAN) ; Neither Inf or NaN. - (inst jmp :np NOINFNAN) ; NaN gives NaN? Continue. - (inst and ah-tn #x02) ; Test sign of Inf. - (inst jmp :z DONE) ; +Inf gives +Inf. - (inst fstp fr0) ; -Inf gives -1.0 - (inst fld1) - (inst fchs) - (inst jmp-short DONE) - NOINFNAN - ;; Free two stack slots leaving the argument on top. - (inst fstp fr2) - (inst fstp fr0) - (inst fldl2e) - (inst fmul fr1) ; Now fr0 = x log2(e) - (inst fst fr1) - (inst frndint) - (inst fsub-sti fr1) - (inst fxch fr1) - (inst f2xm1) - (inst fscale) - (inst fxch fr1) - (inst fld1) - (inst fscale) - (inst fstp fr1) - (inst fld1) - (inst fsub fr1) - (inst fsubr fr2) - DONE - (unless (zerop (tn-offset y)) - (inst fstd y)))) - -(define-vop (flog) - (:translate %log) - (:args (x :scs (double-reg double-stack descriptor-reg) :target fr0)) - (:temporary (:sc double-reg :offset fr0-offset - :from :argument :to :result) fr0) - (:temporary (:sc double-reg :offset fr1-offset - :from :argument :to :result) fr1) - (:results (y :scs (double-reg))) - (:arg-types double-float) - (:result-types double-float) - (:policy :fast-safe) - (:note "inline log function") - (:vop-var vop) - (:save-p :compute-only) - (:generator 5 - (note-this-location vop :internal-error) - (sc-case x - (double-reg - (case (tn-offset x) - (0 - ;; x is in fr0 - (inst fstp fr1) - (inst fldln2) - (inst fxch fr1)) - (1 - ;; x is in fr1 - (inst fstp fr0) - (inst fldln2) - (inst fxch fr1)) - (t - ;; x is in a FP reg, not fr0 or fr1 - (inst fstp fr0) - (inst fstp fr0) - (inst fldln2) - (inst fldd (make-random-tn :kind :normal - :sc (sc-or-lose 'double-reg) - :offset (1- (tn-offset x)))))) - (inst fyl2x)) - ((double-stack descriptor-reg) - (inst fstp fr0) - (inst fstp fr0) - (inst fldln2) - (if (sc-is x double-stack) - (inst fldd (ea-for-df-stack x)) - (inst fldd (ea-for-df-desc x))) - (inst fyl2x))) - (inst fld fr0) - (case (tn-offset y) - ((0 1)) - (t (inst fstd y))))) - -(define-vop (flog10) - (:translate %log10) - (:args (x :scs (double-reg double-stack descriptor-reg) :target fr0)) - (:temporary (:sc double-reg :offset fr0-offset - :from :argument :to :result) fr0) - (:temporary (:sc double-reg :offset fr1-offset - :from :argument :to :result) fr1) - (:results (y :scs (double-reg))) - (:arg-types double-float) - (:result-types double-float) - (:policy :fast-safe) - (:note "inline log10 function") - (:vop-var vop) - (:save-p :compute-only) - (:generator 5 - (note-this-location vop :internal-error) - (sc-case x - (double-reg - (case (tn-offset x) - (0 - ;; x is in fr0 - (inst fstp fr1) - (inst fldlg2) - (inst fxch fr1)) - (1 - ;; x is in fr1 - (inst fstp fr0) - (inst fldlg2) - (inst fxch fr1)) - (t - ;; x is in a FP reg, not fr0 or fr1 - (inst fstp fr0) - (inst fstp fr0) - (inst fldlg2) - (inst fldd (make-random-tn :kind :normal - :sc (sc-or-lose 'double-reg) - :offset (1- (tn-offset x)))))) - (inst fyl2x)) - ((double-stack descriptor-reg) - (inst fstp fr0) - (inst fstp fr0) - (inst fldlg2) - (if (sc-is x double-stack) - (inst fldd (ea-for-df-stack x)) - (inst fldd (ea-for-df-desc x))) - (inst fyl2x))) - (inst fld fr0) - (case (tn-offset y) - ((0 1)) - (t (inst fstd y))))) - -(define-vop (fpow) - (:translate %pow) - (:args (x :scs (double-reg double-stack descriptor-reg) :target fr0) - (y :scs (double-reg double-stack descriptor-reg) :target fr1)) - (:temporary (:sc double-reg :offset fr0-offset - :from (:argument 0) :to :result) fr0) - (:temporary (:sc double-reg :offset fr1-offset - :from (:argument 1) :to :result) fr1) - (:temporary (:sc double-reg :offset fr2-offset - :from :load :to :result) fr2) - (:results (r :scs (double-reg))) - (:arg-types double-float double-float) - (:result-types double-float) - (:policy :fast-safe) - (:note "inline pow function") - (:vop-var vop) - (:save-p :compute-only) - (:generator 5 - (note-this-location vop :internal-error) - ;; Setup x in fr0 and y in fr1 - (cond - ;; x in fr0; y in fr1 - ((and (sc-is x double-reg) (zerop (tn-offset x)) - (sc-is y double-reg) (= 1 (tn-offset y)))) - ;; y in fr1; x not in fr0 - ((and (sc-is y double-reg) (= 1 (tn-offset y))) - ;; Load x to fr0 - (sc-case x - (double-reg - (copy-fp-reg-to-fr0 x)) - (double-stack - (inst fstp fr0) - (inst fldd (ea-for-df-stack x))) - (descriptor-reg - (inst fstp fr0) - (inst fldd (ea-for-df-desc x))))) - ;; x in fr0; y not in fr1 - ((and (sc-is x double-reg) (zerop (tn-offset x))) - (inst fxch fr1) - ;; Now load y to fr0 - (sc-case y - (double-reg - (copy-fp-reg-to-fr0 y)) - (double-stack - (inst fstp fr0) - (inst fldd (ea-for-df-stack y))) - (descriptor-reg - (inst fstp fr0) - (inst fldd (ea-for-df-desc y)))) - (inst fxch fr1)) - ;; x in fr1; y not in fr1 - ((and (sc-is x double-reg) (= 1 (tn-offset x))) - ;; Load y to fr0 - (sc-case y - (double-reg - (copy-fp-reg-to-fr0 y)) - (double-stack - (inst fstp fr0) - (inst fldd (ea-for-df-stack y))) - (descriptor-reg - (inst fstp fr0) - (inst fldd (ea-for-df-desc y)))) - (inst fxch fr1)) - ;; y in fr0; - ((and (sc-is y double-reg) (zerop (tn-offset y))) - (inst fxch fr1) - ;; Now load x to fr0 - (sc-case x - (double-reg - (copy-fp-reg-to-fr0 x)) - (double-stack - (inst fstp fr0) - (inst fldd (ea-for-df-stack x))) - (descriptor-reg - (inst fstp fr0) - (inst fldd (ea-for-df-desc x))))) - ;; Neither x or y are in either fr0 or fr1 - (t - ;; Load y then x - (inst fstp fr0) - (inst fstp fr0) - (sc-case y - (double-reg - (inst fldd (make-random-tn :kind :normal - :sc (sc-or-lose 'double-reg) - :offset (- (tn-offset y) 2)))) - (double-stack - (inst fldd (ea-for-df-stack y))) - (descriptor-reg - (inst fldd (ea-for-df-desc y)))) - ;; Load x to fr0 - (sc-case x - (double-reg - (inst fldd (make-random-tn :kind :normal - :sc (sc-or-lose 'double-reg) - :offset (1- (tn-offset x))))) - (double-stack - (inst fldd (ea-for-df-stack x))) - (descriptor-reg - (inst fldd (ea-for-df-desc x)))))) - - ;; Now have x at fr0; and y at fr1 - (inst fyl2x) - ;; Now fr0=y log2(x) - (inst fld fr0) - (inst frndint) - (inst fst fr2) - (inst fsubp-sti fr1) - (inst f2xm1) - (inst fld1) - (inst faddp-sti fr1) - (inst fscale) - (inst fld fr0) - (case (tn-offset r) - ((0 1)) - (t (inst fstd r))))) - -(define-vop (fscalen) - (:translate %scalbn) - (:args (x :scs (double-reg double-stack descriptor-reg) :target fr0) - (y :scs (signed-stack signed-reg) :target temp)) - (:temporary (:sc double-reg :offset fr0-offset - :from (:argument 0) :to :result) fr0) - (:temporary (:sc double-reg :offset fr1-offset :from :eval :to :result) fr1) - (:temporary (:sc signed-stack :from (:argument 1) :to :result) temp) - (:results (r :scs (double-reg))) - (:arg-types double-float signed-num) - (:result-types double-float) - (:policy :fast-safe) - (:note "inline scalbn function") - (:generator 5 - ;; Setup x in fr0 and y in fr1 - (sc-case x - (double-reg - (case (tn-offset x) - (0 - (inst fstp fr1) - (sc-case y - (signed-reg - (inst mov temp y) - (inst fild temp)) - (signed-stack - (inst fild y))) - (inst fxch fr1)) - (1 - (inst fstp fr0) - (sc-case y - (signed-reg - (inst mov temp y) - (inst fild temp)) - (signed-stack - (inst fild y))) - (inst fxch fr1)) - (t - (inst fstp fr0) - (inst fstp fr0) - (sc-case y - (signed-reg - (inst mov temp y) - (inst fild temp)) - (signed-stack - (inst fild y))) - (inst fld (make-random-tn :kind :normal - :sc (sc-or-lose 'double-reg) - :offset (1- (tn-offset x))))))) - ((double-stack descriptor-reg) - (inst fstp fr0) - (inst fstp fr0) - (sc-case y - (signed-reg - (inst mov temp y) - (inst fild temp)) - (signed-stack - (inst fild y))) - (if (sc-is x double-stack) - (inst fldd (ea-for-df-stack x)) - (inst fldd (ea-for-df-desc x))))) - (inst fscale) - (unless (zerop (tn-offset r)) - (inst fstd r)))) - -(define-vop (fscale) - (:translate %scalb) - (:args (x :scs (double-reg double-stack descriptor-reg) :target fr0) - (y :scs (double-reg double-stack descriptor-reg) :target fr1)) - (:temporary (:sc double-reg :offset fr0-offset - :from (:argument 0) :to :result) fr0) - (:temporary (:sc double-reg :offset fr1-offset - :from (:argument 1) :to :result) fr1) - (:results (r :scs (double-reg))) - (:arg-types double-float double-float) - (:result-types double-float) - (:policy :fast-safe) - (:note "inline scalb function") - (:vop-var vop) - (:save-p :compute-only) - (:generator 5 - (note-this-location vop :internal-error) - ;; Setup x in fr0 and y in fr1 - (cond - ;; x in fr0; y in fr1 - ((and (sc-is x double-reg) (zerop (tn-offset x)) - (sc-is y double-reg) (= 1 (tn-offset y)))) - ;; y in fr1; x not in fr0 - ((and (sc-is y double-reg) (= 1 (tn-offset y))) - ;; Load x to fr0 - (sc-case x - (double-reg - (copy-fp-reg-to-fr0 x)) - (double-stack - (inst fstp fr0) - (inst fldd (ea-for-df-stack x))) - (descriptor-reg - (inst fstp fr0) - (inst fldd (ea-for-df-desc x))))) - ;; x in fr0; y not in fr1 - ((and (sc-is x double-reg) (zerop (tn-offset x))) - (inst fxch fr1) - ;; Now load y to fr0 - (sc-case y - (double-reg - (copy-fp-reg-to-fr0 y)) - (double-stack - (inst fstp fr0) - (inst fldd (ea-for-df-stack y))) - (descriptor-reg - (inst fstp fr0) - (inst fldd (ea-for-df-desc y)))) - (inst fxch fr1)) - ;; x in fr1; y not in fr1 - ((and (sc-is x double-reg) (= 1 (tn-offset x))) - ;; Load y to fr0 - (sc-case y - (double-reg - (copy-fp-reg-to-fr0 y)) - (double-stack - (inst fstp fr0) - (inst fldd (ea-for-df-stack y))) - (descriptor-reg - (inst fstp fr0) - (inst fldd (ea-for-df-desc y)))) - (inst fxch fr1)) - ;; y in fr0; - ((and (sc-is y double-reg) (zerop (tn-offset y))) - (inst fxch fr1) - ;; Now load x to fr0 - (sc-case x - (double-reg - (copy-fp-reg-to-fr0 x)) - (double-stack - (inst fstp fr0) - (inst fldd (ea-for-df-stack x))) - (descriptor-reg - (inst fstp fr0) - (inst fldd (ea-for-df-desc x))))) - ;; Neither x or y are in either fr0 or fr1 - (t - ;; Load y then x - (inst fstp fr0) - (inst fstp fr0) - (sc-case y - (double-reg - (inst fldd (make-random-tn :kind :normal - :sc (sc-or-lose 'double-reg) - :offset (- (tn-offset y) 2)))) - (double-stack - (inst fldd (ea-for-df-stack y))) - (descriptor-reg - (inst fldd (ea-for-df-desc y)))) - ;; Load x to fr0 - (sc-case x - (double-reg - (inst fldd (make-random-tn :kind :normal - :sc (sc-or-lose 'double-reg) - :offset (1- (tn-offset x))))) - (double-stack - (inst fldd (ea-for-df-stack x))) - (descriptor-reg - (inst fldd (ea-for-df-desc x)))))) - - ;; Now have x at fr0; and y at fr1 - (inst fscale) - (unless (zerop (tn-offset r)) - (inst fstd r)))) - -(define-vop (flog1p) - (:translate %log1p) - (:args (x :scs (double-reg) :to :result)) - (:temporary (:sc double-reg :offset fr0-offset - :from :argument :to :result) fr0) - (:temporary (:sc double-reg :offset fr1-offset - :from :argument :to :result) fr1) - (:temporary (:sc word-reg :offset eax-offset :from :eval) temp) - (:results (y :scs (double-reg))) - (:arg-types double-float) - (:result-types double-float) - (:policy :fast-safe) - (:note "inline log1p function") - (:ignore temp) - (:generator 5 - ;; x is in a FP reg, not fr0, fr1. - (inst fstp fr0) - (inst fstp fr0) - (inst fldd (make-random-tn :kind :normal - :sc (sc-or-lose 'double-reg) - :offset (- (tn-offset x) 2))) - ;; Check the range - (inst push #x3e947ae1) ; Constant 0.29 - (inst fabs) - (inst fld (make-ea :dword :base rsp-tn)) - (inst fcompp) - (inst add rsp-tn 4) - (inst fnstsw) ; status word to ax - (inst and ah-tn #x45) - (inst jmp :z WITHIN-RANGE) - ;; Out of range for fyl2xp1. - (inst fld1) - (inst faddd (make-random-tn :kind :normal - :sc (sc-or-lose 'double-reg) - :offset (- (tn-offset x) 1))) - (inst fldln2) - (inst fxch fr1) - (inst fyl2x) - (inst jmp DONE) - - WITHIN-RANGE - (inst fldln2) - (inst fldd (make-random-tn :kind :normal - :sc (sc-or-lose 'double-reg) - :offset (- (tn-offset x) 1))) - (inst fyl2xp1) - DONE - (inst fld fr0) - (case (tn-offset y) - ((0 1)) - (t (inst fstd y))))) - -;;; The Pentium has a less restricted implementation of the fyl2xp1 -;;; instruction and a range check can be avoided. -(define-vop (flog1p-pentium) - (:translate %log1p) - (:args (x :scs (double-reg double-stack descriptor-reg) :target fr0)) - (:temporary (:sc double-reg :offset fr0-offset - :from :argument :to :result) fr0) - (:temporary (:sc double-reg :offset fr1-offset - :from :argument :to :result) fr1) - (:results (y :scs (double-reg))) - (:arg-types double-float) - (:result-types double-float) - (:policy :fast-safe) - (:guard (member :pentium-style-fyl2xp1 *backend-subfeatures*)) - (:note "inline log1p with limited x range function") - (:vop-var vop) - (:save-p :compute-only) - (:generator 4 - (note-this-location vop :internal-error) - (sc-case x - (double-reg - (case (tn-offset x) - (0 - ;; x is in fr0 - (inst fstp fr1) - (inst fldln2) - (inst fxch fr1)) - (1 - ;; x is in fr1 - (inst fstp fr0) - (inst fldln2) - (inst fxch fr1)) - (t - ;; x is in a FP reg, not fr0 or fr1 - (inst fstp fr0) - (inst fstp fr0) - (inst fldln2) - (inst fldd (make-random-tn :kind :normal - :sc (sc-or-lose 'double-reg) - :offset (1- (tn-offset x))))))) - ((double-stack descriptor-reg) - (inst fstp fr0) - (inst fstp fr0) - (inst fldln2) - (if (sc-is x double-stack) - (inst fldd (ea-for-df-stack x)) - (inst fldd (ea-for-df-desc x))))) - (inst fyl2xp1) - (inst fld fr0) - (case (tn-offset y) - ((0 1)) - (t (inst fstd y))))) - -(define-vop (flogb) - (:translate %logb) - (:args (x :scs (double-reg double-stack descriptor-reg) :target fr0)) - (:temporary (:sc double-reg :offset fr0-offset - :from :argument :to :result) fr0) - (:temporary (:sc double-reg :offset fr1-offset - :from :argument :to :result) fr1) - (:results (y :scs (double-reg))) - (:arg-types double-float) - (:result-types double-float) - (:policy :fast-safe) - (:note "inline logb function") - (:vop-var vop) - (:save-p :compute-only) - (:generator 5 - (note-this-location vop :internal-error) - (sc-case x - (double-reg - (case (tn-offset x) - (0 - ;; x is in fr0 - (inst fstp fr1)) - (1 - ;; x is in fr1 - (inst fstp fr0)) - (t - ;; x is in a FP reg, not fr0 or fr1 - (inst fstp fr0) - (inst fstp fr0) - (inst fldd (make-random-tn :kind :normal - :sc (sc-or-lose 'double-reg) - :offset (- (tn-offset x) 2)))))) - ((double-stack descriptor-reg) - (inst fstp fr0) - (inst fstp fr0) - (if (sc-is x double-stack) - (inst fldd (ea-for-df-stack x)) - (inst fldd (ea-for-df-desc x))))) - (inst fxtract) - (case (tn-offset y) - (0 - (inst fxch fr1)) - (1) - (t (inst fxch fr1) - (inst fstd y))))) - -(define-vop (fatan) - (:translate %atan) - (:args (x :scs (double-reg double-stack descriptor-reg) :target fr0)) - (:temporary (:sc double-reg :offset fr0-offset - :from (:argument 0) :to :result) fr0) - (:temporary (:sc double-reg :offset fr1-offset - :from (:argument 0) :to :result) fr1) - (:results (r :scs (double-reg))) - (:arg-types double-float) - (:result-types double-float) - (:policy :fast-safe) - (:note "inline atan function") - (:vop-var vop) - (:save-p :compute-only) - (:generator 5 - (note-this-location vop :internal-error) - ;; Setup x in fr1 and 1.0 in fr0 - (cond - ;; x in fr0 - ((and (sc-is x double-reg) (zerop (tn-offset x))) - (inst fstp fr1)) - ;; x in fr1 - ((and (sc-is x double-reg) (= 1 (tn-offset x))) - (inst fstp fr0)) - ;; x not in fr0 or fr1 - (t - ;; Load x then 1.0 - (inst fstp fr0) - (inst fstp fr0) - (sc-case x - (double-reg - (inst fldd (make-random-tn :kind :normal - :sc (sc-or-lose 'double-reg) - :offset (- (tn-offset x) 2)))) - (double-stack - (inst fldd (ea-for-df-stack x))) - (descriptor-reg - (inst fldd (ea-for-df-desc x)))))) - (inst fld1) - ;; Now have x at fr1; and 1.0 at fr0 - (inst fpatan) - (inst fld fr0) - (case (tn-offset r) - ((0 1)) - (t (inst fstd r))))) - -(define-vop (fatan2) - (:translate %atan2) - (:args (x :scs (double-reg double-stack descriptor-reg) :target fr1) - (y :scs (double-reg double-stack descriptor-reg) :target fr0)) - (:temporary (:sc double-reg :offset fr0-offset - :from (:argument 1) :to :result) fr0) - (:temporary (:sc double-reg :offset fr1-offset - :from (:argument 0) :to :result) fr1) - (:results (r :scs (double-reg))) - (:arg-types double-float double-float) - (:result-types double-float) - (:policy :fast-safe) - (:note "inline atan2 function") - (:vop-var vop) - (:save-p :compute-only) - (:generator 5 - (note-this-location vop :internal-error) - ;; Setup x in fr1 and y in fr0 - (cond - ;; y in fr0; x in fr1 - ((and (sc-is y double-reg) (zerop (tn-offset y)) - (sc-is x double-reg) (= 1 (tn-offset x)))) - ;; x in fr1; y not in fr0 - ((and (sc-is x double-reg) (= 1 (tn-offset x))) - ;; Load y to fr0 - (sc-case y - (double-reg - (copy-fp-reg-to-fr0 y)) - (double-stack - (inst fstp fr0) - (inst fldd (ea-for-df-stack y))) - (descriptor-reg - (inst fstp fr0) - (inst fldd (ea-for-df-desc y))))) - ((and (sc-is x double-reg) (zerop (tn-offset x)) - (sc-is y double-reg) (zerop (tn-offset x))) - ;; copy x to fr1 - (inst fst fr1)) - ;; y in fr0; x not in fr1 - ((and (sc-is y double-reg) (zerop (tn-offset y))) - (inst fxch fr1) - ;; Now load x to fr0 - (sc-case x - (double-reg - (copy-fp-reg-to-fr0 x)) - (double-stack - (inst fstp fr0) - (inst fldd (ea-for-df-stack x))) - (descriptor-reg - (inst fstp fr0) - (inst fldd (ea-for-df-desc x)))) - (inst fxch fr1)) - ;; y in fr1; x not in fr1 - ((and (sc-is y double-reg) (= 1 (tn-offset y))) - ;; Load x to fr0 - (sc-case x - (double-reg - (copy-fp-reg-to-fr0 x)) - (double-stack - (inst fstp fr0) - (inst fldd (ea-for-df-stack x))) - (descriptor-reg - (inst fstp fr0) - (inst fldd (ea-for-df-desc x)))) - (inst fxch fr1)) - ;; x in fr0; - ((and (sc-is x double-reg) (zerop (tn-offset x))) - (inst fxch fr1) - ;; Now load y to fr0 - (sc-case y - (double-reg - (copy-fp-reg-to-fr0 y)) - (double-stack - (inst fstp fr0) - (inst fldd (ea-for-df-stack y))) - (descriptor-reg - (inst fstp fr0) - (inst fldd (ea-for-df-desc y))))) - ;; Neither y or x are in either fr0 or fr1 - (t - ;; Load x then y - (inst fstp fr0) - (inst fstp fr0) - (sc-case x - (double-reg - (inst fldd (make-random-tn :kind :normal - :sc (sc-or-lose 'double-reg) - :offset (- (tn-offset x) 2)))) - (double-stack - (inst fldd (ea-for-df-stack x))) - (descriptor-reg - (inst fldd (ea-for-df-desc x)))) - ;; Load y to fr0 - (sc-case y - (double-reg - (inst fldd (make-random-tn :kind :normal - :sc (sc-or-lose 'double-reg) - :offset (1- (tn-offset y))))) - (double-stack - (inst fldd (ea-for-df-stack y))) - (descriptor-reg - (inst fldd (ea-for-df-desc y)))))) - - ;; Now have y at fr0; and x at fr1 - (inst fpatan) - (inst fld fr0) - (case (tn-offset r) - ((0 1)) - (t (inst fstd r))))) -) ; PROGN #!-LONG-FLOAT ;;;; complex float VOPs (define-vop (make-complex-single-float) (:translate complex) - (:args (real :scs (single-reg) :to :result :target r - :load-if (not (location= real r))) - (imag :scs (single-reg) :to :save)) + (:args (real :scs (single-reg fp-single-zero) + :target r + :load-if (not (sc-is real fp-single-zero))) + (imag :scs (single-reg fp-single-zero) + :load-if (not (sc-is imag fp-single-zero)))) (:arg-types single-float single-float) - (:results (r :scs (complex-single-reg) :from (:argument 0) - :load-if (not (sc-is r complex-single-stack)))) + (:results (r :scs (complex-single-reg) :from (:argument 0))) (:result-types complex-single-float) (:note "inline complex single-float creation") (:policy :fast-safe) (:generator 5 - (sc-case r - (complex-single-reg - (let ((r-real (complex-double-reg-real-tn r))) - (unless (location= real r-real) - (cond ((zerop (tn-offset r-real)) - (copy-fp-reg-to-fr0 real)) - ((zerop (tn-offset real)) - (inst fstd r-real)) - (t - (inst fxch real) - (inst fstd r-real) - (inst fxch real))))) - (let ((r-imag (complex-double-reg-imag-tn r))) - (unless (location= imag r-imag) - (cond ((zerop (tn-offset imag)) - (inst fstd r-imag)) - (t - (inst fxch imag) - (inst fstd r-imag) - (inst fxch imag)))))) - (complex-single-stack - (unless (location= real r) - (cond ((zerop (tn-offset real)) - (inst fst (ea-for-csf-real-stack r))) - (t - (inst fxch real) - (inst fst (ea-for-csf-real-stack r)) - (inst fxch real)))) - (inst fxch imag) - (inst fst (ea-for-csf-imag-stack r)) - (inst fxch imag))))) + (cond ((sc-is real fp-single-zero) + (inst xorps r r) + (unless (sc-is imag fp-single-zero) + (inst unpcklps r imag))) + ((location= real imag) + (move r real) + (inst unpcklps r r)) + (t + (move r real) + (unless (sc-is imag fp-single-zero) + (inst unpcklps r imag)))))) (define-vop (make-complex-double-float) (:translate complex) - (:args (real :scs (double-reg) :target r - :load-if (not (location= real r))) - (imag :scs (double-reg) :to :save)) + (:args (real :scs (double-reg fp-double-zero) + :target r + :load-if (not (sc-is real fp-double-zero))) + (imag :scs (double-reg fp-double-zero) + :load-if (not (sc-is imag fp-double-zero)))) (:arg-types double-float double-float) - (:results (r :scs (complex-double-reg) :from (:argument 0) - :load-if (not (sc-is r complex-double-stack)))) + (:results (r :scs (complex-double-reg) :from (:argument 0))) (:result-types complex-double-float) (:note "inline complex double-float creation") (:policy :fast-safe) (:generator 5 - (sc-case r - (complex-double-reg - (let ((r-real (complex-double-reg-real-tn r))) - (unless (location= real r-real) - (cond ((zerop (tn-offset r-real)) - (copy-fp-reg-to-fr0 real)) - ((zerop (tn-offset real)) - (inst fstd r-real)) - (t - (inst fxch real) - (inst fstd r-real) - (inst fxch real))))) - (let ((r-imag (complex-double-reg-imag-tn r))) - (unless (location= imag r-imag) - (cond ((zerop (tn-offset imag)) - (inst fstd r-imag)) - (t - (inst fxch imag) - (inst fstd r-imag) - (inst fxch imag)))))) - (complex-double-stack - (unless (location= real r) - (cond ((zerop (tn-offset real)) - (inst fstd (ea-for-cdf-real-stack r))) - (t - (inst fxch real) - (inst fstd (ea-for-cdf-real-stack r)) - (inst fxch real)))) - (inst fxch imag) - (inst fstd (ea-for-cdf-imag-stack r)) - (inst fxch imag))))) + (cond ((sc-is real fp-double-zero) + (inst xorpd r r) + (unless (sc-is imag fp-double-zero) + (inst unpcklpd r imag))) + ((location= real imag) + (move r real) + (inst unpcklpd r r)) + (t + (move r real) + (unless (sc-is imag fp-double-zero) + (inst unpcklpd r imag)))))) (define-vop (complex-float-value) (:args (x :target r)) + (:temporary (:sc complex-double-reg) zero) (:results (r)) (:variant-vars offset) (:policy :fast-safe) (:generator 3 - (cond ((sc-is x complex-single-reg complex-double-reg) - (let ((value-tn - (make-random-tn :kind :normal - :sc (sc-or-lose 'double-reg) - :offset (+ offset (tn-offset x))))) - (unless (location= value-tn r) - (cond ((zerop (tn-offset r)) - (copy-fp-reg-to-fr0 value-tn)) - ((zerop (tn-offset value-tn)) - (inst fstd r)) - (t - (inst fxch value-tn) - (inst fstd r) - (inst fxch value-tn)))))) - ((sc-is r single-reg) - (let ((ea (sc-case x - (complex-single-stack - (ecase offset - (0 (ea-for-csf-real-stack x)) - (1 (ea-for-csf-imag-stack x)))) - (descriptor-reg - (ecase offset - (0 (ea-for-csf-real-desc x)) - (1 (ea-for-csf-imag-desc x))))))) - (with-empty-tn@fp-top(r) - (inst fld ea)))) - ((sc-is r double-reg) - (let ((ea (sc-case x - (complex-double-stack - (ecase offset - (0 (ea-for-cdf-real-stack x)) - (1 (ea-for-cdf-imag-stack x)))) - (descriptor-reg - (ecase offset - (0 (ea-for-cdf-real-desc x)) - (1 (ea-for-cdf-imag-desc x))))))) - (with-empty-tn@fp-top(r) - (inst fldd ea)))) - (t (error "COMPLEX-FLOAT-VALUE VOP failure"))))) + (cond ((sc-is x complex-double-reg) + (move r x) + (inst xorpd zero zero) + (ecase offset + (0 (inst unpcklpd r zero)) + (1 (inst unpckhpd r zero)))) + ((sc-is x complex-single-reg) + (move r x) + (ecase offset + (0 (inst shufps r r #b11111100)) + (1 (inst shufps r r #b11111101)))) + ((sc-is r single-reg) + (let ((ea (sc-case x + (complex-single-stack + (ecase offset + (0 (ea-for-csf-real-stack x)) + (1 (ea-for-csf-imag-stack x)))) + (descriptor-reg + (ecase offset + (0 (ea-for-csf-real-desc x)) + (1 (ea-for-csf-imag-desc x))))))) + (inst movss r ea))) + ((sc-is r double-reg) + (let ((ea (sc-case x + (complex-double-stack + (ecase offset + (0 (ea-for-cdf-real-stack x)) + (1 (ea-for-cdf-imag-stack x)))) + (descriptor-reg + (ecase offset + (0 (ea-for-cdf-real-desc x)) + (1 (ea-for-cdf-imag-desc x))))))) + (inst movsd r ea))) + (t (error "COMPLEX-FLOAT-VALUE VOP failure"))))) (define-vop (realpart/complex-single-float complex-float-value) (:translate realpart) (:args (x :scs (complex-single-reg complex-single-stack descriptor-reg) - :target r)) + :target r)) (:arg-types complex-single-float) (:results (r :scs (single-reg))) (:result-types single-float) @@ -2808,7 +1337,7 @@ (define-vop (realpart/complex-double-float complex-float-value) (:translate realpart) (:args (x :scs (complex-double-reg complex-double-stack descriptor-reg) - :target r)) + :target r)) (:arg-types complex-double-float) (:results (r :scs (double-reg))) (:result-types double-float) @@ -2818,7 +1347,7 @@ (define-vop (imagpart/complex-single-float complex-float-value) (:translate imagpart) (:args (x :scs (complex-single-reg complex-single-stack descriptor-reg) - :target r)) + :target r)) (:arg-types complex-single-float) (:results (r :scs (single-reg))) (:result-types single-float) @@ -2828,7 +1357,7 @@ (define-vop (imagpart/complex-double-float complex-float-value) (:translate imagpart) (:args (x :scs (complex-double-reg complex-double-stack descriptor-reg) - :target r)) + :target r)) (:arg-types complex-double-float) (:results (r :scs (double-reg))) (:result-types double-float) @@ -2857,3 +1386,30 @@ (:note "inline dummy FP register bias") (:ignore x) (:generator 0)) + +(defknown swap-complex ((complex float)) (complex float) + (foldable flushable movable always-translatable)) +(defoptimizer (swap-complex derive-type) ((x)) + (sb!c::lvar-type x)) +(defun swap-complex (x) + (complex (imagpart x) (realpart x))) +(define-vop (swap-complex-single-float) + (:translate swap-complex) + (:policy :fast-safe) + (:args (x :scs (complex-single-reg) :target r)) + (:arg-types complex-single-float) + (:results (r :scs (complex-single-reg))) + (:result-types complex-single-float) + (:generator 2 + (move r x) + (inst shufps r r #b11110001))) +(define-vop (swap-complex-double-float) + (:translate swap-complex) + (:policy :fast-safe) + (:args (x :scs (complex-double-reg) :target r)) + (:arg-types complex-double-float) + (:results (r :scs (complex-double-reg))) + (:result-types complex-double-float) + (:generator 2 + (move r x) + (inst shufpd r r #b01)))