1 ;;;; floating point support for the x86
3 ;;;; This software is part of the SBCL system. See the README file for
6 ;;;; This software is derived from the CMU CL system, which was
7 ;;;; written at Carnegie Mellon University and released into the
8 ;;;; public domain. The software is in the public domain and is
9 ;;;; provided with absolutely no warranty. See the COPYING and CREDITS
10 ;;;; files for more information.
14 (macrolet ((ea-for-xf-desc (tn slot)
17 :disp (- (* ,slot n-word-bytes)
18 other-pointer-lowtag))))
19 (defun ea-for-df-desc (tn)
20 (ea-for-xf-desc tn double-float-value-slot))
22 (defun ea-for-csf-data-desc (tn)
23 (ea-for-xf-desc tn complex-single-float-data-slot))
24 (defun ea-for-csf-real-desc (tn)
25 (ea-for-xf-desc tn complex-single-float-data-slot))
26 (defun ea-for-csf-imag-desc (tn)
27 (ea-for-xf-desc tn (+ complex-single-float-data-slot 1/2)))
29 (defun ea-for-cdf-data-desc (tn)
30 (ea-for-xf-desc tn complex-double-float-real-slot))
31 (defun ea-for-cdf-real-desc (tn)
32 (ea-for-xf-desc tn complex-double-float-real-slot))
33 (defun ea-for-cdf-imag-desc (tn)
34 (ea-for-xf-desc tn complex-double-float-imag-slot)))
36 (macrolet ((ea-for-xf-stack (tn kind)
37 (declare (ignore kind))
40 :disp (frame-byte-offset (tn-offset ,tn)))))
41 (defun ea-for-sf-stack (tn)
42 (ea-for-xf-stack tn :single))
43 (defun ea-for-df-stack (tn)
44 (ea-for-xf-stack tn :double)))
46 ;;; complex float stack EAs
47 (macrolet ((ea-for-cxf-stack (tn kind slot &optional base)
50 :disp (frame-byte-offset
52 (cond ((= (tn-offset ,base) rsp-offset)
54 ((= (tn-offset ,base) rbp-offset)
56 (t (error "Unexpected offset.")))
66 (defun ea-for-csf-data-stack (tn &optional (base rbp-tn))
67 (ea-for-cxf-stack tn :single :real base))
68 (defun ea-for-csf-real-stack (tn &optional (base rbp-tn))
69 (ea-for-cxf-stack tn :single :real base))
70 (defun ea-for-csf-imag-stack (tn &optional (base rbp-tn))
71 (ea-for-cxf-stack tn :single :imag base))
73 (defun ea-for-cdf-data-stack (tn &optional (base rbp-tn))
74 (ea-for-cxf-stack tn :double :real base))
75 (defun ea-for-cdf-real-stack (tn &optional (base rbp-tn))
76 (ea-for-cxf-stack tn :double :real base))
77 (defun ea-for-cdf-imag-stack (tn &optional (base rbp-tn))
78 (ea-for-cxf-stack tn :double :imag base)))
82 ;;; X is source, Y is destination.
84 (define-move-fun (load-fp-zero 1) (vop x y)
85 ((fp-single-zero) (single-reg)
86 (fp-double-zero) (double-reg)
87 (fp-complex-single-zero) (complex-single-reg)
88 (fp-complex-double-zero) (complex-double-reg))
91 ((single-reg complex-single-reg) (inst xorps y y))
92 ((double-reg complex-double-reg) (inst xorpd y y))))
94 (define-move-fun (load-fp-immediate 1) (vop x y)
95 ((fp-single-immediate) (single-reg)
96 (fp-double-immediate) (double-reg)
97 (fp-complex-single-immediate) (complex-single-reg)
98 (fp-complex-double-immediate) (complex-double-reg))
99 (let ((x (register-inline-constant (tn-value x))))
101 (single-reg (inst movss y x))
102 (double-reg (inst movsd y x))
103 (complex-single-reg (inst movq y x))
104 (complex-double-reg (inst movapd y x)))))
106 (define-move-fun (load-single 2) (vop x y)
107 ((single-stack) (single-reg))
108 (inst movss y (ea-for-sf-stack x)))
110 (define-move-fun (store-single 2) (vop x y)
111 ((single-reg) (single-stack))
112 (inst movss (ea-for-sf-stack y) x))
114 (define-move-fun (load-double 2) (vop x y)
115 ((double-stack) (double-reg))
116 (inst movsd y (ea-for-df-stack x)))
118 (define-move-fun (store-double 2) (vop x y)
119 ((double-reg) (double-stack))
120 (inst movsd (ea-for-df-stack y) x))
122 (eval-when (:compile-toplevel :execute)
123 (setf *read-default-float-format* 'single-float))
125 ;;;; complex float move functions
127 ;;; X is source, Y is destination.
128 (define-move-fun (load-complex-single 2) (vop x y)
129 ((complex-single-stack) (complex-single-reg))
130 (inst movq y (ea-for-csf-data-stack x)))
132 (define-move-fun (store-complex-single 2) (vop x y)
133 ((complex-single-reg) (complex-single-stack))
134 (inst movq (ea-for-csf-data-stack y) x))
136 (define-move-fun (load-complex-double 2) (vop x y)
137 ((complex-double-stack) (complex-double-reg))
138 (inst movupd y (ea-for-cdf-data-stack x)))
140 (define-move-fun (store-complex-double 2) (vop x y)
141 ((complex-double-reg) (complex-double-stack))
142 (inst movupd (ea-for-cdf-data-stack y) x))
146 ;;; float register to register moves
147 (macrolet ((frob (vop sc)
152 :load-if (not (location= x y))))
153 (:results (y :scs (,sc)
154 :load-if (not (location= x y))))
158 (define-move-vop ,vop :move (,sc) (,sc)))))
159 (frob single-move single-reg)
160 (frob double-move double-reg)
161 (frob complex-single-move complex-single-reg)
162 (frob complex-double-move complex-double-reg))
165 ;;; Move from float to a descriptor reg. allocating a new float
166 ;;; object in the process.
167 (define-vop (move-from-single)
168 (:args (x :scs (single-reg) :to :save))
169 (:results (y :scs (descriptor-reg)))
170 (:note "float to pointer coercion")
174 (inst or y single-float-widetag)))
176 (define-move-vop move-from-single :move
177 (single-reg) (descriptor-reg))
179 (define-vop (move-from-double)
180 (:args (x :scs (double-reg) :to :save))
181 (:results (y :scs (descriptor-reg)))
183 (:note "float to pointer coercion")
185 (with-fixed-allocation (y
189 (inst movsd (ea-for-df-desc y) x))))
190 (define-move-vop move-from-double :move
191 (double-reg) (descriptor-reg))
193 ;;; Move from a descriptor to a float register.
194 (define-vop (move-to-single)
195 (:args (x :scs (descriptor-reg) :target tmp))
196 (:temporary (:sc unsigned-reg) tmp)
197 (:results (y :scs (single-reg)))
198 (:note "pointer to float coercion")
204 (define-move-vop move-to-single :move (descriptor-reg) (single-reg))
206 (define-vop (move-to-double)
207 (:args (x :scs (descriptor-reg)))
208 (:results (y :scs (double-reg)))
209 (:note "pointer to float coercion")
211 (inst movsd y (ea-for-df-desc x))))
212 (define-move-vop move-to-double :move (descriptor-reg) (double-reg))
215 ;;; Move from complex float to a descriptor reg. allocating a new
216 ;;; complex float object in the process.
217 (define-vop (move-from-complex-single)
218 (:args (x :scs (complex-single-reg) :to :save))
219 (:results (y :scs (descriptor-reg)))
221 (:note "complex float to pointer coercion")
223 (with-fixed-allocation (y
224 complex-single-float-widetag
225 complex-single-float-size
227 (inst movq (ea-for-csf-data-desc y) x))))
228 (define-move-vop move-from-complex-single :move
229 (complex-single-reg) (descriptor-reg))
231 (define-vop (move-from-complex-double)
232 (:args (x :scs (complex-double-reg) :to :save))
233 (:results (y :scs (descriptor-reg)))
235 (:note "complex float to pointer coercion")
237 (with-fixed-allocation (y
238 complex-double-float-widetag
239 complex-double-float-size
241 (inst movapd (ea-for-cdf-data-desc y) x))))
242 (define-move-vop move-from-complex-double :move
243 (complex-double-reg) (descriptor-reg))
245 ;;; Move from a descriptor to a complex float register.
246 (macrolet ((frob (name sc format)
249 (:args (x :scs (descriptor-reg)))
250 (:results (y :scs (,sc)))
251 (:note "pointer to complex float coercion")
255 '(inst movq y (ea-for-csf-data-desc x)))
257 '(inst movapd y (ea-for-cdf-data-desc x))))))
258 (define-move-vop ,name :move (descriptor-reg) (,sc)))))
259 (frob move-to-complex-single complex-single-reg :single)
260 (frob move-to-complex-double complex-double-reg :double))
262 ;;;; the move argument vops
264 ;;;; Note these are also used to stuff fp numbers onto the c-call
265 ;;;; stack so the order is different than the lisp-stack.
267 ;;; the general MOVE-ARG VOP
268 (macrolet ((frob (name sc stack-sc format)
271 (:args (x :scs (,sc) :target y)
273 :load-if (not (sc-is y ,sc))))
275 (:note "float argument move")
276 (:generator ,(case format (:single 2) (:double 3) )
281 (if (= (tn-offset fp) esp-offset)
282 (let* ((offset (* (tn-offset y) n-word-bytes))
283 (ea (make-ea :dword :base fp :disp offset)))
285 (:single '((inst movss ea x)))
286 (:double '((inst movsd ea x)))))
289 :disp (frame-byte-offset (tn-offset y)))))
291 (:single '((inst movss ea x)))
292 (:double '((inst movsd ea x))))))))))
293 (define-move-vop ,name :move-arg
294 (,sc descriptor-reg) (,sc)))))
295 (frob move-single-float-arg single-reg single-stack :single)
296 (frob move-double-float-arg double-reg double-stack :double))
298 ;;;; complex float MOVE-ARG VOP
299 (macrolet ((frob (name sc stack-sc format)
302 (:args (x :scs (,sc) :target y)
304 :load-if (not (sc-is y ,sc))))
306 (:note "complex float argument move")
307 (:generator ,(ecase format (:single 2) (:double 3))
314 '(inst movq (ea-for-csf-data-stack y fp) x))
316 '(inst movupd (ea-for-cdf-data-stack y fp) x)))))))
317 (define-move-vop ,name :move-arg
318 (,sc descriptor-reg) (,sc)))))
319 (frob move-complex-single-float-arg
320 complex-single-reg complex-single-stack :single)
321 (frob move-complex-double-float-arg
322 complex-double-reg complex-double-stack :double))
324 (define-move-vop move-arg :move-arg
325 (single-reg double-reg
326 complex-single-reg complex-double-reg)
332 (define-vop (float-op)
336 (:note "inline float arithmetic")
338 (:save-p :compute-only))
340 (macrolet ((frob (name comm-name sc constant-sc ptype)
342 (define-vop (,name float-op)
343 (:args (x :scs (,sc ,constant-sc)
345 :load-if (not (sc-is x ,constant-sc)))
346 (y :scs (,sc ,constant-sc)
347 :load-if (not (sc-is y ,constant-sc))))
348 (:results (r :scs (,sc)))
349 (:arg-types ,ptype ,ptype)
350 (:result-types ,ptype))
351 (define-vop (,comm-name float-op)
352 (:args (x :scs (,sc ,constant-sc)
354 :load-if (not (sc-is x ,constant-sc)))
355 (y :scs (,sc ,constant-sc)
357 :load-if (not (sc-is y ,constant-sc))))
358 (:results (r :scs (,sc)))
359 (:arg-types ,ptype ,ptype)
360 (:result-types ,ptype)))))
361 (frob single-float-op single-float-comm-op
362 single-reg fp-single-immediate single-float)
363 (frob double-float-op double-float-comm-op
364 double-reg fp-double-immediate double-float)
365 (frob complex-single-float-op complex-single-float-comm-op
366 complex-single-reg fp-complex-single-immediate
367 complex-single-float)
368 (frob complex-double-float-op complex-double-float-comm-op
369 complex-double-reg fp-complex-double-immediate
370 complex-double-float))
372 (macrolet ((generate (opinst commutative constant-sc load-inst)
373 `(flet ((get-constant (tn)
374 (register-inline-constant
375 ,@(and (eq constant-sc 'fp-single-immediate)
378 (declare (ignorable #'get-constant))
381 (when (sc-is y ,constant-sc)
382 (setf y (get-constant y)))
384 ((and ,commutative (location= y r))
385 (when (sc-is x ,constant-sc)
386 (setf x (get-constant x)))
388 ((not (location= r y))
389 (if (sc-is x ,constant-sc)
390 (inst ,load-inst r (get-constant x))
392 (when (sc-is y ,constant-sc)
393 (setf y (get-constant y)))
396 (if (sc-is x ,constant-sc)
397 (inst ,load-inst tmp (get-constant x))
401 (frob (op sinst sname scost dinst dname dcost commutative
402 &optional csinst csname cscost cdinst cdname cdcost)
404 (define-vop (,sname ,(if commutative
405 'single-float-comm-op
408 (:temporary (:sc single-reg) tmp)
410 (generate ,sinst ,commutative fp-single-immediate movss)))
411 (define-vop (,dname ,(if commutative
412 'double-float-comm-op
415 (:temporary (:sc double-reg) tmp)
417 (generate ,dinst ,commutative fp-double-immediate movsd)))
419 `(define-vop (,csname
421 'complex-single-float-comm-op
422 'complex-single-float-op))
424 (:temporary (:sc complex-single-reg) tmp)
426 (generate ,csinst ,commutative
427 fp-complex-single-immediate movq))))
429 `(define-vop (,cdname
431 'complex-double-float-comm-op
432 'complex-double-float-op))
434 (:temporary (:sc complex-double-reg) tmp)
436 (generate ,cdinst ,commutative
437 fp-complex-double-immediate movapd)))))))
438 (frob + addss +/single-float 2 addsd +/double-float 2 t
439 addps +/complex-single-float 3 addpd +/complex-double-float 3)
440 (frob - subss -/single-float 2 subsd -/double-float 2 nil
441 subps -/complex-single-float 3 subpd -/complex-double-float 3)
442 (frob * mulss */single-float 4 mulsd */double-float 5 t)
443 (frob / divss //single-float 12 divsd //double-float 19 nil))
445 (macrolet ((frob (op cost commutativep
446 duplicate-inst op-inst real-move-inst complex-move-inst
447 real-sc real-constant-sc real-type
448 complex-sc complex-constant-sc complex-type
449 real-complex-name complex-real-name)
450 (cond ((not duplicate-inst) ; simple case
451 `(flet ((load-into (r x)
454 (inst ,real-move-inst r
455 (register-inline-constant (tn-value x))))
456 (,complex-constant-sc
457 (inst ,complex-move-inst r
458 (register-inline-constant (tn-value x))))
460 ,(when real-complex-name
461 `(define-vop (,real-complex-name float-op)
463 (:args (x :scs (,real-sc ,real-constant-sc)
465 :load-if (not (sc-is x ,real-constant-sc)))
466 (y :scs (,complex-sc ,complex-constant-sc)
467 ,@(when commutativep '(:target r))
468 :load-if (not (sc-is y ,complex-constant-sc))))
469 (:arg-types ,real-type ,complex-type)
470 (:results (r :scs (,complex-sc)
471 ,@(unless commutativep '(:from (:argument 0)))))
472 (:result-types ,complex-type)
475 `(when (location= y r)
478 (when (sc-is y ,real-constant-sc ,complex-constant-sc)
479 (setf y (register-inline-constant
480 :aligned (tn-value y))))
481 (inst ,op-inst r y))))
483 ,(when complex-real-name
484 `(define-vop (,complex-real-name float-op)
486 (:args (x :scs (,complex-sc ,complex-constant-sc)
488 :load-if (not (sc-is x ,complex-constant-sc)))
489 (y :scs (,real-sc ,real-constant-sc)
490 ,@(when commutativep '(:target r))
491 :load-if (not (sc-is y ,real-constant-sc))))
492 (:arg-types ,complex-type ,real-type)
493 (:results (r :scs (,complex-sc)
494 ,@(unless commutativep '(:from (:argument 0)))))
495 (:result-types ,complex-type)
498 `(when (location= y r)
501 (when (sc-is y ,real-constant-sc ,complex-constant-sc)
502 (setf y (register-inline-constant
503 :aligned (tn-value y))))
504 (inst ,op-inst r y))))))
505 (commutativep ; must duplicate, but commutative
507 ,(when real-complex-name
508 `(define-vop (,real-complex-name float-op)
510 (:args (x :scs (,real-sc ,real-constant-sc)
512 :load-if (not (sc-is x ,real-constant-sc)))
513 (y :scs (,complex-sc ,complex-constant-sc)
516 :load-if (not (sc-is y ,complex-constant-sc))))
517 (:arg-types ,real-type ,complex-type)
518 (:temporary (:sc ,complex-sc :target r
522 (:results (r :scs (,complex-sc)))
523 (:result-types ,complex-type)
525 (if (sc-is x ,real-constant-sc)
526 (inst ,complex-move-inst dup
527 (register-inline-constant
528 (complex (tn-value x) (tn-value x))))
532 (when (location= dup r)
534 (if (sc-is y ,complex-constant-sc)
535 (inst ,complex-move-inst r
536 (register-inline-constant (tn-value y)))
538 (when (sc-is dup ,complex-constant-sc)
539 (setf dup (register-inline-constant
540 :aligned (tn-value dup))))
541 (inst ,op-inst r dup))))
543 ,(when complex-real-name
544 `(define-vop (,complex-real-name float-op)
546 (:args (x :scs (,complex-sc ,complex-constant-sc)
549 :load-if (not (sc-is x ,complex-constant-sc)))
550 (y :scs (,real-sc ,real-constant-sc)
552 :load-if (not (sc-is y ,real-constant-sc))))
553 (:arg-types ,complex-type ,real-type)
554 (:temporary (:sc ,complex-sc :target r
558 (:results (r :scs (,complex-sc)))
559 (:result-types ,complex-type)
561 (if (sc-is y ,real-constant-sc)
562 (inst ,complex-move-inst dup
563 (register-inline-constant
564 (complex (tn-value y) (tn-value y))))
567 (when (location= dup r)
569 (if (sc-is x ,complex-constant-sc)
570 (inst ,complex-move-inst r
571 (register-inline-constant (tn-value x)))
573 (when (sc-is dup ,complex-constant-sc)
574 (setf dup (register-inline-constant
575 :aligned (tn-value dup))))
576 (inst ,op-inst r dup))))))
577 (t ; duplicate, not commutative
579 ,(when real-complex-name
580 `(define-vop (,real-complex-name float-op)
582 (:args (x :scs (,real-sc ,real-constant-sc)
584 :load-if (not (sc-is x ,real-constant-sc)))
585 (y :scs (,complex-sc ,complex-constant-sc)
587 :load-if (not (sc-is y ,complex-constant-sc))))
588 (:arg-types ,real-type ,complex-type)
589 (:results (r :scs (,complex-sc) :from (:argument 0)))
590 (:result-types ,complex-type)
592 (if (sc-is x ,real-constant-sc)
593 (inst ,complex-move-inst dup
594 (register-inline-constant
595 (complex (tn-value x) (tn-value x))))
599 (when (sc-is y ,complex-constant-sc)
600 (setf y (register-inline-constant
601 :aligned (tn-value y))))
602 (inst ,op-inst r y))))
604 ,(when complex-real-name
605 `(define-vop (,complex-real-name float-op)
607 (:args (x :scs (,complex-sc)
610 (y :scs (,real-sc ,real-constant-sc)
612 :load-if (not (sc-is y ,complex-constant-sc))))
613 (:arg-types ,complex-type ,real-type)
614 (:temporary (:sc ,complex-sc :from (:argument 1))
616 (:results (r :scs (,complex-sc) :from :eval))
617 (:result-types ,complex-type)
619 (if (sc-is y ,real-constant-sc)
620 (setf dup (register-inline-constant
621 :aligned (complex (tn-value y)
626 (inst ,op-inst r dup))))))))
627 (def-real-complex-op (op commutativep duplicatep
628 single-inst single-real-complex-name single-complex-real-name single-cost
629 double-inst double-real-complex-name double-complex-real-name double-cost)
631 (frob ,op ,single-cost ,commutativep
635 (inst unpcklps dup dup)))
636 ,single-inst movss movq
637 single-reg fp-single-immediate single-float
638 complex-single-reg fp-complex-single-immediate complex-single-float
639 ,single-real-complex-name ,single-complex-real-name)
640 (frob ,op ,double-cost ,commutativep
644 (inst unpcklpd dup dup)))
645 ,double-inst movsd movapd
646 double-reg fp-double-immediate double-float
647 complex-double-reg fp-complex-double-immediate complex-double-float
648 ,double-real-complex-name ,double-complex-real-name))))
649 (def-real-complex-op + t nil
650 addps +/real-complex-single-float +/complex-real-single-float 3
651 addpd +/real-complex-double-float +/complex-real-double-float 4)
652 (def-real-complex-op - nil nil
653 subps -/real-complex-single-float -/complex-real-single-float 3
654 subpd -/real-complex-double-float -/complex-real-double-float 4)
655 (def-real-complex-op * t t
656 mulps */real-complex-single-float */complex-real-single-float 4
657 mulpd */real-complex-double-float */complex-real-double-float 5)
658 (def-real-complex-op / nil t
660 divpd nil //complex-real-double-float 19))
662 (define-vop (//complex-real-single-float float-op)
664 (:args (x :scs (complex-single-reg fp-complex-single-immediate fp-complex-single-zero)
667 :load-if (not (sc-is x fp-complex-single-immediate fp-complex-single-zero)))
668 (y :scs (single-reg fp-single-immediate fp-single-zero)
670 :load-if (not (sc-is y fp-single-immediate fp-single-zero))))
671 (:arg-types complex-single-float single-float)
672 (:temporary (:sc complex-single-reg :from (:argument 1)) dup)
673 (:results (r :scs (complex-single-reg)))
674 (:result-types complex-single-float)
676 (flet ((duplicate (x)
677 (let ((word (ldb (byte 64 0)
678 (logior (ash (single-float-bits (imagpart x)) 32)
680 (single-float-bits (realpart x)))))))
681 (register-inline-constant :oword (logior (ash word 64) word)))))
684 (setf dup (duplicate (complex (tn-value y) (tn-value y)))))
686 (inst xorps dup dup))
688 (inst shufps dup dup #b00000000)))
690 (fp-complex-single-immediate
691 (inst movaps r (duplicate (tn-value x))))
692 (fp-complex-single-zero
696 (inst unpcklpd r r)))
700 ;; Complex multiplication
701 ;; r := rx * ry - ix * iy
702 ;; i := rx * iy + ix * ry
704 ;; Transpose for SIMDness
709 ;;+ [ix ix] * [-iy ry]
712 (macrolet ((define-complex-* (name cost type sc tmp-p &body body)
713 `(define-vop (,name float-op)
715 (:args (x :scs (,sc) :target r)
716 (y :scs (,sc) :target copy-y))
717 (:arg-types ,type ,type)
718 (:temporary (:sc ,sc) imag)
719 (:temporary (:sc ,sc :from :eval) copy-y)
721 `((:temporary (:sc ,sc) xmm)))
722 (:results (r :scs (,sc) :from :eval))
723 (:result-types ,type)
725 (when (or (location= x copy-y)
729 (define-complex-* */complex-single-float 20
730 complex-single-float complex-single-reg t
735 (inst unpckhpd imag xmm)
736 (inst unpcklpd r xmm)
737 (move copy-y y) ; y == r only if y == x == r
742 (inst shufps y y #b11110001)
743 (inst xorps y (register-inline-constant :oword (ash 1 31)))
747 (define-complex-* */complex-double-float 25
748 complex-double-float complex-double-reg nil
754 (inst unpckhpd imag imag)
758 (inst shufpd y y #b01)
759 (inst xorpd y (register-inline-constant :oword (ash 1 63)))
762 (inst addpd r imag)))
765 (:args (x :scs (double-reg)))
766 (:results (y :scs (double-reg)))
769 (:arg-types double-float)
770 (:result-types double-float)
771 (:note "inline float arithmetic")
773 (:save-p :compute-only)
775 (note-this-location vop :internal-error)
778 (macrolet ((frob ((name translate sc type) &body body)
780 (:args (x :scs (,sc) :target y))
781 (:results (y :scs (,sc)))
782 (:translate ,translate)
785 (:result-types ,type)
786 (:note "inline float arithmetic")
788 (:save-p :compute-only)
790 (note-this-location vop :internal-error)
791 ;; we should be able to do this better. what we
792 ;; really would like to do is use the target as the
793 ;; temp whenever it's not also the source
796 (frob (%negate/double-float %negate double-reg double-float)
797 (inst xorpd y (register-inline-constant :oword (ash 1 63))))
798 (frob (%negate/complex-double-float %negate complex-double-reg complex-double-float)
799 (inst xorpd y (register-inline-constant
800 :oword (logior (ash 1 127) (ash 1 63)))))
801 (frob (conjugate/complex-double-float conjugate complex-double-reg complex-double-float)
802 (inst xorpd y (register-inline-constant :oword (ash 1 127))))
803 (frob (%negate/single-float %negate single-reg single-float)
804 (inst xorps y (register-inline-constant :oword (ash 1 31))))
805 (frob (%negate/complex-single-float %negate complex-single-reg complex-single-float)
806 (inst xorps y (register-inline-constant
807 :oword (logior (ash 1 31) (ash 1 63)))))
808 (frob (conjugate/complex-single-float conjugate complex-single-reg complex-single-float)
809 (inst xorpd y (register-inline-constant :oword (ash 1 63))))
810 (frob (abs/double-float abs double-reg double-float)
811 (inst andpd y (register-inline-constant :oword (ldb (byte 63 0) -1))))
812 (frob (abs/single-float abs single-reg single-float)
813 (inst andps y (register-inline-constant :oword (ldb (byte 31 0) -1)))))
818 (define-vop (float-compare)
821 (:save-p :compute-only)
822 (:note "inline float comparison"))
825 (macrolet ((define-float-eql (name cost sc constant-sc type)
826 `(define-vop (,name float-compare)
828 (:args (x :scs (,sc ,constant-sc)
830 :load-if (not (sc-is x ,constant-sc)))
831 (y :scs (,sc ,constant-sc)
833 :load-if (not (sc-is y ,constant-sc))))
834 (:arg-types ,type ,type)
835 (:temporary (:sc ,sc :from :eval) mask)
836 (:temporary (:sc any-reg) bits)
839 (when (or (location= y mask)
840 (not (xmm-register-p x)))
842 (aver (xmm-register-p x))
844 (when (sc-is y ,constant-sc)
845 (setf y (register-inline-constant :aligned (tn-value y))))
846 (inst pcmpeqd mask y)
847 (inst movmskps bits mask)
848 (inst cmp bits #b1111)))))
849 (define-float-eql eql/single-float 4
850 single-reg fp-single-immediate single-float)
851 (define-float-eql eql/double-float 4
852 double-reg fp-double-immediate double-float)
853 (define-float-eql eql/complex-single-float 5
854 complex-single-reg fp-complex-single-immediate complex-single-float)
855 (define-float-eql eql/complex-double-float 5
856 complex-double-reg fp-complex-double-immediate complex-double-float))
858 ;;; comiss and comisd can cope with one or other arg in memory: we
859 ;;; could (should, indeed) extend these to cope with descriptor args
862 (define-vop (single-float-compare float-compare)
863 (:args (x :scs (single-reg))
864 (y :scs (single-reg single-stack fp-single-immediate)
865 :load-if (not (sc-is y single-stack fp-single-immediate))))
866 (:arg-types single-float single-float))
867 (define-vop (double-float-compare float-compare)
868 (:args (x :scs (double-reg))
869 (y :scs (double-reg double-stack descriptor-reg fp-double-immediate)
870 :load-if (not (sc-is y double-stack descriptor-reg fp-double-immediate))))
871 (:arg-types double-float double-float))
873 (define-vop (=/single-float single-float-compare)
875 (:args (x :scs (single-reg single-stack fp-single-immediate)
877 :load-if (not (sc-is x single-stack fp-single-immediate)))
878 (y :scs (single-reg single-stack fp-single-immediate)
880 :load-if (not (sc-is y single-stack fp-single-immediate))))
881 (:temporary (:sc single-reg :from :eval) xmm)
883 (:conditional not :p :ne)
886 (when (or (location= y xmm)
887 (and (not (xmm-register-p x))
891 (single-reg (setf xmm x))
892 (single-stack (inst movss xmm (ea-for-sf-stack x)))
894 (inst movss xmm (register-inline-constant (tn-value x)))))
897 (setf y (ea-for-sf-stack y)))
899 (setf y (register-inline-constant (tn-value y))))
901 (note-this-location vop :internal-error)
903 ;; if PF&CF, there was a NaN involved => not equal
904 ;; otherwise, ZF => equal
907 (define-vop (=/double-float double-float-compare)
909 (:args (x :scs (double-reg double-stack fp-double-immediate descriptor-reg)
911 :load-if (not (sc-is x double-stack fp-double-immediate descriptor-reg)))
912 (y :scs (double-reg double-stack fp-double-immediate descriptor-reg)
914 :load-if (not (sc-is y double-stack fp-double-immediate descriptor-reg))))
915 (:temporary (:sc double-reg :from :eval) xmm)
917 (:conditional not :p :ne)
920 (when (or (location= y xmm)
921 (and (not (xmm-register-p x))
928 (inst movsd xmm (ea-for-df-stack x)))
930 (inst movsd xmm (register-inline-constant (tn-value x))))
932 (inst movsd xmm (ea-for-df-desc x))))
935 (setf y (ea-for-df-stack y)))
937 (setf y (register-inline-constant (tn-value y))))
939 (setf y (ea-for-df-desc y)))
941 (note-this-location vop :internal-error)
942 (inst comisd xmm y)))
944 (macrolet ((define-complex-float-= (complex-complex-name complex-real-name real-complex-name
945 real-sc real-constant-sc real-type
946 complex-sc complex-constant-sc complex-type
947 real-move-inst complex-move-inst
948 cmp-inst mask-inst mask)
950 (define-vop (,complex-complex-name float-compare)
952 (:args (x :scs (,complex-sc ,complex-constant-sc)
954 :load-if (not (sc-is x ,complex-constant-sc)))
955 (y :scs (,complex-sc ,complex-constant-sc)
957 :load-if (not (sc-is y ,complex-constant-sc))))
958 (:arg-types ,complex-type ,complex-type)
959 (:temporary (:sc ,complex-sc :from :eval) cmp)
960 (:temporary (:sc unsigned-reg) bits)
964 (when (location= y cmp)
968 (inst ,real-move-inst cmp (register-inline-constant
970 (,complex-constant-sc
971 (inst ,complex-move-inst cmp (register-inline-constant
975 (when (sc-is y ,real-constant-sc ,complex-constant-sc)
976 (setf y (register-inline-constant :aligned (tn-value y))))
977 (note-this-location vop :internal-error)
978 (inst ,cmp-inst :eq cmp y)
979 (inst ,mask-inst bits cmp)
980 (inst cmp bits ,mask)))
981 (define-vop (,complex-real-name ,complex-complex-name)
982 (:args (x :scs (,complex-sc ,complex-constant-sc)
984 :load-if (not (sc-is x ,complex-constant-sc)))
985 (y :scs (,real-sc ,real-constant-sc)
987 :load-if (not (sc-is y ,real-constant-sc))))
988 (:arg-types ,complex-type ,real-type))
989 (define-vop (,real-complex-name ,complex-complex-name)
990 (:args (x :scs (,real-sc ,real-constant-sc)
992 :load-if (not (sc-is x ,real-constant-sc)))
993 (y :scs (,complex-sc ,complex-constant-sc)
995 :load-if (not (sc-is y ,complex-constant-sc))))
996 (:arg-types ,real-type ,complex-type)))))
997 (define-complex-float-= =/complex-single-float =/complex-real-single-float =/real-complex-single-float
998 single-reg fp-single-immediate single-float
999 complex-single-reg fp-complex-single-immediate complex-single-float
1000 movss movq cmpps movmskps #b1111)
1001 (define-complex-float-= =/complex-double-float =/complex-real-double-float =/real-complex-double-float
1002 double-reg fp-double-immediate double-float
1003 complex-double-reg fp-complex-double-immediate complex-double-float
1004 movsd movapd cmppd movmskpd #b11))
1006 (macrolet ((define-</> (op single-name double-name &rest flags)
1008 (define-vop (,double-name double-float-compare)
1011 (:conditional ,@flags)
1015 (setf y (ea-for-df-stack y)))
1017 (setf y (ea-for-df-desc y)))
1018 (fp-double-immediate
1019 (setf y (register-inline-constant (tn-value y))))
1022 (define-vop (,single-name single-float-compare)
1025 (:conditional ,@flags)
1029 (setf y (ea-for-sf-stack y)))
1030 (fp-single-immediate
1031 (setf y (register-inline-constant (tn-value y))))
1033 (inst comiss x y))))))
1034 (define-</> < <single-float <double-float not :p :nc)
1035 (define-</> > >single-float >double-float not :p :na))
1040 (macrolet ((frob (name translate inst to-sc to-type)
1041 `(define-vop (,name)
1042 (:args (x :scs (signed-stack signed-reg) :target temp))
1043 (:temporary (:sc signed-stack) temp)
1044 (:results (y :scs (,to-sc)))
1045 (:arg-types signed-num)
1046 (:result-types ,to-type)
1047 (:policy :fast-safe)
1048 (:note "inline float coercion")
1049 (:translate ,translate)
1051 (:save-p :compute-only)
1056 (note-this-location vop :internal-error)
1057 (inst ,inst y temp))
1059 (note-this-location vop :internal-error)
1060 (inst ,inst y x)))))))
1061 (frob %single-float/signed %single-float cvtsi2ss single-reg single-float)
1062 (frob %double-float/signed %double-float cvtsi2sd double-reg double-float))
1064 (macrolet ((frob (name translate inst from-sc from-type to-sc to-type)
1065 `(define-vop (,name)
1066 (:args (x :scs (,from-sc) :target y))
1067 (:results (y :scs (,to-sc)))
1068 (:arg-types ,from-type)
1069 (:result-types ,to-type)
1070 (:policy :fast-safe)
1071 (:note "inline float coercion")
1072 (:translate ,translate)
1074 (:save-p :compute-only)
1076 (note-this-location vop :internal-error)
1077 (inst ,inst y x)))))
1078 (frob %single-float/double-float %single-float cvtsd2ss double-reg
1079 double-float single-reg single-float)
1081 (frob %double-float/single-float %double-float cvtss2sd
1082 single-reg single-float double-reg double-float))
1084 (macrolet ((frob (trans inst from-sc from-type round-p)
1085 (declare (ignore round-p))
1086 `(define-vop (,(symbolicate trans "/" from-type))
1087 (:args (x :scs (,from-sc)))
1088 (:temporary (:sc any-reg) temp-reg)
1089 (:results (y :scs (signed-reg)))
1090 (:arg-types ,from-type)
1091 (:result-types signed-num)
1093 (:policy :fast-safe)
1094 (:note "inline float truncate")
1096 (:save-p :compute-only)
1100 (inst ,inst temp-reg x)
1105 (frob %unary-truncate/single-float cvttss2si single-reg single-float nil)
1106 (frob %unary-truncate/double-float cvttsd2si double-reg double-float nil)
1108 (frob %unary-round cvtss2si single-reg single-float t)
1109 (frob %unary-round cvtsd2si double-reg double-float t))
1111 (define-vop (make-single-float)
1112 (:args (bits :scs (signed-reg) :target res
1113 :load-if (not (or (and (sc-is bits signed-stack)
1114 (sc-is res single-reg))
1115 (and (sc-is bits signed-stack)
1116 (sc-is res single-stack)
1117 (location= bits res))))))
1118 (:results (res :scs (single-reg single-stack)))
1119 (:arg-types signed-num)
1120 (:result-types single-float)
1121 (:translate make-single-float)
1122 (:policy :fast-safe)
1129 (inst mov res bits))
1131 (aver (location= bits res)))))
1135 (inst movd res bits))
1137 (inst movd res bits)))))))
1139 (define-vop (make-double-float)
1140 (:args (hi-bits :scs (signed-reg))
1141 (lo-bits :scs (unsigned-reg)))
1142 (:results (res :scs (double-reg)))
1143 (:temporary (:sc unsigned-reg) temp)
1144 (:arg-types signed-num unsigned-num)
1145 (:result-types double-float)
1146 (:translate make-double-float)
1147 (:policy :fast-safe)
1152 (inst or temp lo-bits)
1153 (inst movd res temp)))
1155 (define-vop (single-float-bits)
1156 (:args (float :scs (single-reg descriptor-reg)
1157 :load-if (not (sc-is float single-stack))))
1158 (:results (bits :scs (signed-reg)))
1159 (:temporary (:sc signed-stack :from :argument :to :result) stack-temp)
1160 (:arg-types single-float)
1161 (:result-types signed-num)
1162 (:translate single-float-bits)
1163 (:policy :fast-safe)
1170 (inst movss stack-temp float)
1171 (move bits stack-temp))
1176 (inst shr bits 32))))
1180 (inst movss bits float)))))
1183 (inst sar bits 32)))
1185 (define-vop (double-float-high-bits)
1186 (:args (float :scs (double-reg descriptor-reg)
1187 :load-if (not (sc-is float double-stack))))
1188 (:results (hi-bits :scs (signed-reg)))
1189 (:temporary (:sc signed-stack :from :argument :to :result) temp)
1190 (:arg-types double-float)
1191 (:result-types signed-num)
1192 (:translate double-float-high-bits)
1193 (:policy :fast-safe)
1198 (inst movsd temp float)
1199 (move hi-bits temp))
1201 (loadw hi-bits ebp-tn (frame-word-offset (tn-offset float))))
1203 (loadw hi-bits float double-float-value-slot
1204 other-pointer-lowtag)))
1205 (inst sar hi-bits 32)))
1207 (define-vop (double-float-low-bits)
1208 (:args (float :scs (double-reg descriptor-reg)
1209 :load-if (not (sc-is float double-stack))))
1210 (:results (lo-bits :scs (unsigned-reg)))
1211 (:temporary (:sc signed-stack :from :argument :to :result) temp)
1212 (:arg-types double-float)
1213 (:result-types unsigned-num)
1214 (:translate double-float-low-bits)
1215 (:policy :fast-safe)
1220 (inst movsd temp float)
1221 (move lo-bits temp))
1223 (loadw lo-bits ebp-tn (frame-word-offset (tn-offset float))))
1225 (loadw lo-bits float double-float-value-slot
1226 other-pointer-lowtag)))
1227 (inst shl lo-bits 32)
1228 (inst shr lo-bits 32)))
1232 ;;;; complex float VOPs
1234 (define-vop (make-complex-single-float)
1235 (:translate complex)
1236 (:args (real :scs (single-reg fp-single-zero)
1238 :load-if (not (sc-is real fp-single-zero)))
1239 (imag :scs (single-reg fp-single-zero)
1240 :load-if (not (sc-is imag fp-single-zero))))
1241 (:arg-types single-float single-float)
1242 (:results (r :scs (complex-single-reg) :from (:argument 0)))
1243 (:result-types complex-single-float)
1244 (:note "inline complex single-float creation")
1245 (:policy :fast-safe)
1247 (cond ((sc-is real fp-single-zero)
1249 (unless (sc-is imag fp-single-zero)
1250 (inst unpcklps r imag)))
1251 ((location= real imag)
1253 (inst unpcklps r r))
1256 (unless (sc-is imag fp-single-zero)
1257 (inst unpcklps r imag))))))
1259 (define-vop (make-complex-double-float)
1260 (:translate complex)
1261 (:args (real :scs (double-reg fp-double-zero)
1263 :load-if (not (sc-is real fp-double-zero)))
1264 (imag :scs (double-reg fp-double-zero)
1265 :load-if (not (sc-is imag fp-double-zero))))
1266 (:arg-types double-float double-float)
1267 (:results (r :scs (complex-double-reg) :from (:argument 0)))
1268 (:result-types complex-double-float)
1269 (:note "inline complex double-float creation")
1270 (:policy :fast-safe)
1272 (cond ((sc-is real fp-double-zero)
1274 (unless (sc-is imag fp-double-zero)
1275 (inst unpcklpd r imag)))
1276 ((location= real imag)
1278 (inst unpcklpd r r))
1281 (unless (sc-is imag fp-double-zero)
1282 (inst unpcklpd r imag))))))
1284 (define-vop (complex-float-value)
1285 (:args (x :target r))
1286 (:temporary (:sc complex-double-reg) zero)
1288 (:variant-vars offset)
1289 (:policy :fast-safe)
1291 (cond ((sc-is x complex-double-reg)
1293 (inst xorpd zero zero)
1295 (0 (inst unpcklpd r zero))
1296 (1 (inst unpckhpd r zero))))
1297 ((sc-is x complex-single-reg)
1300 (0 (inst shufps r r #b11111100))
1301 (1 (inst shufps r r #b11111101))))
1302 ((sc-is r single-reg)
1303 (let ((ea (sc-case x
1304 (complex-single-stack
1306 (0 (ea-for-csf-real-stack x))
1307 (1 (ea-for-csf-imag-stack x))))
1310 (0 (ea-for-csf-real-desc x))
1311 (1 (ea-for-csf-imag-desc x)))))))
1313 ((sc-is r double-reg)
1314 (let ((ea (sc-case x
1315 (complex-double-stack
1317 (0 (ea-for-cdf-real-stack x))
1318 (1 (ea-for-cdf-imag-stack x))))
1321 (0 (ea-for-cdf-real-desc x))
1322 (1 (ea-for-cdf-imag-desc x)))))))
1324 (t (error "COMPLEX-FLOAT-VALUE VOP failure")))))
1326 (define-vop (realpart/complex-single-float complex-float-value)
1327 (:translate realpart)
1328 (:args (x :scs (complex-single-reg complex-single-stack descriptor-reg)
1330 (:arg-types complex-single-float)
1331 (:results (r :scs (single-reg)))
1332 (:result-types single-float)
1333 (:note "complex float realpart")
1336 (define-vop (realpart/complex-double-float complex-float-value)
1337 (:translate realpart)
1338 (:args (x :scs (complex-double-reg complex-double-stack descriptor-reg)
1340 (:arg-types complex-double-float)
1341 (:results (r :scs (double-reg)))
1342 (:result-types double-float)
1343 (:note "complex float realpart")
1346 (define-vop (imagpart/complex-single-float complex-float-value)
1347 (:translate imagpart)
1348 (:args (x :scs (complex-single-reg complex-single-stack descriptor-reg)
1350 (:arg-types complex-single-float)
1351 (:results (r :scs (single-reg)))
1352 (:result-types single-float)
1353 (:note "complex float imagpart")
1356 (define-vop (imagpart/complex-double-float complex-float-value)
1357 (:translate imagpart)
1358 (:args (x :scs (complex-double-reg complex-double-stack descriptor-reg)
1360 (:arg-types complex-double-float)
1361 (:results (r :scs (double-reg)))
1362 (:result-types double-float)
1363 (:note "complex float imagpart")
1367 ;;; hack dummy VOPs to bias the representation selection of their
1368 ;;; arguments towards a FP register, which can help avoid consing at
1369 ;;; inappropriate locations
1370 (defknown double-float-reg-bias (double-float) (values))
1371 (define-vop (double-float-reg-bias)
1372 (:translate double-float-reg-bias)
1373 (:args (x :scs (double-reg double-stack) :load-if nil))
1374 (:arg-types double-float)
1375 (:policy :fast-safe)
1376 (:note "inline dummy FP register bias")
1379 (defknown single-float-reg-bias (single-float) (values))
1380 (define-vop (single-float-reg-bias)
1381 (:translate single-float-reg-bias)
1382 (:args (x :scs (single-reg single-stack) :load-if nil))
1383 (:arg-types single-float)
1384 (:policy :fast-safe)
1385 (:note "inline dummy FP register bias")
1389 (defknown swap-complex ((complex float)) (complex float)
1390 (foldable flushable movable always-translatable))
1391 (defoptimizer (swap-complex derive-type) ((x))
1392 (sb!c::lvar-type x))
1393 (defun swap-complex (x)
1394 (complex (imagpart x) (realpart x)))
1395 (define-vop (swap-complex-single-float)
1396 (:translate swap-complex)
1397 (:policy :fast-safe)
1398 (:args (x :scs (complex-single-reg) :target r))
1399 (:arg-types complex-single-float)
1400 (:results (r :scs (complex-single-reg)))
1401 (:result-types complex-single-float)
1404 (inst shufps r r #b11110001)))
1405 (define-vop (swap-complex-double-float)
1406 (:translate swap-complex)
1407 (:policy :fast-safe)
1408 (:args (x :scs (complex-double-reg) :target r))
1409 (:arg-types complex-double-float)
1410 (:results (r :scs (complex-double-reg)))
1411 (:result-types complex-double-float)
1414 (inst shufpd r r #b01)))