1 ;;;; floating point support for the x86
3 ;;;; This software is part of the SBCL system. See the README file for
6 ;;;; This software is derived from the CMU CL system, which was
7 ;;;; written at Carnegie Mellon University and released into the
8 ;;;; public domain. The software is in the public domain and is
9 ;;;; provided with absolutely no warranty. See the COPYING and CREDITS
10 ;;;; files for more information.
14 (macrolet ((ea-for-xf-desc (tn slot)
17 :disp (- (* ,slot n-word-bytes)
18 other-pointer-lowtag))))
19 (defun ea-for-df-desc (tn)
20 (ea-for-xf-desc tn double-float-value-slot))
22 (defun ea-for-csf-data-desc (tn)
23 (ea-for-xf-desc tn complex-single-float-data-slot))
24 (defun ea-for-csf-real-desc (tn)
25 (ea-for-xf-desc tn complex-single-float-data-slot))
26 (defun ea-for-csf-imag-desc (tn)
27 (ea-for-xf-desc tn (+ complex-single-float-data-slot 1/2)))
29 (defun ea-for-cdf-data-desc (tn)
30 (ea-for-xf-desc tn complex-double-float-real-slot))
31 (defun ea-for-cdf-real-desc (tn)
32 (ea-for-xf-desc tn complex-double-float-real-slot))
33 (defun ea-for-cdf-imag-desc (tn)
34 (ea-for-xf-desc tn complex-double-float-imag-slot)))
36 (macrolet ((ea-for-xf-stack (tn kind)
37 (declare (ignore kind))
40 :disp (frame-byte-offset (tn-offset ,tn)))))
41 (defun ea-for-sf-stack (tn)
42 (ea-for-xf-stack tn :single))
43 (defun ea-for-df-stack (tn)
44 (ea-for-xf-stack tn :double)))
46 ;;; complex float stack EAs
47 (macrolet ((ea-for-cxf-stack (tn kind slot &optional base)
50 :disp (frame-byte-offset
52 (cond ((= (tn-offset ,base) rsp-offset)
54 ((= (tn-offset ,base) rbp-offset)
56 (t (error "Unexpected offset.")))
66 (defun ea-for-csf-data-stack (tn &optional (base rbp-tn))
67 (ea-for-cxf-stack tn :single :real base))
68 (defun ea-for-csf-real-stack (tn &optional (base rbp-tn))
69 (ea-for-cxf-stack tn :single :real base))
70 (defun ea-for-csf-imag-stack (tn &optional (base rbp-tn))
71 (ea-for-cxf-stack tn :single :imag base))
73 (defun ea-for-cdf-data-stack (tn &optional (base rbp-tn))
74 (ea-for-cxf-stack tn :double :real base))
75 (defun ea-for-cdf-real-stack (tn &optional (base rbp-tn))
76 (ea-for-cxf-stack tn :double :real base))
77 (defun ea-for-cdf-imag-stack (tn &optional (base rbp-tn))
78 (ea-for-cxf-stack tn :double :imag base)))
82 ;;; X is source, Y is destination.
84 (define-move-fun (load-fp-zero 1) (vop x y)
85 ((fp-single-zero) (single-reg)
86 (fp-double-zero) (double-reg)
87 (fp-complex-single-zero) (complex-single-reg)
88 (fp-complex-double-zero) (complex-double-reg))
91 ((single-reg complex-single-reg) (inst xorps y y))
92 ((double-reg complex-double-reg) (inst xorpd y y))))
94 (define-move-fun (load-fp-immediate 1) (vop x y)
95 ((fp-single-immediate) (single-reg)
96 (fp-double-immediate) (double-reg)
97 (fp-complex-single-immediate) (complex-single-reg)
98 (fp-complex-double-immediate) (complex-double-reg))
99 (let ((x (register-inline-constant (tn-value x))))
101 (single-reg (inst movss y x))
102 (double-reg (inst movsd y x))
103 (complex-single-reg (inst movq y x))
104 (complex-double-reg (inst movapd y x)))))
106 (define-move-fun (load-single 2) (vop x y)
107 ((single-stack) (single-reg))
108 (inst movss y (ea-for-sf-stack x)))
110 (define-move-fun (store-single 2) (vop x y)
111 ((single-reg) (single-stack))
112 (inst movss (ea-for-sf-stack y) x))
114 (define-move-fun (load-double 2) (vop x y)
115 ((double-stack) (double-reg))
116 (inst movsd y (ea-for-df-stack x)))
118 (define-move-fun (store-double 2) (vop x y)
119 ((double-reg) (double-stack))
120 (inst movsd (ea-for-df-stack y) x))
122 (eval-when (:compile-toplevel :execute)
123 (setf *read-default-float-format* 'single-float))
125 ;;;; complex float move functions
127 ;;; X is source, Y is destination.
128 (define-move-fun (load-complex-single 2) (vop x y)
129 ((complex-single-stack) (complex-single-reg))
130 (inst movq y (ea-for-csf-data-stack x)))
132 (define-move-fun (store-complex-single 2) (vop x y)
133 ((complex-single-reg) (complex-single-stack))
134 (inst movq (ea-for-csf-data-stack y) x))
136 (define-move-fun (load-complex-double 2) (vop x y)
137 ((complex-double-stack) (complex-double-reg))
138 (inst movupd y (ea-for-cdf-data-stack x)))
140 (define-move-fun (store-complex-double 2) (vop x y)
141 ((complex-double-reg) (complex-double-stack))
142 (inst movupd (ea-for-cdf-data-stack y) x))
146 ;;; float register to register moves
147 (macrolet ((frob (vop sc)
152 :load-if (not (location= x y))))
153 (:results (y :scs (,sc)
154 :load-if (not (location= x y))))
158 (define-move-vop ,vop :move (,sc) (,sc)))))
159 (frob single-move single-reg)
160 (frob double-move double-reg)
161 (frob complex-single-move complex-single-reg)
162 (frob complex-double-move complex-double-reg))
165 ;;; Move from float to a descriptor reg. allocating a new float
166 ;;; object in the process.
167 (define-vop (move-from-single)
168 (:args (x :scs (single-reg) :to :save))
169 (:results (y :scs (descriptor-reg)))
170 (:note "float to pointer coercion")
174 (inst or y single-float-widetag)))
176 (define-move-vop move-from-single :move
177 (single-reg) (descriptor-reg))
179 (define-vop (move-from-double)
180 (:args (x :scs (double-reg) :to :save))
181 (:results (y :scs (descriptor-reg)))
183 (:note "float to pointer coercion")
185 (with-fixed-allocation (y
189 (inst movsd (ea-for-df-desc y) x))))
190 (define-move-vop move-from-double :move
191 (double-reg) (descriptor-reg))
193 ;;; Move from a descriptor to a float register.
194 (define-vop (move-to-single-reg)
195 (:args (x :scs (descriptor-reg) :target tmp
196 :load-if (not (sc-is x control-stack))))
197 (:temporary (:sc unsigned-reg :from :argument :to :result) tmp)
198 (:results (y :scs (single-reg)))
199 (:note "pointer to float coercion")
207 ;; When the single-float descriptor is in memory, the untagging
208 ;; is done in the target XMM register. This is faster than going
209 ;; through a general-purpose register and the code is smaller.
211 (inst shufps y y #4r3331)))))
212 (define-move-vop move-to-single-reg :move (descriptor-reg) (single-reg))
214 ;;; Move from a descriptor to a float stack.
215 (define-vop (move-to-single-stack)
216 (:args (x :scs (descriptor-reg) :target tmp))
217 (:temporary (:sc unsigned-reg :from :argument :to :result) tmp)
218 (:results (y :scs (single-stack)))
219 (:note "pointer to float coercion")
223 (let ((slot (make-ea :dword :base rbp-tn
224 :disp (frame-byte-offset (tn-offset y)))))
225 (inst mov slot (reg-in-size tmp :dword)))))
226 (define-move-vop move-to-single-stack :move (descriptor-reg) (single-stack))
228 (define-vop (move-to-double)
229 (:args (x :scs (descriptor-reg)))
230 (:results (y :scs (double-reg)))
231 (:note "pointer to float coercion")
233 (inst movsd y (ea-for-df-desc x))))
234 (define-move-vop move-to-double :move (descriptor-reg) (double-reg))
237 ;;; Move from complex float to a descriptor reg. allocating a new
238 ;;; complex float object in the process.
239 (define-vop (move-from-complex-single)
240 (:args (x :scs (complex-single-reg) :to :save))
241 (:results (y :scs (descriptor-reg)))
243 (:note "complex float to pointer coercion")
245 (with-fixed-allocation (y
246 complex-single-float-widetag
247 complex-single-float-size
249 (inst movq (ea-for-csf-data-desc y) x))))
250 (define-move-vop move-from-complex-single :move
251 (complex-single-reg) (descriptor-reg))
253 (define-vop (move-from-complex-double)
254 (:args (x :scs (complex-double-reg) :to :save))
255 (:results (y :scs (descriptor-reg)))
257 (:note "complex float to pointer coercion")
259 (with-fixed-allocation (y
260 complex-double-float-widetag
261 complex-double-float-size
263 (inst movapd (ea-for-cdf-data-desc y) x))))
264 (define-move-vop move-from-complex-double :move
265 (complex-double-reg) (descriptor-reg))
267 ;;; Move from a descriptor to a complex float register.
268 (macrolet ((frob (name sc format)
271 (:args (x :scs (descriptor-reg)))
272 (:results (y :scs (,sc)))
273 (:note "pointer to complex float coercion")
277 '(inst movq y (ea-for-csf-data-desc x)))
279 '(inst movapd y (ea-for-cdf-data-desc x))))))
280 (define-move-vop ,name :move (descriptor-reg) (,sc)))))
281 (frob move-to-complex-single complex-single-reg :single)
282 (frob move-to-complex-double complex-double-reg :double))
284 ;;;; the move argument vops
286 ;;;; Note these are also used to stuff fp numbers onto the c-call
287 ;;;; stack so the order is different than the lisp-stack.
289 ;;; the general MOVE-ARG VOP
290 (macrolet ((frob (name sc stack-sc format)
293 (:args (x :scs (,sc) :target y)
295 :load-if (not (sc-is y ,sc))))
297 (:note "float argument move")
298 (:generator ,(case format (:single 2) (:double 3) )
303 (if (= (tn-offset fp) esp-offset)
304 (let* ((offset (* (tn-offset y) n-word-bytes))
305 (ea (make-ea :dword :base fp :disp offset)))
307 (:single '((inst movss ea x)))
308 (:double '((inst movsd ea x)))))
311 :disp (frame-byte-offset (tn-offset y)))))
313 (:single '((inst movss ea x)))
314 (:double '((inst movsd ea x))))))))))
315 (define-move-vop ,name :move-arg
316 (,sc descriptor-reg) (,sc)))))
317 (frob move-single-float-arg single-reg single-stack :single)
318 (frob move-double-float-arg double-reg double-stack :double))
320 ;;;; complex float MOVE-ARG VOP
321 (macrolet ((frob (name sc stack-sc format)
324 (:args (x :scs (,sc) :target y)
326 :load-if (not (sc-is y ,sc))))
328 (:note "complex float argument move")
329 (:generator ,(ecase format (:single 2) (:double 3))
336 '(inst movq (ea-for-csf-data-stack y fp) x))
338 '(inst movupd (ea-for-cdf-data-stack y fp) x)))))))
339 (define-move-vop ,name :move-arg
340 (,sc descriptor-reg) (,sc)))))
341 (frob move-complex-single-float-arg
342 complex-single-reg complex-single-stack :single)
343 (frob move-complex-double-float-arg
344 complex-double-reg complex-double-stack :double))
346 (define-move-vop move-arg :move-arg
347 (single-reg double-reg
348 complex-single-reg complex-double-reg)
354 (define-vop (float-op)
358 (:note "inline float arithmetic")
360 (:save-p :compute-only))
362 (macrolet ((frob (name comm-name sc constant-sc ptype)
364 (define-vop (,name float-op)
365 (:args (x :scs (,sc ,constant-sc)
367 :load-if (not (sc-is x ,constant-sc)))
368 (y :scs (,sc ,constant-sc)
369 :load-if (not (sc-is y ,constant-sc))))
370 (:results (r :scs (,sc)))
371 (:arg-types ,ptype ,ptype)
372 (:result-types ,ptype))
373 (define-vop (,comm-name float-op)
374 (:args (x :scs (,sc ,constant-sc)
376 :load-if (not (sc-is x ,constant-sc)))
377 (y :scs (,sc ,constant-sc)
379 :load-if (not (sc-is y ,constant-sc))))
380 (:results (r :scs (,sc)))
381 (:arg-types ,ptype ,ptype)
382 (:result-types ,ptype)))))
383 (frob single-float-op single-float-comm-op
384 single-reg fp-single-immediate single-float)
385 (frob double-float-op double-float-comm-op
386 double-reg fp-double-immediate double-float)
387 (frob complex-single-float-op complex-single-float-comm-op
388 complex-single-reg fp-complex-single-immediate
389 complex-single-float)
390 (frob complex-double-float-op complex-double-float-comm-op
391 complex-double-reg fp-complex-double-immediate
392 complex-double-float))
394 (macrolet ((generate (opinst commutative constant-sc load-inst)
395 `(flet ((get-constant (tn)
396 (register-inline-constant
397 ,@(and (eq constant-sc 'fp-single-immediate)
400 (declare (ignorable #'get-constant))
403 (when (sc-is y ,constant-sc)
404 (setf y (get-constant y)))
406 ((and ,commutative (location= y r))
407 (when (sc-is x ,constant-sc)
408 (setf x (get-constant x)))
410 ((not (location= r y))
411 (if (sc-is x ,constant-sc)
412 (inst ,load-inst r (get-constant x))
414 (when (sc-is y ,constant-sc)
415 (setf y (get-constant y)))
418 (if (sc-is x ,constant-sc)
419 (inst ,load-inst tmp (get-constant x))
423 (frob (op sinst sname scost dinst dname dcost commutative
424 &optional csinst csname cscost cdinst cdname cdcost)
426 (define-vop (,sname ,(if commutative
427 'single-float-comm-op
430 (:temporary (:sc single-reg) tmp)
432 (generate ,sinst ,commutative fp-single-immediate movss)))
433 (define-vop (,dname ,(if commutative
434 'double-float-comm-op
437 (:temporary (:sc double-reg) tmp)
439 (generate ,dinst ,commutative fp-double-immediate movsd)))
441 `(define-vop (,csname
443 'complex-single-float-comm-op
444 'complex-single-float-op))
446 (:temporary (:sc complex-single-reg) tmp)
448 (generate ,csinst ,commutative
449 fp-complex-single-immediate movq))))
451 `(define-vop (,cdname
453 'complex-double-float-comm-op
454 'complex-double-float-op))
456 (:temporary (:sc complex-double-reg) tmp)
458 (generate ,cdinst ,commutative
459 fp-complex-double-immediate movapd)))))))
460 (frob + addss +/single-float 2 addsd +/double-float 2 t
461 addps +/complex-single-float 3 addpd +/complex-double-float 3)
462 (frob - subss -/single-float 2 subsd -/double-float 2 nil
463 subps -/complex-single-float 3 subpd -/complex-double-float 3)
464 (frob * mulss */single-float 4 mulsd */double-float 5 t)
465 (frob / divss //single-float 12 divsd //double-float 19 nil))
467 (macrolet ((frob (op cost commutativep
468 duplicate-inst op-inst real-move-inst complex-move-inst
469 real-sc real-constant-sc real-type
470 complex-sc complex-constant-sc complex-type
471 real-complex-name complex-real-name)
472 (cond ((not duplicate-inst) ; simple case
473 `(flet ((load-into (r x)
476 (inst ,real-move-inst r
477 (register-inline-constant (tn-value x))))
478 (,complex-constant-sc
479 (inst ,complex-move-inst r
480 (register-inline-constant (tn-value x))))
482 ,(when real-complex-name
483 `(define-vop (,real-complex-name float-op)
485 (:args (x :scs (,real-sc ,real-constant-sc)
487 :load-if (not (sc-is x ,real-constant-sc)))
488 (y :scs (,complex-sc ,complex-constant-sc)
489 ,@(when commutativep '(:target r))
490 :load-if (not (sc-is y ,complex-constant-sc))))
491 (:arg-types ,real-type ,complex-type)
492 (:results (r :scs (,complex-sc)
493 ,@(unless commutativep '(:from (:argument 0)))))
494 (:result-types ,complex-type)
497 `(when (location= y r)
500 (when (sc-is y ,real-constant-sc ,complex-constant-sc)
501 (setf y (register-inline-constant
502 :aligned (tn-value y))))
503 (inst ,op-inst r y))))
505 ,(when complex-real-name
506 `(define-vop (,complex-real-name float-op)
508 (:args (x :scs (,complex-sc ,complex-constant-sc)
510 :load-if (not (sc-is x ,complex-constant-sc)))
511 (y :scs (,real-sc ,real-constant-sc)
512 ,@(when commutativep '(:target r))
513 :load-if (not (sc-is y ,real-constant-sc))))
514 (:arg-types ,complex-type ,real-type)
515 (:results (r :scs (,complex-sc)
516 ,@(unless commutativep '(:from (:argument 0)))))
517 (:result-types ,complex-type)
520 `(when (location= y r)
523 (when (sc-is y ,real-constant-sc ,complex-constant-sc)
524 (setf y (register-inline-constant
525 :aligned (tn-value y))))
526 (inst ,op-inst r y))))))
527 (commutativep ; must duplicate, but commutative
529 ,(when real-complex-name
530 `(define-vop (,real-complex-name float-op)
532 (:args (x :scs (,real-sc ,real-constant-sc)
534 :load-if (not (sc-is x ,real-constant-sc)))
535 (y :scs (,complex-sc ,complex-constant-sc)
538 :load-if (not (sc-is y ,complex-constant-sc))))
539 (:arg-types ,real-type ,complex-type)
540 (:temporary (:sc ,complex-sc :target r
544 (:results (r :scs (,complex-sc)))
545 (:result-types ,complex-type)
547 (if (sc-is x ,real-constant-sc)
548 (inst ,complex-move-inst dup
549 (register-inline-constant
550 (complex (tn-value x) (tn-value x))))
554 (when (location= dup r)
556 (if (sc-is y ,complex-constant-sc)
557 (inst ,complex-move-inst r
558 (register-inline-constant (tn-value y)))
560 (when (sc-is dup ,complex-constant-sc)
561 (setf dup (register-inline-constant
562 :aligned (tn-value dup))))
563 (inst ,op-inst r dup))))
565 ,(when complex-real-name
566 `(define-vop (,complex-real-name float-op)
568 (:args (x :scs (,complex-sc ,complex-constant-sc)
571 :load-if (not (sc-is x ,complex-constant-sc)))
572 (y :scs (,real-sc ,real-constant-sc)
574 :load-if (not (sc-is y ,real-constant-sc))))
575 (:arg-types ,complex-type ,real-type)
576 (:temporary (:sc ,complex-sc :target r
580 (:results (r :scs (,complex-sc)))
581 (:result-types ,complex-type)
583 (if (sc-is y ,real-constant-sc)
584 (inst ,complex-move-inst dup
585 (register-inline-constant
586 (complex (tn-value y) (tn-value y))))
589 (when (location= dup r)
591 (if (sc-is x ,complex-constant-sc)
592 (inst ,complex-move-inst r
593 (register-inline-constant (tn-value x)))
595 (when (sc-is dup ,complex-constant-sc)
596 (setf dup (register-inline-constant
597 :aligned (tn-value dup))))
598 (inst ,op-inst r dup))))))
599 (t ; duplicate, not commutative
601 ,(when real-complex-name
602 `(define-vop (,real-complex-name float-op)
604 (:args (x :scs (,real-sc ,real-constant-sc)
606 :load-if (not (sc-is x ,real-constant-sc)))
607 (y :scs (,complex-sc ,complex-constant-sc)
609 :load-if (not (sc-is y ,complex-constant-sc))))
610 (:arg-types ,real-type ,complex-type)
611 (:results (r :scs (,complex-sc) :from (:argument 0)))
612 (:result-types ,complex-type)
614 (if (sc-is x ,real-constant-sc)
615 (inst ,complex-move-inst dup
616 (register-inline-constant
617 (complex (tn-value x) (tn-value x))))
621 (when (sc-is y ,complex-constant-sc)
622 (setf y (register-inline-constant
623 :aligned (tn-value y))))
624 (inst ,op-inst r y))))
626 ,(when complex-real-name
627 `(define-vop (,complex-real-name float-op)
629 (:args (x :scs (,complex-sc)
632 (y :scs (,real-sc ,real-constant-sc)
634 :load-if (not (sc-is y ,complex-constant-sc))))
635 (:arg-types ,complex-type ,real-type)
636 (:temporary (:sc ,complex-sc :from (:argument 1))
638 (:results (r :scs (,complex-sc) :from :eval))
639 (:result-types ,complex-type)
641 (if (sc-is y ,real-constant-sc)
642 (setf dup (register-inline-constant
643 :aligned (complex (tn-value y)
648 (inst ,op-inst r dup))))))))
649 (def-real-complex-op (op commutativep duplicatep
650 single-inst single-real-complex-name single-complex-real-name single-cost
651 double-inst double-real-complex-name double-complex-real-name double-cost)
653 (frob ,op ,single-cost ,commutativep
657 (inst unpcklps dup dup)))
658 ,single-inst movss movq
659 single-reg fp-single-immediate single-float
660 complex-single-reg fp-complex-single-immediate complex-single-float
661 ,single-real-complex-name ,single-complex-real-name)
662 (frob ,op ,double-cost ,commutativep
666 (inst unpcklpd dup dup)))
667 ,double-inst movsd movapd
668 double-reg fp-double-immediate double-float
669 complex-double-reg fp-complex-double-immediate complex-double-float
670 ,double-real-complex-name ,double-complex-real-name))))
671 (def-real-complex-op + t nil
672 addps +/real-complex-single-float +/complex-real-single-float 3
673 addpd +/real-complex-double-float +/complex-real-double-float 4)
674 (def-real-complex-op - nil nil
675 subps -/real-complex-single-float -/complex-real-single-float 3
676 subpd -/real-complex-double-float -/complex-real-double-float 4)
677 (def-real-complex-op * t t
678 mulps */real-complex-single-float */complex-real-single-float 4
679 mulpd */real-complex-double-float */complex-real-double-float 5)
680 (def-real-complex-op / nil t
682 divpd nil //complex-real-double-float 19))
684 (define-vop (//complex-real-single-float float-op)
686 (:args (x :scs (complex-single-reg fp-complex-single-immediate fp-complex-single-zero)
689 :load-if (not (sc-is x fp-complex-single-immediate fp-complex-single-zero)))
690 (y :scs (single-reg fp-single-immediate fp-single-zero)
692 :load-if (not (sc-is y fp-single-immediate fp-single-zero))))
693 (:arg-types complex-single-float single-float)
694 (:temporary (:sc complex-single-reg :from (:argument 1)) dup)
695 (:results (r :scs (complex-single-reg)))
696 (:result-types complex-single-float)
698 (flet ((duplicate (x)
699 (let ((word (ldb (byte 64 0)
700 (logior (ash (single-float-bits (imagpart x)) 32)
702 (single-float-bits (realpart x)))))))
703 (register-inline-constant :oword (logior (ash word 64) word)))))
706 (setf dup (duplicate (complex (tn-value y) (tn-value y)))))
708 (inst xorps dup dup))
710 (inst shufps dup dup #b00000000)))
712 (fp-complex-single-immediate
713 (inst movaps r (duplicate (tn-value x))))
714 (fp-complex-single-zero
718 (inst unpcklpd r r)))
722 ;; Complex multiplication
723 ;; r := rx * ry - ix * iy
724 ;; i := rx * iy + ix * ry
726 ;; Transpose for SIMDness
731 ;;+ [ix ix] * [-iy ry]
734 (macrolet ((define-complex-* (name cost type sc tmp-p &body body)
735 `(define-vop (,name float-op)
737 (:args (x :scs (,sc) :target r)
738 (y :scs (,sc) :target copy-y))
739 (:arg-types ,type ,type)
740 (:temporary (:sc ,sc) imag)
741 (:temporary (:sc ,sc :from :eval) copy-y)
743 `((:temporary (:sc ,sc) xmm)))
744 (:results (r :scs (,sc) :from :eval))
745 (:result-types ,type)
747 (when (or (location= x copy-y)
751 (define-complex-* */complex-single-float 20
752 complex-single-float complex-single-reg t
757 (inst unpckhpd imag xmm)
758 (inst unpcklpd r xmm)
759 (move copy-y y) ; y == r only if y == x == r
764 (inst shufps y y #b11110001)
765 (inst xorps y (register-inline-constant :oword (ash 1 31)))
769 (define-complex-* */complex-double-float 25
770 complex-double-float complex-double-reg nil
776 (inst unpckhpd imag imag)
780 (inst shufpd y y #b01)
781 (inst xorpd y (register-inline-constant :oword (ash 1 63)))
784 (inst addpd r imag)))
787 (:args (x :scs (double-reg)))
788 (:results (y :scs (double-reg)))
791 (:arg-types double-float)
792 (:result-types double-float)
793 (:note "inline float arithmetic")
795 (:save-p :compute-only)
797 (unless (location= x y)
799 (note-this-location vop :internal-error)
802 (macrolet ((frob ((name translate sc type) &body body)
804 (:args (x :scs (,sc) :target y))
805 (:results (y :scs (,sc)))
806 (:translate ,translate)
809 (:result-types ,type)
810 (:note "inline float arithmetic")
812 (:save-p :compute-only)
814 (note-this-location vop :internal-error)
815 ;; we should be able to do this better. what we
816 ;; really would like to do is use the target as the
817 ;; temp whenever it's not also the source
820 (frob (%negate/double-float %negate double-reg double-float)
821 (inst xorpd y (register-inline-constant :oword (ash 1 63))))
822 (frob (%negate/complex-double-float %negate complex-double-reg complex-double-float)
823 (inst xorpd y (register-inline-constant
824 :oword (logior (ash 1 127) (ash 1 63)))))
825 (frob (conjugate/complex-double-float conjugate complex-double-reg complex-double-float)
826 (inst xorpd y (register-inline-constant :oword (ash 1 127))))
827 (frob (%negate/single-float %negate single-reg single-float)
828 (inst xorps y (register-inline-constant :oword (ash 1 31))))
829 (frob (%negate/complex-single-float %negate complex-single-reg complex-single-float)
830 (inst xorps y (register-inline-constant
831 :oword (logior (ash 1 31) (ash 1 63)))))
832 (frob (conjugate/complex-single-float conjugate complex-single-reg complex-single-float)
833 (inst xorpd y (register-inline-constant :oword (ash 1 63))))
834 (frob (abs/double-float abs double-reg double-float)
835 (inst andpd y (register-inline-constant :oword (ldb (byte 63 0) -1))))
836 (frob (abs/single-float abs single-reg single-float)
837 (inst andps y (register-inline-constant :oword (ldb (byte 31 0) -1)))))
842 (define-vop (float-compare)
845 (:save-p :compute-only)
846 (:note "inline float comparison"))
849 (macrolet ((define-float-eql (name cost sc constant-sc type)
850 `(define-vop (,name float-compare)
852 (:args (x :scs (,sc ,constant-sc)
854 :load-if (not (sc-is x ,constant-sc)))
855 (y :scs (,sc ,constant-sc)
857 :load-if (not (sc-is y ,constant-sc))))
858 (:arg-types ,type ,type)
859 (:temporary (:sc ,sc :from :eval) mask)
860 (:temporary (:sc any-reg) bits)
863 (when (or (location= y mask)
864 (not (xmm-register-p x)))
866 (aver (xmm-register-p x))
868 (when (sc-is y ,constant-sc)
869 (setf y (register-inline-constant :aligned (tn-value y))))
870 (inst pcmpeqd mask y)
871 (inst movmskps bits mask)
872 (inst cmp bits #b1111)))))
873 (define-float-eql eql/single-float 4
874 single-reg fp-single-immediate single-float)
875 (define-float-eql eql/double-float 4
876 double-reg fp-double-immediate double-float)
877 (define-float-eql eql/complex-single-float 5
878 complex-single-reg fp-complex-single-immediate complex-single-float)
879 (define-float-eql eql/complex-double-float 5
880 complex-double-reg fp-complex-double-immediate complex-double-float))
882 ;;; comiss and comisd can cope with one or other arg in memory: we
883 ;;; could (should, indeed) extend these to cope with descriptor args
886 (define-vop (single-float-compare float-compare)
887 (:args (x :scs (single-reg))
888 (y :scs (single-reg single-stack fp-single-immediate)
889 :load-if (not (sc-is y single-stack fp-single-immediate))))
890 (:arg-types single-float single-float))
891 (define-vop (double-float-compare float-compare)
892 (:args (x :scs (double-reg))
893 (y :scs (double-reg double-stack descriptor-reg fp-double-immediate)
894 :load-if (not (sc-is y double-stack descriptor-reg fp-double-immediate))))
895 (:arg-types double-float double-float))
897 (define-vop (=/single-float single-float-compare)
899 (:args (x :scs (single-reg single-stack fp-single-immediate)
901 :load-if (not (sc-is x single-stack fp-single-immediate)))
902 (y :scs (single-reg single-stack fp-single-immediate)
904 :load-if (not (sc-is y single-stack fp-single-immediate))))
905 (:temporary (:sc single-reg :from :eval) xmm)
907 (:conditional not :p :ne)
910 (when (or (location= y xmm)
911 (and (not (xmm-register-p x))
915 (single-reg (setf xmm x))
916 (single-stack (inst movss xmm (ea-for-sf-stack x)))
918 (inst movss xmm (register-inline-constant (tn-value x)))))
921 (setf y (ea-for-sf-stack y)))
923 (setf y (register-inline-constant (tn-value y))))
925 (note-this-location vop :internal-error)
927 ;; if PF&CF, there was a NaN involved => not equal
928 ;; otherwise, ZF => equal
931 (define-vop (=/double-float double-float-compare)
933 (:args (x :scs (double-reg double-stack fp-double-immediate descriptor-reg)
935 :load-if (not (sc-is x double-stack fp-double-immediate descriptor-reg)))
936 (y :scs (double-reg double-stack fp-double-immediate descriptor-reg)
938 :load-if (not (sc-is y double-stack fp-double-immediate descriptor-reg))))
939 (:temporary (:sc double-reg :from :eval) xmm)
941 (:conditional not :p :ne)
944 (when (or (location= y xmm)
945 (and (not (xmm-register-p x))
952 (inst movsd xmm (ea-for-df-stack x)))
954 (inst movsd xmm (register-inline-constant (tn-value x))))
956 (inst movsd xmm (ea-for-df-desc x))))
959 (setf y (ea-for-df-stack y)))
961 (setf y (register-inline-constant (tn-value y))))
963 (setf y (ea-for-df-desc y)))
965 (note-this-location vop :internal-error)
966 (inst comisd xmm y)))
968 (macrolet ((define-complex-float-= (complex-complex-name complex-real-name real-complex-name
969 real-sc real-constant-sc real-type
970 complex-sc complex-constant-sc complex-type
971 real-move-inst complex-move-inst
972 cmp-inst mask-inst mask)
974 (define-vop (,complex-complex-name float-compare)
976 (:args (x :scs (,complex-sc ,complex-constant-sc)
978 :load-if (not (sc-is x ,complex-constant-sc)))
979 (y :scs (,complex-sc ,complex-constant-sc)
981 :load-if (not (sc-is y ,complex-constant-sc))))
982 (:arg-types ,complex-type ,complex-type)
983 (:temporary (:sc ,complex-sc :from :eval) cmp)
984 (:temporary (:sc unsigned-reg) bits)
988 (when (location= y cmp)
992 (inst ,real-move-inst cmp (register-inline-constant
994 (,complex-constant-sc
995 (inst ,complex-move-inst cmp (register-inline-constant
999 (when (sc-is y ,real-constant-sc ,complex-constant-sc)
1000 (setf y (register-inline-constant :aligned (tn-value y))))
1001 (note-this-location vop :internal-error)
1002 (inst ,cmp-inst :eq cmp y)
1003 (inst ,mask-inst bits cmp)
1004 (inst cmp bits ,mask)))
1005 (define-vop (,complex-real-name ,complex-complex-name)
1006 (:args (x :scs (,complex-sc ,complex-constant-sc)
1008 :load-if (not (sc-is x ,complex-constant-sc)))
1009 (y :scs (,real-sc ,real-constant-sc)
1011 :load-if (not (sc-is y ,real-constant-sc))))
1012 (:arg-types ,complex-type ,real-type))
1013 (define-vop (,real-complex-name ,complex-complex-name)
1014 (:args (x :scs (,real-sc ,real-constant-sc)
1016 :load-if (not (sc-is x ,real-constant-sc)))
1017 (y :scs (,complex-sc ,complex-constant-sc)
1019 :load-if (not (sc-is y ,complex-constant-sc))))
1020 (:arg-types ,real-type ,complex-type)))))
1021 (define-complex-float-= =/complex-single-float =/complex-real-single-float =/real-complex-single-float
1022 single-reg fp-single-immediate single-float
1023 complex-single-reg fp-complex-single-immediate complex-single-float
1024 movss movq cmpps movmskps #b1111)
1025 (define-complex-float-= =/complex-double-float =/complex-real-double-float =/real-complex-double-float
1026 double-reg fp-double-immediate double-float
1027 complex-double-reg fp-complex-double-immediate complex-double-float
1028 movsd movapd cmppd movmskpd #b11))
1030 (macrolet ((define-</> (op single-name double-name &rest flags)
1032 (define-vop (,double-name double-float-compare)
1035 (:conditional ,@flags)
1039 (setf y (ea-for-df-stack y)))
1041 (setf y (ea-for-df-desc y)))
1042 (fp-double-immediate
1043 (setf y (register-inline-constant (tn-value y))))
1046 (define-vop (,single-name single-float-compare)
1049 (:conditional ,@flags)
1053 (setf y (ea-for-sf-stack y)))
1054 (fp-single-immediate
1055 (setf y (register-inline-constant (tn-value y))))
1057 (inst comiss x y))))))
1058 (define-</> < <single-float <double-float not :p :nc)
1059 (define-</> > >single-float >double-float not :p :na))
1064 (macrolet ((frob (name translate inst to-sc to-type)
1065 `(define-vop (,name)
1066 (:args (x :scs (signed-stack signed-reg)))
1067 (:results (y :scs (,to-sc)))
1068 (:arg-types signed-num)
1069 (:result-types ,to-type)
1070 (:policy :fast-safe)
1071 (:note "inline float coercion")
1072 (:translate ,translate)
1074 (:save-p :compute-only)
1077 (single-reg (inst xorps y y))
1078 (double-reg (inst xorpd y y)))
1079 (note-this-location vop :internal-error)
1080 (inst ,inst y x)))))
1081 (frob %single-float/signed %single-float cvtsi2ss single-reg single-float)
1082 (frob %double-float/signed %double-float cvtsi2sd double-reg double-float))
1084 (macrolet ((frob (name translate inst from-scs from-type ea-func to-sc to-type)
1085 `(define-vop (,name)
1086 (:args (x :scs ,from-scs :target y))
1087 (:results (y :scs (,to-sc)))
1088 (:arg-types ,from-type)
1089 (:result-types ,to-type)
1090 (:policy :fast-safe)
1091 (:note "inline float coercion")
1092 (:translate ,translate)
1094 (:save-p :compute-only)
1096 (unless (location= x y)
1098 (single-reg (inst xorps y y))
1099 (double-reg (inst xorpd y y))))
1100 (note-this-location vop :internal-error)
1101 (inst ,inst y (sc-case x
1102 (,(first from-scs) x)
1103 (,(second from-scs) (,ea-func x))))
1104 ,(when (and (eq from-type 'double-float) ; if the input is wider
1105 (eq to-type 'single-float)) ; than the output, clear
1106 `(when (location= x y) ; noise in the high part
1107 (inst shufps y y #4r3330)))))))
1108 (frob %single-float/double-float %single-float cvtsd2ss
1109 (double-reg double-stack) double-float ea-for-df-stack
1110 single-reg single-float)
1112 (frob %double-float/single-float %double-float cvtss2sd
1113 (single-reg single-stack) single-float ea-for-sf-stack
1114 double-reg double-float))
1116 (macrolet ((frob (trans inst from-scs from-type ea-func)
1117 `(define-vop (,(symbolicate trans "/" from-type))
1118 (:args (x :scs ,from-scs))
1119 (:results (y :scs (signed-reg)))
1120 (:arg-types ,from-type)
1121 (:result-types signed-num)
1123 (:policy :fast-safe)
1124 (:note "inline float truncate")
1126 (:save-p :compute-only)
1128 (inst ,inst y (sc-case x
1129 (,(first from-scs) x)
1130 (,(second from-scs) (,ea-func x))))))))
1131 (frob %unary-truncate/single-float cvttss2si
1132 (single-reg single-stack) single-float ea-for-sf-stack)
1133 (frob %unary-truncate/double-float cvttsd2si
1134 (double-reg double-stack) double-float ea-for-df-stack)
1136 (frob %unary-round cvtss2si
1137 (single-reg single-stack) single-float ea-for-sf-stack)
1138 (frob %unary-round cvtsd2si
1139 (double-reg double-stack) double-float ea-for-df-stack))
1141 (define-vop (make-single-float)
1142 (:args (bits :scs (signed-reg) :target res
1143 :load-if (not (or (and (sc-is bits signed-stack)
1144 (sc-is res single-reg))
1145 (and (sc-is bits signed-stack)
1146 (sc-is res single-stack)
1147 (location= bits res))))))
1148 (:results (res :scs (single-reg single-stack)))
1149 (:arg-types signed-num)
1150 (:result-types single-float)
1151 (:translate make-single-float)
1152 (:policy :fast-safe)
1159 (inst mov res bits))
1161 (aver (location= bits res)))))
1165 (inst movd res bits))
1167 (inst movd res bits)))))))
1169 (define-vop (make-double-float)
1170 (:args (hi-bits :scs (signed-reg))
1171 (lo-bits :scs (unsigned-reg)))
1172 (:results (res :scs (double-reg)))
1173 (:temporary (:sc unsigned-reg) temp)
1174 (:arg-types signed-num unsigned-num)
1175 (:result-types double-float)
1176 (:translate make-double-float)
1177 (:policy :fast-safe)
1182 (inst or temp lo-bits)
1183 (inst movd res temp)))
1185 (define-vop (single-float-bits)
1186 (:args (float :scs (single-reg descriptor-reg)
1187 :load-if (not (sc-is float single-stack))))
1188 (:results (bits :scs (signed-reg)))
1189 (:temporary (:sc signed-stack :from :argument :to :result) stack-temp)
1190 (:arg-types single-float)
1191 (:result-types signed-num)
1192 (:translate single-float-bits)
1193 (:policy :fast-safe)
1200 (inst movss stack-temp float)
1201 (move bits stack-temp))
1206 (inst shr bits 32))))
1210 (inst movss bits float)))))
1213 (inst sar bits 32)))
1215 (define-vop (double-float-high-bits)
1216 (:args (float :scs (double-reg descriptor-reg)
1217 :load-if (not (sc-is float double-stack))))
1218 (:results (hi-bits :scs (signed-reg)))
1219 (:temporary (:sc signed-stack :from :argument :to :result) temp)
1220 (:arg-types double-float)
1221 (:result-types signed-num)
1222 (:translate double-float-high-bits)
1223 (:policy :fast-safe)
1228 (inst movsd temp float)
1229 (move hi-bits temp))
1231 (loadw hi-bits ebp-tn (frame-word-offset (tn-offset float))))
1233 (loadw hi-bits float double-float-value-slot
1234 other-pointer-lowtag)))
1235 (inst sar hi-bits 32)))
1237 (define-vop (double-float-low-bits)
1238 (:args (float :scs (double-reg descriptor-reg)
1239 :load-if (not (sc-is float double-stack))))
1240 (:results (lo-bits :scs (unsigned-reg)))
1241 (:temporary (:sc signed-stack :from :argument :to :result) temp)
1242 (:arg-types double-float)
1243 (:result-types unsigned-num)
1244 (:translate double-float-low-bits)
1245 (:policy :fast-safe)
1250 (inst movsd temp float)
1251 (move lo-bits temp))
1253 (loadw lo-bits ebp-tn (frame-word-offset (tn-offset float))))
1255 (loadw lo-bits float double-float-value-slot
1256 other-pointer-lowtag)))
1257 (inst shl lo-bits 32)
1258 (inst shr lo-bits 32)))
1262 ;;;; complex float VOPs
1264 (define-vop (make-complex-single-float)
1265 (:translate complex)
1266 (:args (real :scs (single-reg fp-single-zero)
1268 :load-if (not (sc-is real fp-single-zero)))
1269 (imag :scs (single-reg fp-single-zero)
1270 :load-if (not (sc-is imag fp-single-zero))))
1271 (:arg-types single-float single-float)
1272 (:results (r :scs (complex-single-reg) :from (:argument 0)))
1273 (:result-types complex-single-float)
1274 (:note "inline complex single-float creation")
1275 (:policy :fast-safe)
1277 (cond ((sc-is real fp-single-zero)
1279 (unless (sc-is imag fp-single-zero)
1280 (inst unpcklps r imag)))
1281 ((location= real imag)
1283 (inst unpcklps r r))
1286 (unless (sc-is imag fp-single-zero)
1287 (inst unpcklps r imag))))))
1289 (define-vop (make-complex-double-float)
1290 (:translate complex)
1291 (:args (real :scs (double-reg fp-double-zero)
1293 :load-if (not (sc-is real fp-double-zero)))
1294 (imag :scs (double-reg fp-double-zero)
1295 :load-if (not (sc-is imag fp-double-zero))))
1296 (:arg-types double-float double-float)
1297 (:results (r :scs (complex-double-reg) :from (:argument 0)))
1298 (:result-types complex-double-float)
1299 (:note "inline complex double-float creation")
1300 (:policy :fast-safe)
1302 (cond ((sc-is real fp-double-zero)
1304 (unless (sc-is imag fp-double-zero)
1305 (inst unpcklpd r imag)))
1306 ((location= real imag)
1308 (inst unpcklpd r r))
1311 (unless (sc-is imag fp-double-zero)
1312 (inst unpcklpd r imag))))))
1314 (define-vop (complex-float-value)
1315 (:args (x :target r))
1316 (:temporary (:sc complex-double-reg) zero)
1318 (:variant-vars offset)
1319 (:policy :fast-safe)
1321 (cond ((sc-is x complex-double-reg)
1323 (inst xorpd zero zero)
1325 (0 (inst unpcklpd r zero))
1326 (1 (inst unpckhpd r zero))))
1327 ((sc-is x complex-single-reg)
1330 (0 (inst shufps r r #b11111100))
1331 (1 (inst shufps r r #b11111101))))
1332 ((sc-is r single-reg)
1333 (let ((ea (sc-case x
1334 (complex-single-stack
1336 (0 (ea-for-csf-real-stack x))
1337 (1 (ea-for-csf-imag-stack x))))
1340 (0 (ea-for-csf-real-desc x))
1341 (1 (ea-for-csf-imag-desc x)))))))
1343 ((sc-is r double-reg)
1344 (let ((ea (sc-case x
1345 (complex-double-stack
1347 (0 (ea-for-cdf-real-stack x))
1348 (1 (ea-for-cdf-imag-stack x))))
1351 (0 (ea-for-cdf-real-desc x))
1352 (1 (ea-for-cdf-imag-desc x)))))))
1354 (t (error "COMPLEX-FLOAT-VALUE VOP failure")))))
1356 (define-vop (realpart/complex-single-float complex-float-value)
1357 (:translate realpart)
1358 (:args (x :scs (complex-single-reg complex-single-stack descriptor-reg)
1360 (:arg-types complex-single-float)
1361 (:results (r :scs (single-reg)))
1362 (:result-types single-float)
1363 (:note "complex float realpart")
1366 (define-vop (realpart/complex-double-float complex-float-value)
1367 (:translate realpart)
1368 (:args (x :scs (complex-double-reg complex-double-stack descriptor-reg)
1370 (:arg-types complex-double-float)
1371 (:results (r :scs (double-reg)))
1372 (:result-types double-float)
1373 (:note "complex float realpart")
1376 (define-vop (imagpart/complex-single-float complex-float-value)
1377 (:translate imagpart)
1378 (:args (x :scs (complex-single-reg complex-single-stack descriptor-reg)
1380 (:arg-types complex-single-float)
1381 (:results (r :scs (single-reg)))
1382 (:result-types single-float)
1383 (:note "complex float imagpart")
1386 (define-vop (imagpart/complex-double-float complex-float-value)
1387 (:translate imagpart)
1388 (:args (x :scs (complex-double-reg complex-double-stack descriptor-reg)
1390 (:arg-types complex-double-float)
1391 (:results (r :scs (double-reg)))
1392 (:result-types double-float)
1393 (:note "complex float imagpart")
1397 ;;; hack dummy VOPs to bias the representation selection of their
1398 ;;; arguments towards a FP register, which can help avoid consing at
1399 ;;; inappropriate locations
1400 (defknown double-float-reg-bias (double-float) (values))
1401 (define-vop (double-float-reg-bias)
1402 (:translate double-float-reg-bias)
1403 (:args (x :scs (double-reg double-stack) :load-if nil))
1404 (:arg-types double-float)
1405 (:policy :fast-safe)
1406 (:note "inline dummy FP register bias")
1409 (defknown single-float-reg-bias (single-float) (values))
1410 (define-vop (single-float-reg-bias)
1411 (:translate single-float-reg-bias)
1412 (:args (x :scs (single-reg single-stack) :load-if nil))
1413 (:arg-types single-float)
1414 (:policy :fast-safe)
1415 (:note "inline dummy FP register bias")
1419 (defknown swap-complex ((complex float)) (complex float)
1420 (foldable flushable movable always-translatable))
1421 (defoptimizer (swap-complex derive-type) ((x))
1422 (sb!c::lvar-type x))
1423 (defun swap-complex (x)
1424 (complex (imagpart x) (realpart x)))
1425 (define-vop (swap-complex-single-float)
1426 (:translate swap-complex)
1427 (:policy :fast-safe)
1428 (:args (x :scs (complex-single-reg) :target r))
1429 (:arg-types complex-single-float)
1430 (:results (r :scs (complex-single-reg)))
1431 (:result-types complex-single-float)
1434 (inst shufps r r #b11110001)))
1435 (define-vop (swap-complex-double-float)
1436 (:translate swap-complex)
1437 (:policy :fast-safe)
1438 (:args (x :scs (complex-double-reg) :target r))
1439 (:arg-types complex-double-float)
1440 (:results (r :scs (complex-double-reg)))
1441 (:result-types complex-double-float)
1444 (inst shufpd r r #b01)))