1 ;;;; floating point support for the x86
3 ;;;; This software is part of the SBCL system. See the README file for
6 ;;;; This software is derived from the CMU CL system, which was
7 ;;;; written at Carnegie Mellon University and released into the
8 ;;;; public domain. The software is in the public domain and is
9 ;;;; provided with absolutely no warranty. See the COPYING and CREDITS
10 ;;;; files for more information.
14 (macrolet ((ea-for-xf-desc (tn slot)
17 :disp (- (* ,slot n-word-bytes)
18 other-pointer-lowtag))))
19 (defun ea-for-df-desc (tn)
20 (ea-for-xf-desc tn double-float-value-slot))
22 (defun ea-for-csf-data-desc (tn)
23 (ea-for-xf-desc tn complex-single-float-data-slot))
24 (defun ea-for-csf-real-desc (tn)
25 (ea-for-xf-desc tn complex-single-float-data-slot))
26 (defun ea-for-csf-imag-desc (tn)
27 (ea-for-xf-desc tn (+ complex-single-float-data-slot 1/2)))
29 (defun ea-for-cdf-data-desc (tn)
30 (ea-for-xf-desc tn complex-double-float-real-slot))
31 (defun ea-for-cdf-real-desc (tn)
32 (ea-for-xf-desc tn complex-double-float-real-slot))
33 (defun ea-for-cdf-imag-desc (tn)
34 (ea-for-xf-desc tn complex-double-float-imag-slot)))
36 (macrolet ((ea-for-xf-stack (tn kind)
37 (declare (ignore kind))
40 :disp (frame-byte-offset (tn-offset ,tn)))))
41 (defun ea-for-sf-stack (tn)
42 (ea-for-xf-stack tn :single))
43 (defun ea-for-df-stack (tn)
44 (ea-for-xf-stack tn :double)))
46 ;;; complex float stack EAs
47 (macrolet ((ea-for-cxf-stack (tn kind slot &optional base)
50 :disp (frame-byte-offset
52 (cond ((= (tn-offset ,base) rsp-offset)
64 (defun ea-for-csf-data-stack (tn &optional (base rbp-tn))
65 (ea-for-cxf-stack tn :single :real base))
66 (defun ea-for-csf-real-stack (tn &optional (base rbp-tn))
67 (ea-for-cxf-stack tn :single :real base))
68 (defun ea-for-csf-imag-stack (tn &optional (base rbp-tn))
69 (ea-for-cxf-stack tn :single :imag base))
71 (defun ea-for-cdf-data-stack (tn &optional (base rbp-tn))
72 (ea-for-cxf-stack tn :double :real base))
73 (defun ea-for-cdf-real-stack (tn &optional (base rbp-tn))
74 (ea-for-cxf-stack tn :double :real base))
75 (defun ea-for-cdf-imag-stack (tn &optional (base rbp-tn))
76 (ea-for-cxf-stack tn :double :imag base)))
80 ;;; X is source, Y is destination.
82 (define-move-fun (load-fp-zero 1) (vop x y)
83 ((fp-single-zero) (single-reg)
84 (fp-double-zero) (double-reg)
85 (fp-complex-single-zero) (complex-single-reg)
86 (fp-complex-double-zero) (complex-double-reg))
89 ((single-reg complex-single-reg) (inst xorps y y))
90 ((double-reg complex-double-reg) (inst xorpd y y))))
92 (define-move-fun (load-fp-immediate 1) (vop x y)
93 ((fp-single-immediate) (single-reg)
94 (fp-double-immediate) (double-reg)
95 (fp-complex-single-immediate) (complex-single-reg)
96 (fp-complex-double-immediate) (complex-double-reg))
97 (let ((x (register-inline-constant (tn-value x))))
99 (single-reg (inst movss y x))
100 (double-reg (inst movsd y x))
101 (complex-single-reg (inst movq y x))
102 (complex-double-reg (inst movapd y x)))))
104 (define-move-fun (load-single 2) (vop x y)
105 ((single-stack) (single-reg))
106 (inst movss y (ea-for-sf-stack x)))
108 (define-move-fun (store-single 2) (vop x y)
109 ((single-reg) (single-stack))
110 (inst movss (ea-for-sf-stack y) x))
112 (define-move-fun (load-double 2) (vop x y)
113 ((double-stack) (double-reg))
114 (inst movsd y (ea-for-df-stack x)))
116 (define-move-fun (store-double 2) (vop x y)
117 ((double-reg) (double-stack))
118 (inst movsd (ea-for-df-stack y) x))
120 (eval-when (:compile-toplevel :execute)
121 (setf *read-default-float-format* 'single-float))
123 ;;;; complex float move functions
125 ;;; X is source, Y is destination.
126 (define-move-fun (load-complex-single 2) (vop x y)
127 ((complex-single-stack) (complex-single-reg))
128 (inst movq y (ea-for-csf-data-stack x)))
130 (define-move-fun (store-complex-single 2) (vop x y)
131 ((complex-single-reg) (complex-single-stack))
132 (inst movq (ea-for-csf-data-stack y) x))
134 (define-move-fun (load-complex-double 2) (vop x y)
135 ((complex-double-stack) (complex-double-reg))
136 (inst movupd y (ea-for-cdf-data-stack x)))
138 (define-move-fun (store-complex-double 2) (vop x y)
139 ((complex-double-reg) (complex-double-stack))
140 (inst movupd (ea-for-cdf-data-stack y) x))
144 ;;; float register to register moves
145 (macrolet ((frob (vop sc)
150 :load-if (not (location= x y))))
151 (:results (y :scs (,sc)
152 :load-if (not (location= x y))))
156 (define-move-vop ,vop :move (,sc) (,sc)))))
157 (frob single-move single-reg)
158 (frob double-move double-reg)
159 (frob complex-single-move complex-single-reg)
160 (frob complex-double-move complex-double-reg))
163 ;;; Move from float to a descriptor reg. allocating a new float
164 ;;; object in the process.
165 (define-vop (move-from-single)
166 (:args (x :scs (single-reg) :to :save))
167 (:results (y :scs (descriptor-reg)))
168 (:note "float to pointer coercion")
172 (inst or y single-float-widetag)))
174 (define-move-vop move-from-single :move
175 (single-reg) (descriptor-reg))
177 (define-vop (move-from-double)
178 (:args (x :scs (double-reg) :to :save))
179 (:results (y :scs (descriptor-reg)))
181 (:note "float to pointer coercion")
183 (with-fixed-allocation (y
187 (inst movsd (ea-for-df-desc y) x))))
188 (define-move-vop move-from-double :move
189 (double-reg) (descriptor-reg))
191 ;;; Move from a descriptor to a float register.
192 (define-vop (move-to-single-reg)
193 (:args (x :scs (descriptor-reg) :target tmp
194 :load-if (not (sc-is x control-stack))))
195 (:temporary (:sc unsigned-reg :from :argument :to :result) tmp)
196 (:results (y :scs (single-reg)))
197 (:note "pointer to float coercion")
205 ;; When the single-float descriptor is in memory, the untagging
206 ;; is done in the target XMM register. This is faster than going
207 ;; through a general-purpose register and the code is smaller.
209 (inst shufps y y #4r3331)))))
210 (define-move-vop move-to-single-reg :move (descriptor-reg) (single-reg))
212 ;;; Move from a descriptor to a float stack.
213 (define-vop (move-to-single-stack)
214 (:args (x :scs (descriptor-reg) :target tmp))
215 (:temporary (:sc unsigned-reg :from :argument :to :result) tmp)
216 (:results (y :scs (single-stack)))
217 (:note "pointer to float coercion")
221 (let ((slot (make-ea :dword :base rbp-tn
222 :disp (frame-byte-offset (tn-offset y)))))
223 (inst mov slot (reg-in-size tmp :dword)))))
224 (define-move-vop move-to-single-stack :move (descriptor-reg) (single-stack))
226 (define-vop (move-to-double)
227 (:args (x :scs (descriptor-reg)))
228 (:results (y :scs (double-reg)))
229 (:note "pointer to float coercion")
231 (inst movsd y (ea-for-df-desc x))))
232 (define-move-vop move-to-double :move (descriptor-reg) (double-reg))
235 ;;; Move from complex float to a descriptor reg. allocating a new
236 ;;; complex float object in the process.
237 (define-vop (move-from-complex-single)
238 (:args (x :scs (complex-single-reg) :to :save))
239 (:results (y :scs (descriptor-reg)))
241 (:note "complex float to pointer coercion")
243 (with-fixed-allocation (y
244 complex-single-float-widetag
245 complex-single-float-size
247 (inst movq (ea-for-csf-data-desc y) x))))
248 (define-move-vop move-from-complex-single :move
249 (complex-single-reg) (descriptor-reg))
251 (define-vop (move-from-complex-double)
252 (:args (x :scs (complex-double-reg) :to :save))
253 (:results (y :scs (descriptor-reg)))
255 (:note "complex float to pointer coercion")
257 (with-fixed-allocation (y
258 complex-double-float-widetag
259 complex-double-float-size
261 (inst movapd (ea-for-cdf-data-desc y) x))))
262 (define-move-vop move-from-complex-double :move
263 (complex-double-reg) (descriptor-reg))
265 ;;; Move from a descriptor to a complex float register.
266 (macrolet ((frob (name sc format)
269 (:args (x :scs (descriptor-reg)))
270 (:results (y :scs (,sc)))
271 (:note "pointer to complex float coercion")
275 '(inst movq y (ea-for-csf-data-desc x)))
277 '(inst movapd y (ea-for-cdf-data-desc x))))))
278 (define-move-vop ,name :move (descriptor-reg) (,sc)))))
279 (frob move-to-complex-single complex-single-reg :single)
280 (frob move-to-complex-double complex-double-reg :double))
282 ;;;; the move argument vops
284 ;;;; Note these are also used to stuff fp numbers onto the c-call
285 ;;;; stack so the order is different than the lisp-stack.
287 ;;; the general MOVE-ARG VOP
288 (macrolet ((frob (name sc stack-sc format)
291 (:args (x :scs (,sc) :target y)
293 :load-if (not (sc-is y ,sc))))
295 (:note "float argument move")
296 (:generator ,(case format (:single 2) (:double 3) )
301 (if (= (tn-offset fp) esp-offset)
302 (let* ((offset (* (tn-offset y) n-word-bytes))
303 (ea (make-ea :dword :base fp :disp offset)))
305 (:single '((inst movss ea x)))
306 (:double '((inst movsd ea x)))))
309 :disp (frame-byte-offset (tn-offset y)))))
311 (:single '((inst movss ea x)))
312 (:double '((inst movsd ea x))))))))))
313 (define-move-vop ,name :move-arg
314 (,sc descriptor-reg) (,sc)))))
315 (frob move-single-float-arg single-reg single-stack :single)
316 (frob move-double-float-arg double-reg double-stack :double))
318 ;;;; complex float MOVE-ARG VOP
319 (macrolet ((frob (name sc stack-sc format)
322 (:args (x :scs (,sc) :target y)
324 :load-if (not (sc-is y ,sc))))
326 (:note "complex float argument move")
327 (:generator ,(ecase format (:single 2) (:double 3))
334 '(inst movq (ea-for-csf-data-stack y fp) x))
336 '(inst movupd (ea-for-cdf-data-stack y fp) x)))))))
337 (define-move-vop ,name :move-arg
338 (,sc descriptor-reg) (,sc)))))
339 (frob move-complex-single-float-arg
340 complex-single-reg complex-single-stack :single)
341 (frob move-complex-double-float-arg
342 complex-double-reg complex-double-stack :double))
344 (define-move-vop move-arg :move-arg
345 (single-reg double-reg
346 complex-single-reg complex-double-reg)
352 (define-vop (float-op)
356 (:note "inline float arithmetic")
358 (:save-p :compute-only))
360 (macrolet ((frob (name comm-name sc constant-sc ptype)
362 (define-vop (,name float-op)
363 (:args (x :scs (,sc ,constant-sc)
365 :load-if (not (sc-is x ,constant-sc)))
366 (y :scs (,sc ,constant-sc)
367 :load-if (not (sc-is y ,constant-sc))))
368 (:results (r :scs (,sc)))
369 (:arg-types ,ptype ,ptype)
370 (:result-types ,ptype))
371 (define-vop (,comm-name float-op)
372 (:args (x :scs (,sc ,constant-sc)
374 :load-if (not (sc-is x ,constant-sc)))
375 (y :scs (,sc ,constant-sc)
377 :load-if (not (sc-is y ,constant-sc))))
378 (:results (r :scs (,sc)))
379 (:arg-types ,ptype ,ptype)
380 (:result-types ,ptype)))))
381 (frob single-float-op single-float-comm-op
382 single-reg fp-single-immediate single-float)
383 (frob double-float-op double-float-comm-op
384 double-reg fp-double-immediate double-float)
385 (frob complex-single-float-op complex-single-float-comm-op
386 complex-single-reg fp-complex-single-immediate
387 complex-single-float)
388 (frob complex-double-float-op complex-double-float-comm-op
389 complex-double-reg fp-complex-double-immediate
390 complex-double-float))
392 (macrolet ((generate (opinst commutative constant-sc load-inst)
393 `(flet ((get-constant (tn &optional maybe-aligned)
394 (declare (ignorable maybe-aligned))
395 (let ((value (tn-value tn)))
396 ,(if (eq constant-sc 'fp-complex-single-immediate)
398 (register-inline-constant
400 (register-inline-constant value))
401 `(register-inline-constant value)))))
402 (declare (ignorable #'get-constant))
405 (when (sc-is y ,constant-sc)
406 (setf y (get-constant y t)))
408 ((and ,commutative (location= y r))
409 (when (sc-is x ,constant-sc)
410 (setf x (get-constant x t)))
412 ((not (location= r y))
413 (if (sc-is x ,constant-sc)
414 (inst ,load-inst r (get-constant x))
416 (when (sc-is y ,constant-sc)
417 (setf y (get-constant y t)))
420 (if (sc-is x ,constant-sc)
421 (inst ,load-inst tmp (get-constant x))
425 (frob (op sinst sname scost dinst dname dcost commutative
426 &optional csinst csname cscost cdinst cdname cdcost)
428 (define-vop (,sname ,(if commutative
429 'single-float-comm-op
432 (:temporary (:sc single-reg) tmp)
434 (generate ,sinst ,commutative fp-single-immediate movss)))
435 (define-vop (,dname ,(if commutative
436 'double-float-comm-op
439 (:temporary (:sc double-reg) tmp)
441 (generate ,dinst ,commutative fp-double-immediate movsd)))
443 `(define-vop (,csname
445 'complex-single-float-comm-op
446 'complex-single-float-op))
448 (:temporary (:sc complex-single-reg) tmp)
450 (generate ,csinst ,commutative
451 fp-complex-single-immediate movq))))
453 `(define-vop (,cdname
455 'complex-double-float-comm-op
456 'complex-double-float-op))
458 (:temporary (:sc complex-double-reg) tmp)
460 (generate ,cdinst ,commutative
461 fp-complex-double-immediate movapd)))))))
462 (frob + addss +/single-float 2 addsd +/double-float 2 t
463 addps +/complex-single-float 3 addpd +/complex-double-float 3)
464 (frob - subss -/single-float 2 subsd -/double-float 2 nil
465 subps -/complex-single-float 3 subpd -/complex-double-float 3)
466 (frob * mulss */single-float 4 mulsd */double-float 5 t)
467 (frob / divss //single-float 12 divsd //double-float 19 nil))
469 (macrolet ((frob (op cost commutativep
470 duplicate-inst op-inst real-move-inst complex-move-inst
471 real-sc real-constant-sc real-type
472 complex-sc complex-constant-sc complex-type
473 real-complex-name complex-real-name)
474 (cond ((not duplicate-inst) ; simple case
475 `(flet ((load-into (r x)
478 (inst ,real-move-inst r
479 (register-inline-constant (tn-value x))))
480 (,complex-constant-sc
481 (inst ,complex-move-inst r
482 (register-inline-constant (tn-value x))))
484 ,(when real-complex-name
485 `(define-vop (,real-complex-name float-op)
487 (:args (x :scs (,real-sc ,real-constant-sc)
489 :load-if (not (sc-is x ,real-constant-sc)))
490 (y :scs (,complex-sc ,complex-constant-sc)
491 ,@(when commutativep '(:target r))
492 :load-if (not (sc-is y ,complex-constant-sc))))
493 (:arg-types ,real-type ,complex-type)
494 (:results (r :scs (,complex-sc)
495 ,@(unless commutativep '(:from (:argument 0)))))
496 (:result-types ,complex-type)
499 `(when (location= y r)
502 (when (sc-is y ,real-constant-sc ,complex-constant-sc)
503 (setf y (register-inline-constant
504 :aligned (tn-value y))))
505 (inst ,op-inst r y))))
507 ,(when complex-real-name
508 `(define-vop (,complex-real-name float-op)
510 (:args (x :scs (,complex-sc ,complex-constant-sc)
512 :load-if (not (sc-is x ,complex-constant-sc)))
513 (y :scs (,real-sc ,real-constant-sc)
514 ,@(when commutativep '(:target r))
515 :load-if (not (sc-is y ,real-constant-sc))))
516 (:arg-types ,complex-type ,real-type)
517 (:results (r :scs (,complex-sc)
518 ,@(unless commutativep '(:from (:argument 0)))))
519 (:result-types ,complex-type)
522 `(when (location= y r)
525 (when (sc-is y ,real-constant-sc ,complex-constant-sc)
526 (setf y (register-inline-constant
527 :aligned (tn-value y))))
528 (inst ,op-inst r y))))))
529 (commutativep ; must duplicate, but commutative
531 ,(when real-complex-name
532 `(define-vop (,real-complex-name float-op)
534 (:args (x :scs (,real-sc ,real-constant-sc)
536 :load-if (not (sc-is x ,real-constant-sc)))
537 (y :scs (,complex-sc ,complex-constant-sc)
540 :load-if (not (sc-is y ,complex-constant-sc))))
541 (:arg-types ,real-type ,complex-type)
542 (:temporary (:sc ,complex-sc :target r
546 (:results (r :scs (,complex-sc)))
547 (:result-types ,complex-type)
549 (if (sc-is x ,real-constant-sc)
550 (inst ,complex-move-inst dup
551 (register-inline-constant
552 (complex (tn-value x) (tn-value x))))
556 (when (location= dup r)
558 (if (sc-is y ,complex-constant-sc)
559 (inst ,complex-move-inst r
560 (register-inline-constant (tn-value y)))
562 (when (sc-is dup ,complex-constant-sc)
563 (setf dup (register-inline-constant
564 :aligned (tn-value dup))))
565 (inst ,op-inst r dup))))
567 ,(when complex-real-name
568 `(define-vop (,complex-real-name float-op)
570 (:args (x :scs (,complex-sc ,complex-constant-sc)
573 :load-if (not (sc-is x ,complex-constant-sc)))
574 (y :scs (,real-sc ,real-constant-sc)
576 :load-if (not (sc-is y ,real-constant-sc))))
577 (:arg-types ,complex-type ,real-type)
578 (:temporary (:sc ,complex-sc :target r
582 (:results (r :scs (,complex-sc)))
583 (:result-types ,complex-type)
585 (if (sc-is y ,real-constant-sc)
586 (inst ,complex-move-inst dup
587 (register-inline-constant
588 (complex (tn-value y) (tn-value y))))
591 (when (location= dup r)
593 (if (sc-is x ,complex-constant-sc)
594 (inst ,complex-move-inst r
595 (register-inline-constant (tn-value x)))
597 (when (sc-is dup ,complex-constant-sc)
598 (setf dup (register-inline-constant
599 :aligned (tn-value dup))))
600 (inst ,op-inst r dup))))))
601 (t ; duplicate, not commutative
603 ,(when real-complex-name
604 `(define-vop (,real-complex-name float-op)
606 (:args (x :scs (,real-sc ,real-constant-sc)
608 :load-if (not (sc-is x ,real-constant-sc)))
609 (y :scs (,complex-sc ,complex-constant-sc)
611 :load-if (not (sc-is y ,complex-constant-sc))))
612 (:arg-types ,real-type ,complex-type)
613 (:results (r :scs (,complex-sc) :from (:argument 0)))
614 (:result-types ,complex-type)
616 (if (sc-is x ,real-constant-sc)
617 (inst ,complex-move-inst dup
618 (register-inline-constant
619 (complex (tn-value x) (tn-value x))))
623 (when (sc-is y ,complex-constant-sc)
624 (setf y (register-inline-constant
625 :aligned (tn-value y))))
626 (inst ,op-inst r y))))
628 ,(when complex-real-name
629 `(define-vop (,complex-real-name float-op)
631 (:args (x :scs (,complex-sc)
634 (y :scs (,real-sc ,real-constant-sc)
636 :load-if (not (sc-is y ,complex-constant-sc))))
637 (:arg-types ,complex-type ,real-type)
638 (:temporary (:sc ,complex-sc :from (:argument 1))
640 (:results (r :scs (,complex-sc) :from :eval))
641 (:result-types ,complex-type)
643 (if (sc-is y ,real-constant-sc)
644 (setf dup (register-inline-constant
645 :aligned (complex (tn-value y)
650 (inst ,op-inst r dup))))))))
651 (def-real-complex-op (op commutativep duplicatep
652 single-inst single-real-complex-name single-complex-real-name single-cost
653 double-inst double-real-complex-name double-complex-real-name double-cost)
655 (frob ,op ,single-cost ,commutativep
659 (inst unpcklps dup dup)))
660 ,single-inst movss movq
661 single-reg fp-single-immediate single-float
662 complex-single-reg fp-complex-single-immediate complex-single-float
663 ,single-real-complex-name ,single-complex-real-name)
664 (frob ,op ,double-cost ,commutativep
668 (inst unpcklpd dup dup)))
669 ,double-inst movsd movapd
670 double-reg fp-double-immediate double-float
671 complex-double-reg fp-complex-double-immediate complex-double-float
672 ,double-real-complex-name ,double-complex-real-name))))
673 (def-real-complex-op + t nil
674 addps +/real-complex-single-float +/complex-real-single-float 3
675 addpd +/real-complex-double-float +/complex-real-double-float 4)
676 (def-real-complex-op - nil nil
677 subps -/real-complex-single-float -/complex-real-single-float 3
678 subpd -/real-complex-double-float -/complex-real-double-float 4)
679 (def-real-complex-op * t t
680 mulps */real-complex-single-float */complex-real-single-float 4
681 mulpd */real-complex-double-float */complex-real-double-float 5)
682 (def-real-complex-op / nil t
684 divpd nil //complex-real-double-float 19))
686 (define-vop (//complex-real-single-float float-op)
688 (:args (x :scs (complex-single-reg fp-complex-single-immediate fp-complex-single-zero)
691 :load-if (not (sc-is x fp-complex-single-immediate fp-complex-single-zero)))
692 (y :scs (single-reg fp-single-immediate fp-single-zero)
694 :load-if (not (sc-is y fp-single-immediate fp-single-zero))))
695 (:arg-types complex-single-float single-float)
696 (:temporary (:sc complex-single-reg :from (:argument 1)) dup)
697 (:results (r :scs (complex-single-reg)))
698 (:result-types complex-single-float)
700 (flet ((duplicate (x)
701 (let ((word (ldb (byte 64 0)
702 (logior (ash (single-float-bits (imagpart x)) 32)
704 (single-float-bits (realpart x)))))))
705 (register-inline-constant :oword (logior (ash word 64) word)))))
708 (setf dup (duplicate (complex (tn-value y) (tn-value y)))))
710 (inst xorps dup dup))
712 (inst shufps dup dup #b00000000)))
714 (fp-complex-single-immediate
715 (inst movaps r (duplicate (tn-value x))))
716 (fp-complex-single-zero
720 (inst unpcklpd r r)))
724 ;; Complex multiplication
725 ;; r := rx * ry - ix * iy
726 ;; i := rx * iy + ix * ry
728 ;; Transpose for SIMDness
733 ;;+ [ix ix] * [-iy ry]
736 (macrolet ((define-complex-* (name cost type sc tmp-p &body body)
737 `(define-vop (,name float-op)
739 (:args (x :scs (,sc) :target r)
740 (y :scs (,sc) :target copy-y))
741 (:arg-types ,type ,type)
742 (:temporary (:sc ,sc) imag)
743 (:temporary (:sc ,sc :from :eval) copy-y)
745 `((:temporary (:sc ,sc) xmm)))
746 (:results (r :scs (,sc) :from :eval))
747 (:result-types ,type)
749 (when (or (location= x copy-y)
753 (define-complex-* */complex-single-float 20
754 complex-single-float complex-single-reg t
759 (inst unpckhpd imag xmm)
760 (inst unpcklpd r xmm)
761 (move copy-y y) ; y == r only if y == x == r
766 (inst shufps y y #b11110001)
767 (inst xorps y (register-inline-constant :oword (ash 1 31)))
771 (define-complex-* */complex-double-float 25
772 complex-double-float complex-double-reg nil
778 (inst unpckhpd imag imag)
782 (inst shufpd y y #b01)
783 (inst xorpd y (register-inline-constant :oword (ash 1 63)))
786 (inst addpd r imag)))
789 (:args (x :scs (double-reg)))
790 (:results (y :scs (double-reg)))
793 (:arg-types double-float)
794 (:result-types double-float)
795 (:note "inline float arithmetic")
797 (:save-p :compute-only)
799 (unless (location= x y)
801 (note-this-location vop :internal-error)
804 (macrolet ((frob ((name translate sc type) &body body)
806 (:args (x :scs (,sc) :target y))
807 (:results (y :scs (,sc)))
808 (:translate ,translate)
811 (:result-types ,type)
812 (:note "inline float arithmetic")
814 (:save-p :compute-only)
816 (note-this-location vop :internal-error)
819 (frob (%negate/double-float %negate double-reg double-float)
820 (inst xorpd y (register-inline-constant :oword (ash 1 63))))
821 (frob (%negate/complex-double-float %negate complex-double-reg complex-double-float)
822 (inst xorpd y (register-inline-constant
823 :oword (logior (ash 1 127) (ash 1 63)))))
824 (frob (conjugate/complex-double-float conjugate complex-double-reg complex-double-float)
825 (inst xorpd y (register-inline-constant :oword (ash 1 127))))
826 (frob (%negate/single-float %negate single-reg single-float)
827 (inst xorps y (register-inline-constant :oword (ash 1 31))))
828 (frob (%negate/complex-single-float %negate complex-single-reg complex-single-float)
829 (inst xorps y (register-inline-constant
830 :oword (logior (ash 1 31) (ash 1 63)))))
831 (frob (conjugate/complex-single-float conjugate complex-single-reg complex-single-float)
832 (inst xorpd y (register-inline-constant :oword (ash 1 63))))
833 (frob (abs/double-float abs double-reg double-float)
834 (inst andpd y (register-inline-constant :oword (ldb (byte 63 0) -1))))
835 (frob (abs/single-float abs single-reg single-float)
836 (inst andps y (register-inline-constant :oword (ldb (byte 31 0) -1)))))
841 (define-vop (float-compare)
844 (:save-p :compute-only)
845 (:note "inline float comparison"))
848 (macrolet ((define-float-eql (name cost sc constant-sc type)
849 `(define-vop (,name float-compare)
851 (:args (x :scs (,sc ,constant-sc)
853 :load-if (not (sc-is x ,constant-sc)))
854 (y :scs (,sc ,constant-sc)
856 :load-if (not (sc-is y ,constant-sc))))
857 (:arg-types ,type ,type)
858 (:temporary (:sc ,sc :from :eval) mask)
859 (:temporary (:sc any-reg) bits)
862 (when (or (location= y mask)
863 (not (xmm-register-p x)))
865 (aver (xmm-register-p x))
867 (when (sc-is y ,constant-sc)
868 (setf y (register-inline-constant :aligned (tn-value y))))
869 (inst pcmpeqd mask y)
870 (inst movmskps bits mask)
871 (inst cmp bits #b1111)))))
872 (define-float-eql eql/single-float 4
873 single-reg fp-single-immediate single-float)
874 (define-float-eql eql/double-float 4
875 double-reg fp-double-immediate double-float)
876 (define-float-eql eql/complex-single-float 5
877 complex-single-reg fp-complex-single-immediate complex-single-float)
878 (define-float-eql eql/complex-double-float 5
879 complex-double-reg fp-complex-double-immediate complex-double-float))
881 ;;; comiss and comisd can cope with one or other arg in memory: we
882 ;;; could (should, indeed) extend these to cope with descriptor args
885 (define-vop (single-float-compare float-compare)
886 (:args (x :scs (single-reg))
887 (y :scs (single-reg single-stack fp-single-immediate)
888 :load-if (not (sc-is y single-stack fp-single-immediate))))
889 (:arg-types single-float single-float))
890 (define-vop (double-float-compare float-compare)
891 (:args (x :scs (double-reg))
892 (y :scs (double-reg double-stack descriptor-reg fp-double-immediate)
893 :load-if (not (sc-is y double-stack descriptor-reg fp-double-immediate))))
894 (:arg-types double-float double-float))
896 (define-vop (=/single-float single-float-compare)
898 (:args (x :scs (single-reg single-stack fp-single-immediate)
900 :load-if (not (sc-is x single-stack fp-single-immediate)))
901 (y :scs (single-reg single-stack fp-single-immediate)
903 :load-if (not (sc-is y single-stack fp-single-immediate))))
904 (:temporary (:sc single-reg :from :eval) xmm)
906 (:conditional not :p :ne)
909 (when (or (location= y xmm)
910 (and (not (xmm-register-p x))
914 (single-reg (setf xmm x))
915 (single-stack (inst movss xmm (ea-for-sf-stack x)))
917 (inst movss xmm (register-inline-constant (tn-value x)))))
920 (setf y (ea-for-sf-stack y)))
922 (setf y (register-inline-constant (tn-value y))))
924 (note-this-location vop :internal-error)
926 ;; if PF&CF, there was a NaN involved => not equal
927 ;; otherwise, ZF => equal
930 (define-vop (=/double-float double-float-compare)
932 (:args (x :scs (double-reg double-stack fp-double-immediate descriptor-reg)
934 :load-if (not (sc-is x double-stack fp-double-immediate descriptor-reg)))
935 (y :scs (double-reg double-stack fp-double-immediate descriptor-reg)
937 :load-if (not (sc-is y double-stack fp-double-immediate descriptor-reg))))
938 (:temporary (:sc double-reg :from :eval) xmm)
940 (:conditional not :p :ne)
943 (when (or (location= y xmm)
944 (and (not (xmm-register-p x))
951 (inst movsd xmm (ea-for-df-stack x)))
953 (inst movsd xmm (register-inline-constant (tn-value x))))
955 (inst movsd xmm (ea-for-df-desc x))))
958 (setf y (ea-for-df-stack y)))
960 (setf y (register-inline-constant (tn-value y))))
962 (setf y (ea-for-df-desc y)))
964 (note-this-location vop :internal-error)
965 (inst comisd xmm y)))
967 (macrolet ((define-complex-float-= (complex-complex-name complex-real-name real-complex-name
968 real-sc real-constant-sc real-type
969 complex-sc complex-constant-sc complex-type
970 real-move-inst complex-move-inst
971 cmp-inst mask-inst mask)
973 (define-vop (,complex-complex-name float-compare)
975 (:args (x :scs (,complex-sc ,complex-constant-sc)
977 :load-if (not (sc-is x ,complex-constant-sc)))
978 (y :scs (,complex-sc ,complex-constant-sc)
980 :load-if (not (sc-is y ,complex-constant-sc))))
981 (:arg-types ,complex-type ,complex-type)
982 (:temporary (:sc ,complex-sc :from :eval) cmp)
983 (:temporary (:sc unsigned-reg) bits)
987 (when (location= y cmp)
991 (inst ,real-move-inst cmp (register-inline-constant
993 (,complex-constant-sc
994 (inst ,complex-move-inst cmp (register-inline-constant
998 (when (sc-is y ,real-constant-sc ,complex-constant-sc)
999 (setf y (register-inline-constant :aligned (tn-value y))))
1000 (note-this-location vop :internal-error)
1001 (inst ,cmp-inst :eq cmp y)
1002 (inst ,mask-inst bits cmp)
1003 (inst cmp bits ,mask)))
1004 (define-vop (,complex-real-name ,complex-complex-name)
1005 (:args (x :scs (,complex-sc ,complex-constant-sc)
1007 :load-if (not (sc-is x ,complex-constant-sc)))
1008 (y :scs (,real-sc ,real-constant-sc)
1010 :load-if (not (sc-is y ,real-constant-sc))))
1011 (:arg-types ,complex-type ,real-type))
1012 (define-vop (,real-complex-name ,complex-complex-name)
1013 (:args (x :scs (,real-sc ,real-constant-sc)
1015 :load-if (not (sc-is x ,real-constant-sc)))
1016 (y :scs (,complex-sc ,complex-constant-sc)
1018 :load-if (not (sc-is y ,complex-constant-sc))))
1019 (:arg-types ,real-type ,complex-type)))))
1020 (define-complex-float-= =/complex-single-float =/complex-real-single-float =/real-complex-single-float
1021 single-reg fp-single-immediate single-float
1022 complex-single-reg fp-complex-single-immediate complex-single-float
1023 movss movq cmpps movmskps #b1111)
1024 (define-complex-float-= =/complex-double-float =/complex-real-double-float =/real-complex-double-float
1025 double-reg fp-double-immediate double-float
1026 complex-double-reg fp-complex-double-immediate complex-double-float
1027 movsd movapd cmppd movmskpd #b11))
1029 (macrolet ((define-</> (op single-name double-name &rest flags)
1031 (define-vop (,double-name double-float-compare)
1034 (:conditional ,@flags)
1038 (setf y (ea-for-df-stack y)))
1040 (setf y (ea-for-df-desc y)))
1041 (fp-double-immediate
1042 (setf y (register-inline-constant (tn-value y))))
1045 (define-vop (,single-name single-float-compare)
1048 (:conditional ,@flags)
1052 (setf y (ea-for-sf-stack y)))
1053 (fp-single-immediate
1054 (setf y (register-inline-constant (tn-value y))))
1056 (inst comiss x y))))))
1057 (define-</> < <single-float <double-float not :p :nc)
1058 (define-</> > >single-float >double-float not :p :na))
1063 (macrolet ((frob (name translate inst to-sc to-type)
1064 `(define-vop (,name)
1065 (:args (x :scs (signed-stack signed-reg)))
1066 (:results (y :scs (,to-sc)))
1067 (:arg-types signed-num)
1068 (:result-types ,to-type)
1069 (:policy :fast-safe)
1070 (:note "inline float coercion")
1071 (:translate ,translate)
1073 (:save-p :compute-only)
1076 (single-reg (inst xorps y y))
1077 (double-reg (inst xorpd y y)))
1078 (note-this-location vop :internal-error)
1079 (inst ,inst y x)))))
1080 (frob %single-float/signed %single-float cvtsi2ss single-reg single-float)
1081 (frob %double-float/signed %double-float cvtsi2sd double-reg double-float))
1083 (macrolet ((frob (name translate inst from-scs from-type ea-func to-sc to-type)
1084 `(define-vop (,name)
1085 (:args (x :scs ,from-scs :target y))
1086 (:results (y :scs (,to-sc)))
1087 (:arg-types ,from-type)
1088 (:result-types ,to-type)
1089 (:policy :fast-safe)
1090 (:note "inline float coercion")
1091 (:translate ,translate)
1093 (:save-p :compute-only)
1095 (unless (location= x y)
1097 (single-reg (inst xorps y y))
1098 (double-reg (inst xorpd y y))))
1099 (note-this-location vop :internal-error)
1100 (inst ,inst y (sc-case x
1101 (,(first from-scs) x)
1102 (,(second from-scs) (,ea-func x))))
1103 ,(when (and (eq from-type 'double-float) ; if the input is wider
1104 (eq to-type 'single-float)) ; than the output, clear
1105 `(when (location= x y) ; noise in the high part
1106 (inst shufps y y #4r3330)))))))
1107 (frob %single-float/double-float %single-float cvtsd2ss
1108 (double-reg double-stack) double-float ea-for-df-stack
1109 single-reg single-float)
1111 (frob %double-float/single-float %double-float cvtss2sd
1112 (single-reg single-stack) single-float ea-for-sf-stack
1113 double-reg double-float))
1115 (macrolet ((frob (trans inst from-scs from-type ea-func)
1116 `(define-vop (,(symbolicate trans "/" from-type))
1117 (:args (x :scs ,from-scs))
1118 (:results (y :scs (signed-reg)))
1119 (:arg-types ,from-type)
1120 (:result-types signed-num)
1122 (:policy :fast-safe)
1123 (:note "inline float truncate")
1125 (:save-p :compute-only)
1127 (inst ,inst y (sc-case x
1128 (,(first from-scs) x)
1129 (,(second from-scs) (,ea-func x))))))))
1130 (frob %unary-truncate/single-float cvttss2si
1131 (single-reg single-stack) single-float ea-for-sf-stack)
1132 (frob %unary-truncate/double-float cvttsd2si
1133 (double-reg double-stack) double-float ea-for-df-stack)
1135 (frob %unary-round cvtss2si
1136 (single-reg single-stack) single-float ea-for-sf-stack)
1137 (frob %unary-round cvtsd2si
1138 (double-reg double-stack) double-float ea-for-df-stack))
1140 (define-vop (make-single-float)
1141 (:args (bits :scs (signed-reg) :target res
1142 :load-if (not (or (and (sc-is bits signed-stack)
1143 (sc-is res single-reg))
1144 (and (sc-is bits signed-stack)
1145 (sc-is res single-stack)
1146 (location= bits res))))))
1147 (:results (res :scs (single-reg single-stack)))
1148 (:arg-types signed-num)
1149 (:result-types single-float)
1150 (:translate make-single-float)
1151 (:policy :fast-safe)
1158 (inst mov res bits))
1160 (aver (location= bits res)))))
1164 (inst movd res bits))
1166 (inst movd res bits)))))))
1168 (define-vop (make-single-float-c)
1169 (:results (res :scs (single-reg single-stack descriptor-reg)))
1170 (:arg-types (:constant (signed-byte 32)))
1171 (:result-types single-float)
1173 (:translate make-single-float)
1174 (:policy :fast-safe)
1179 (inst mov res bits))
1181 (inst movss res (register-inline-constant :dword bits)))
1183 (inst mov res (logior (ash bits 32)
1184 single-float-widetag))))))
1186 (define-vop (make-double-float)
1187 (:args (hi-bits :scs (signed-reg))
1188 (lo-bits :scs (unsigned-reg)))
1189 (:results (res :scs (double-reg)))
1190 (:temporary (:sc unsigned-reg) temp)
1191 (:arg-types signed-num unsigned-num)
1192 (:result-types double-float)
1193 (:translate make-double-float)
1194 (:policy :fast-safe)
1199 (inst or temp lo-bits)
1200 (inst movd res temp)))
1202 (define-vop (make-double-float-c)
1203 (:results (res :scs (double-reg)))
1204 (:arg-types (:constant (signed-byte 32)) (:constant (unsigned-byte 32)))
1205 (:result-types double-float)
1207 (:translate make-double-float)
1208 (:policy :fast-safe)
1211 (inst movsd res (register-inline-constant :qword (logior (ash hi 32) lo)))))
1213 (define-vop (single-float-bits)
1214 (:args (float :scs (single-reg descriptor-reg)
1215 :load-if (not (sc-is float single-stack))))
1216 (:results (bits :scs (signed-reg)))
1217 (:arg-types single-float)
1218 (:result-types signed-num)
1219 (:translate single-float-bits)
1220 (:policy :fast-safe)
1224 (inst movd bits float)
1225 (inst movsxd bits (reg-in-size bits :dword)))
1227 (inst movsxd bits (make-ea :dword ; c.f. ea-for-sf-stack
1229 :disp (frame-byte-offset (tn-offset float)))))
1232 (inst sar bits 32)))))
1234 (define-vop (double-float-high-bits)
1235 (:args (float :scs (double-reg descriptor-reg)
1236 :load-if (not (sc-is float double-stack))))
1237 (:results (hi-bits :scs (signed-reg)))
1238 (:temporary (:sc signed-stack :from :argument :to :result) temp)
1239 (:arg-types double-float)
1240 (:result-types signed-num)
1241 (:translate double-float-high-bits)
1242 (:policy :fast-safe)
1247 (inst movsd temp float)
1248 (move hi-bits temp))
1250 (loadw hi-bits ebp-tn (frame-word-offset (tn-offset float))))
1252 (loadw hi-bits float double-float-value-slot
1253 other-pointer-lowtag)))
1254 (inst sar hi-bits 32)))
1256 (define-vop (double-float-low-bits)
1257 (:args (float :scs (double-reg descriptor-reg)
1258 :load-if (not (sc-is float double-stack))))
1259 (:results (lo-bits :scs (unsigned-reg)))
1260 (:temporary (:sc signed-stack :from :argument :to :result) temp)
1261 (:arg-types double-float)
1262 (:result-types unsigned-num)
1263 (:translate double-float-low-bits)
1264 (:policy :fast-safe)
1269 (inst movsd temp float)
1270 (move lo-bits temp))
1272 (loadw lo-bits ebp-tn (frame-word-offset (tn-offset float))))
1274 (loadw lo-bits float double-float-value-slot
1275 other-pointer-lowtag)))
1276 (inst shl lo-bits 32)
1277 (inst shr lo-bits 32)))
1281 ;;;; complex float VOPs
1283 (define-vop (make-complex-single-float)
1284 (:translate complex)
1285 (:args (real :scs (single-reg fp-single-zero)
1287 :load-if (not (sc-is real fp-single-zero)))
1288 (imag :scs (single-reg fp-single-zero)
1289 :load-if (not (sc-is imag fp-single-zero))))
1290 (:arg-types single-float single-float)
1291 (:results (r :scs (complex-single-reg) :from (:argument 0)))
1292 (:result-types complex-single-float)
1293 (:note "inline complex single-float creation")
1294 (:policy :fast-safe)
1296 (cond ((sc-is real fp-single-zero)
1298 (unless (sc-is imag fp-single-zero)
1299 (inst unpcklps r imag)))
1300 ((location= real imag)
1302 (inst unpcklps r r))
1305 (unless (sc-is imag fp-single-zero)
1306 (inst unpcklps r imag))))))
1308 (define-vop (make-complex-double-float)
1309 (:translate complex)
1310 (:args (real :scs (double-reg fp-double-zero)
1312 :load-if (not (sc-is real fp-double-zero)))
1313 (imag :scs (double-reg fp-double-zero)
1314 :load-if (not (sc-is imag fp-double-zero))))
1315 (:arg-types double-float double-float)
1316 (:results (r :scs (complex-double-reg) :from (:argument 0)))
1317 (:result-types complex-double-float)
1318 (:note "inline complex double-float creation")
1319 (:policy :fast-safe)
1321 (cond ((sc-is real fp-double-zero)
1323 (unless (sc-is imag fp-double-zero)
1324 (inst unpcklpd r imag)))
1325 ((location= real imag)
1327 (inst unpcklpd r r))
1330 (unless (sc-is imag fp-double-zero)
1331 (inst unpcklpd r imag))))))
1333 (define-vop (complex-float-value)
1334 (:args (x :target r))
1335 (:temporary (:sc complex-double-reg) zero)
1337 (:variant-vars offset)
1338 (:policy :fast-safe)
1340 (cond ((sc-is x complex-double-reg)
1342 (inst xorpd zero zero)
1344 (0 (inst unpcklpd r zero))
1345 (1 (inst unpckhpd r zero))))
1346 ((sc-is x complex-single-reg)
1349 (0 (inst shufps r r #b11111100))
1350 (1 (inst shufps r r #b11111101))))
1351 ((sc-is r single-reg)
1352 (let ((ea (sc-case x
1353 (complex-single-stack
1355 (0 (ea-for-csf-real-stack x))
1356 (1 (ea-for-csf-imag-stack x))))
1359 (0 (ea-for-csf-real-desc x))
1360 (1 (ea-for-csf-imag-desc x)))))))
1362 ((sc-is r double-reg)
1363 (let ((ea (sc-case x
1364 (complex-double-stack
1366 (0 (ea-for-cdf-real-stack x))
1367 (1 (ea-for-cdf-imag-stack x))))
1370 (0 (ea-for-cdf-real-desc x))
1371 (1 (ea-for-cdf-imag-desc x)))))))
1373 (t (error "COMPLEX-FLOAT-VALUE VOP failure")))))
1375 (define-vop (realpart/complex-single-float complex-float-value)
1376 (:translate realpart)
1377 (:args (x :scs (complex-single-reg complex-single-stack descriptor-reg)
1379 (:arg-types complex-single-float)
1380 (:results (r :scs (single-reg)))
1381 (:result-types single-float)
1382 (:note "complex float realpart")
1385 (define-vop (realpart/complex-double-float complex-float-value)
1386 (:translate realpart)
1387 (:args (x :scs (complex-double-reg complex-double-stack descriptor-reg)
1389 (:arg-types complex-double-float)
1390 (:results (r :scs (double-reg)))
1391 (:result-types double-float)
1392 (:note "complex float realpart")
1395 (define-vop (imagpart/complex-single-float complex-float-value)
1396 (:translate imagpart)
1397 (:args (x :scs (complex-single-reg complex-single-stack descriptor-reg)
1399 (:arg-types complex-single-float)
1400 (:results (r :scs (single-reg)))
1401 (:result-types single-float)
1402 (:note "complex float imagpart")
1405 (define-vop (imagpart/complex-double-float complex-float-value)
1406 (:translate imagpart)
1407 (:args (x :scs (complex-double-reg complex-double-stack descriptor-reg)
1409 (:arg-types complex-double-float)
1410 (:results (r :scs (double-reg)))
1411 (:result-types double-float)
1412 (:note "complex float imagpart")
1416 ;;; hack dummy VOPs to bias the representation selection of their
1417 ;;; arguments towards a FP register, which can help avoid consing at
1418 ;;; inappropriate locations
1419 (defknown double-float-reg-bias (double-float) (values))
1420 (define-vop (double-float-reg-bias)
1421 (:translate double-float-reg-bias)
1422 (:args (x :scs (double-reg double-stack) :load-if nil))
1423 (:arg-types double-float)
1424 (:policy :fast-safe)
1425 (:note "inline dummy FP register bias")
1428 (defknown single-float-reg-bias (single-float) (values))
1429 (define-vop (single-float-reg-bias)
1430 (:translate single-float-reg-bias)
1431 (:args (x :scs (single-reg single-stack) :load-if nil))
1432 (:arg-types single-float)
1433 (:policy :fast-safe)
1434 (:note "inline dummy FP register bias")
1438 (defknown swap-complex ((complex float)) (complex float)
1439 (foldable flushable movable always-translatable))
1440 (defoptimizer (swap-complex derive-type) ((x))
1441 (sb!c::lvar-type x))
1442 (defun swap-complex (x)
1443 (complex (imagpart x) (realpart x)))
1444 (define-vop (swap-complex-single-float)
1445 (:translate swap-complex)
1446 (:policy :fast-safe)
1447 (:args (x :scs (complex-single-reg) :target r))
1448 (:arg-types complex-single-float)
1449 (:results (r :scs (complex-single-reg)))
1450 (:result-types complex-single-float)
1453 (inst shufps r r #b11110001)))
1454 (define-vop (swap-complex-double-float)
1455 (:translate swap-complex)
1456 (:policy :fast-safe)
1457 (:args (x :scs (complex-double-reg) :target r))
1458 (:arg-types complex-double-float)
1459 (:results (r :scs (complex-double-reg)))
1460 (:result-types complex-double-float)
1463 (inst shufpd r r #b01)))