1 ;;;; the x86-64 VM definition of operand loading/saving and the MOVE vop
3 ;;;; This software is part of the SBCL system. See the README file for
6 ;;;; This software is derived from the CMU CL system, which was
7 ;;;; written at Carnegie Mellon University and released into the
8 ;;;; public domain. The software is in the public domain and is
9 ;;;; provided with absolutely no warranty. See the COPYING and CREDITS
10 ;;;; files for more information.
14 (defun make-byte-tn (tn)
15 (aver (sc-is tn any-reg descriptor-reg unsigned-reg signed-reg))
16 (make-random-tn :kind :normal
17 :sc (sc-or-lose 'byte-reg)
18 :offset (tn-offset tn)))
20 (defun make-dword-tn (tn)
21 (aver (sc-is tn any-reg descriptor-reg character-reg
22 unsigned-reg signed-reg))
23 (make-random-tn :kind :normal
24 :sc (sc-or-lose 'dword-reg)
25 :offset (tn-offset tn)))
28 (let ((offset (tn-offset tn)))
29 ;; Using the 32-bit instruction accomplishes the same thing and is
31 (if (<= offset edi-offset)
32 (let ((tn (make-random-tn :kind :normal
33 :sc (sc-or-lose 'dword-reg)
38 (define-move-fun (load-immediate 1) (vop x y)
40 (any-reg descriptor-reg))
41 (let ((val (tn-value x)))
46 (inst mov y (fixnumize val))))
50 (inst mov y (logior (ash (char-code val) n-widetag-bits)
51 character-widetag))))))
53 (define-move-fun (load-number 1) (vop x y)
54 ((immediate) (signed-reg unsigned-reg))
55 (let ((val (tn-value x)))
60 (define-move-fun (load-character 1) (vop x y)
61 ((immediate) (character-reg))
62 (inst mov y (char-code (tn-value x))))
64 (define-move-fun (load-system-area-pointer 1) (vop x y)
65 ((immediate) (sap-reg))
66 (inst mov y (sap-int (tn-value x))))
68 (define-move-fun (load-constant 5) (vop x y)
69 ((constant) (descriptor-reg any-reg))
72 (define-move-fun (load-stack 5) (vop x y)
73 ((control-stack) (any-reg descriptor-reg)
74 (character-stack) (character-reg)
76 (signed-stack) (signed-reg)
77 (unsigned-stack) (unsigned-reg))
80 (define-move-fun (store-stack 5) (vop x y)
81 ((any-reg descriptor-reg) (control-stack)
82 (character-reg) (character-stack)
84 (signed-reg) (signed-stack)
85 (unsigned-reg) (unsigned-stack))
90 (:args (x :scs (any-reg descriptor-reg immediate) :target y
91 :load-if (not (location= x y))))
92 (:results (y :scs (any-reg descriptor-reg)
94 (not (or (location= x y)
95 (and (sc-is x any-reg descriptor-reg immediate)
96 (sc-is y control-stack))))))
97 (:temporary (:sc unsigned-reg) temp)
101 (if (and (sc-is x immediate)
102 (sc-is y any-reg descriptor-reg control-stack))
103 (let ((val (tn-value x)))
106 (move-immediate y (fixnumize val) temp))
108 (inst mov y (+ nil-value (static-symbol-offset val))))
110 (inst mov y (logior (ash (char-code val) n-widetag-bits)
111 character-widetag)))))
114 (define-move-vop move :move
115 (any-reg descriptor-reg immediate)
116 (any-reg descriptor-reg))
118 ;;; Make MOVE the check VOP for T so that type check generation
119 ;;; doesn't think it is a hairy type. This also allows checking of a
120 ;;; few of the values in a continuation to fall out.
121 (primitive-type-vop move (:check) t)
123 (defun move-immediate (target val &optional tmp-tn)
125 ;; If target is a register, we can just mov it there directly
127 (sc-is target signed-reg unsigned-reg descriptor-reg any-reg))
130 (inst mov target val)))
131 ;; Likewise if the value is small enough.
132 ((typep val '(signed-byte 32))
133 (inst mov target val))
134 ;; Otherwise go through the temporary register
136 (inst mov tmp-tn val)
137 (inst mov target tmp-tn))
139 (error "~A is not a register, no temporary given, and immediate ~A too large" target val))))
141 ;;; The MOVE-ARG VOP is used for moving descriptor values into
142 ;;; another frame for argument or known value passing.
144 ;;; Note: It is not going to be possible to move a constant directly
145 ;;; to another frame, except if the destination is a register and in
146 ;;; this case the loading works out.
147 (define-vop (move-arg)
148 (:args (x :scs (any-reg descriptor-reg immediate) :target y
149 :load-if (not (and (sc-is y any-reg descriptor-reg)
150 (sc-is x control-stack))))
152 :load-if (not (sc-is y any-reg descriptor-reg))))
156 ((any-reg descriptor-reg)
157 (if (sc-is x immediate)
158 (let ((val (tn-value x)))
163 (inst mov y (fixnumize val)))
167 (inst mov y (logior (ash (char-code val) n-widetag-bits)
168 character-widetag)))))
171 (if (sc-is x immediate)
172 (let ((val (tn-value x)))
173 (if (= (tn-offset fp) esp-offset)
177 (storew (fixnumize val) fp (tn-offset y)))
179 (storew (+ nil-value (static-symbol-offset val))
182 (storew (logior (ash (char-code val) n-widetag-bits)
188 (storew (fixnumize val) fp (frame-word-offset (tn-offset y))))
190 (storew (+ nil-value (static-symbol-offset val))
191 fp (frame-word-offset (tn-offset y))))
193 (storew (logior (ash (char-code val) n-widetag-bits)
195 fp (frame-word-offset (tn-offset y)))))))
196 (if (= (tn-offset fp) esp-offset)
198 (storew x fp (tn-offset y))
200 (storew x fp (frame-word-offset (tn-offset y)))))))))
202 (define-move-vop move-arg :move-arg
203 (any-reg descriptor-reg)
204 (any-reg descriptor-reg))
208 ;;; This VOP exists just to begin the lifetime of a TN that couldn't
209 ;;; be written legally due to a type error. An error is signalled
210 ;;; before this VOP is so we don't need to do anything (not that there
211 ;;; would be anything sensible to do anyway.)
212 (define-vop (illegal-move)
217 (:save-p :compute-only)
219 (error-call vop 'object-not-type-error x type)))
221 ;;;; moves and coercions
223 ;;; These MOVE-TO-WORD VOPs move a tagged integer to a raw full-word
224 ;;; representation. Similarly, the MOVE-FROM-WORD VOPs converts a raw
225 ;;; integer to a tagged bignum or fixnum.
227 ;;; Arg is a fixnum, so just shift it. We need a type restriction
228 ;;; because some possible arg SCs (control-stack) overlap with
229 ;;; possible bignum arg SCs.
230 (define-vop (move-to-word/fixnum)
231 (:args (x :scs (any-reg descriptor-reg) :target y
232 :load-if (not (location= x y))))
233 (:results (y :scs (signed-reg unsigned-reg)
234 :load-if (not (location= x y))))
235 (:arg-types tagged-num)
236 (:note "fixnum untagging")
239 (inst sar y n-fixnum-tag-bits)))
240 (define-move-vop move-to-word/fixnum :move
241 (any-reg descriptor-reg) (signed-reg unsigned-reg))
243 ;;; Arg is a non-immediate constant, load it.
244 (define-vop (move-to-word-c)
245 (:args (x :scs (constant)))
246 (:results (y :scs (signed-reg unsigned-reg)))
247 (:note "constant load")
249 (cond ((sb!c::tn-leaf x)
250 (inst mov y (tn-value x)))
253 (inst sar y n-fixnum-tag-bits)))))
254 (define-move-vop move-to-word-c :move
255 (constant) (signed-reg unsigned-reg))
258 ;;; Arg is a fixnum or bignum, figure out which and load if necessary.
259 #-#.(cl:if (cl:= sb!vm:n-fixnum-tag-bits 1) '(:and) '(:or))
260 (define-vop (move-to-word/integer)
261 (:args (x :scs (descriptor-reg) :target rax))
262 (:results (y :scs (signed-reg unsigned-reg)))
263 (:note "integer to untagged word coercion")
264 ;; I'm not convinced that increasing the demand for rAX is
265 ;; better than adding 1 byte to some instruction encodings.
266 ;; I'll leave it alone though.
267 (:temporary (:sc unsigned-reg :offset rax-offset
268 :from (:argument 0) :to (:result 0) :target y) rax)
271 (inst test al-tn fixnum-tag-mask)
273 (loadw y rax bignum-digits-offset other-pointer-lowtag)
276 (inst sar rax n-fixnum-tag-bits)
280 #+#.(cl:if (cl:= sb!vm:n-fixnum-tag-bits 1) '(:and) '(:or))
281 (define-vop (move-to-word/integer)
282 (:args (x :scs (descriptor-reg) :target y))
283 (:results (y :scs (signed-reg unsigned-reg)))
284 (:note "integer to untagged word coercion")
285 (:temporary (:sc unsigned-reg) backup)
289 ;; It would be great if a principled way existed to advise GC of
290 ;; algebraic transforms such as 2*R being a conservative root.
291 ;; Until that is possible, emit straightforward code that uses
292 ;; a copy of the potential reference.
295 (inst sar y 1) ; optimistically assume it's a fixnum
296 (inst jmp :nc DONE) ; no carry implies tag was 0
297 (loadw y backup bignum-digits-offset other-pointer-lowtag)
300 (define-move-vop move-to-word/integer :move
301 (descriptor-reg) (signed-reg unsigned-reg))
303 ;;; Result is a fixnum, so we can just shift. We need the result type
304 ;;; restriction because of the control-stack ambiguity noted above.
305 (define-vop (move-from-word/fixnum)
306 (:args (x :scs (signed-reg unsigned-reg) :target y
307 :load-if (not (location= x y))))
308 (:results (y :scs (any-reg descriptor-reg)
309 :load-if (not (location= x y))))
310 (:result-types tagged-num)
311 (:note "fixnum tagging")
313 (cond ((and (sc-is x signed-reg unsigned-reg)
314 (not (location= x y)))
315 (if (= n-fixnum-tag-bits 1)
316 (inst lea y (make-ea :qword :base x :index x))
317 (inst lea y (make-ea :qword :index x
318 :scale (ash 1 n-fixnum-tag-bits)))))
320 ;; Uses: If x is a reg 2 + 3; if x = y uses only 3 bytes
322 (inst shl y n-fixnum-tag-bits)))))
323 (define-move-vop move-from-word/fixnum :move
324 (signed-reg unsigned-reg) (any-reg descriptor-reg))
326 ;;; Convert an untagged signed word to a lispobj -- fixnum or bignum
327 ;;; as the case may be. Fixnum case inline, bignum case in an assembly
329 (define-vop (move-from-signed)
330 (:args (x :scs (signed-reg unsigned-reg) :to :result . #.(and (= 1 n-fixnum-tag-bits)
332 (:results (y :scs (any-reg descriptor-reg) . #.(and (> n-fixnum-tag-bits 1)
333 '(:from :argument))))
334 (:note "signed word to integer coercion")
335 ;; Worst case cost to make sure people know they may be number consing.
337 (cond ((= 1 n-fixnum-tag-bits)
342 (inst rcr y 1) ; we're about to cons a bignum. this RCR is noise
345 (aver (not (location= x y)))
346 (inst imul y x #.(ash 1 n-fixnum-tag-bits))
349 (inst mov temp-reg-tn
350 (make-fixup (ecase (tn-offset y)
351 (#.rax-offset 'alloc-signed-bignum-in-rax)
352 (#.rcx-offset 'alloc-signed-bignum-in-rcx)
353 (#.rdx-offset 'alloc-signed-bignum-in-rdx)
354 (#.rbx-offset 'alloc-signed-bignum-in-rbx)
355 (#.rsi-offset 'alloc-signed-bignum-in-rsi)
356 (#.rdi-offset 'alloc-signed-bignum-in-rdi)
357 (#.r8-offset 'alloc-signed-bignum-in-r8)
358 (#.r9-offset 'alloc-signed-bignum-in-r9)
359 (#.r10-offset 'alloc-signed-bignum-in-r10)
360 (#.r12-offset 'alloc-signed-bignum-in-r12)
361 (#.r13-offset 'alloc-signed-bignum-in-r13)
362 (#.r14-offset 'alloc-signed-bignum-in-r14)
363 (#.r15-offset 'alloc-signed-bignum-in-r15))
365 (inst call temp-reg-tn)
367 (define-move-vop move-from-signed :move
368 (signed-reg) (descriptor-reg))
370 ;;; Convert an untagged unsigned word to a lispobj -- fixnum or bignum
371 ;;; as the case may be. Fixnum case inline, bignum case in an assembly
373 (define-vop (move-from-unsigned)
374 (:args (x :scs (signed-reg unsigned-reg) :to :result))
375 (:results (y :scs (any-reg descriptor-reg) :from :argument))
376 (:note "unsigned word to integer coercion")
377 ;; Worst case cost to make sure people know they may be number consing.
379 (aver (not (location= x y)))
380 (let ((done (gen-label)))
381 (inst mov y #.(ash (1- (ash 1 (1+ n-fixnum-tag-bits)))
382 n-positive-fixnum-bits))
383 ;; The assembly routines test the sign flag from this one, so if
384 ;; you change stuff here, make sure the sign flag doesn't get
385 ;; overwritten before the CALL!
387 ;; Using LEA is faster but bigger than MOV+SHL; it also doesn't
388 ;; twiddle the sign flag. The cost of doing this speculatively
389 ;; should be noise compared to bignum consing if that is needed
390 ;; and saves one branch.
391 (if (= n-fixnum-tag-bits 1)
392 (inst lea y (make-ea :qword :base x :index x))
393 (inst lea y (make-ea :qword :index x
394 :scale (ash 1 n-fixnum-tag-bits))))
397 (inst mov temp-reg-tn
398 (make-fixup (ecase (tn-offset y)
399 (#.rax-offset 'alloc-unsigned-bignum-in-rax)
400 (#.rcx-offset 'alloc-unsigned-bignum-in-rcx)
401 (#.rdx-offset 'alloc-unsigned-bignum-in-rdx)
402 (#.rbx-offset 'alloc-unsigned-bignum-in-rbx)
403 (#.rsi-offset 'alloc-unsigned-bignum-in-rsi)
404 (#.rdi-offset 'alloc-unsigned-bignum-in-rdi)
405 (#.r8-offset 'alloc-unsigned-bignum-in-r8)
406 (#.r9-offset 'alloc-unsigned-bignum-in-r9)
407 (#.r10-offset 'alloc-unsigned-bignum-in-r10)
408 (#.r12-offset 'alloc-unsigned-bignum-in-r12)
409 (#.r13-offset 'alloc-unsigned-bignum-in-r13)
410 (#.r14-offset 'alloc-unsigned-bignum-in-r14)
411 (#.r15-offset 'alloc-unsigned-bignum-in-r15))
413 (inst call temp-reg-tn)
415 (define-move-vop move-from-unsigned :move
416 (unsigned-reg) (descriptor-reg))
418 ;;; Move untagged numbers.
419 (define-vop (word-move)
420 (:args (x :scs (signed-reg unsigned-reg) :target y
421 :load-if (not (location= x y))))
422 (:results (y :scs (signed-reg unsigned-reg)
424 (not (or (location= x y)
425 (and (sc-is x signed-reg unsigned-reg)
426 (sc-is y signed-stack unsigned-stack))))))
429 (:note "word integer move")
432 (define-move-vop word-move :move
433 (signed-reg unsigned-reg) (signed-reg unsigned-reg))
435 ;;; Move untagged number arguments/return-values.
436 (define-vop (move-word-arg)
437 (:args (x :scs (signed-reg unsigned-reg) :target y)
438 (fp :scs (any-reg) :load-if (not (sc-is y sap-reg))))
440 (:note "word integer argument move")
443 ((signed-reg unsigned-reg)
445 ((signed-stack unsigned-stack)
446 (if (= (tn-offset fp) esp-offset)
447 (storew x fp (tn-offset y)) ; c-call
448 (storew x fp (frame-word-offset (tn-offset y))))))))
449 (define-move-vop move-word-arg :move-arg
450 (descriptor-reg any-reg signed-reg unsigned-reg) (signed-reg unsigned-reg))
452 ;;; Use standard MOVE-ARG and coercion to move an untagged number
453 ;;; to a descriptor passing location.
454 (define-move-vop move-arg :move-arg
455 (signed-reg unsigned-reg) (any-reg descriptor-reg))