X-Git-Url: http://repo.macrolet.net/gitweb/?a=blobdiff_plain;ds=sidebyside;f=src%2Fcompiler%2Fx86-64%2Fcall.lisp;h=4c4fdab84d4715e56d81f4d2ae3cd304dc16d911;hb=debae3c18d31b5222be4d5de8dcb2601336e24a4;hp=e6085295f5d7dff42ed3c9384259ce2cc0dec9cd;hpb=2c3112ebb0945849876108dccfdedf4c19678ea9;p=sbcl.git diff --git a/src/compiler/x86-64/call.lisp b/src/compiler/x86-64/call.lisp index e608529..4c4fdab 100644 --- a/src/compiler/x86-64/call.lisp +++ b/src/compiler/x86-64/call.lisp @@ -15,7 +15,7 @@ ;;; Return a wired TN describing the N'th full call argument passing ;;; location. -(!def-vm-support-routine standard-arg-location (n) +(defun standard-arg-location (n) (declare (type unsigned-byte n)) (if (< n register-arg-count) (make-wired-tn *backend-t-primitive-type* descriptor-reg-sc-number @@ -26,7 +26,7 @@ ;;; ;;; Always wire the return PC location to the stack in its standard ;;; location. -(!def-vm-support-routine make-return-pc-passing-location (standard) +(defun make-return-pc-passing-location (standard) (declare (ignore standard)) (make-wired-tn (primitive-type-or-lose 'system-area-pointer) sap-stack-sc-number return-pc-save-offset)) @@ -38,7 +38,7 @@ ;;; because we want to be able to assume it's always there. Besides, ;;; the x86 doesn't have enough registers to really make it profitable ;;; to pass it in a register. -(!def-vm-support-routine make-old-fp-passing-location (standard) +(defun make-old-fp-passing-location (standard) (declare (ignore standard)) (make-wired-tn *fixnum-primitive-type* control-stack-sc-number ocfp-save-offset)) @@ -49,12 +49,12 @@ ;;; ;;; Without using a save-tn - which does not make much sense if it is ;;; wired to the stack? -(!def-vm-support-routine make-old-fp-save-location (physenv) +(defun make-old-fp-save-location (physenv) (physenv-debug-live-tn (make-wired-tn *fixnum-primitive-type* control-stack-sc-number ocfp-save-offset) physenv)) -(!def-vm-support-routine make-return-pc-save-location (physenv) +(defun make-return-pc-save-location (physenv) (physenv-debug-live-tn (make-wired-tn (primitive-type-or-lose 'system-area-pointer) sap-stack-sc-number return-pc-save-offset) @@ -63,23 +63,23 @@ ;;; Make a TN for the standard argument count passing location. We only ;;; need to make the standard location, since a count is never passed when we ;;; are using non-standard conventions. -(!def-vm-support-routine make-arg-count-location () +(defun make-arg-count-location () (make-wired-tn *fixnum-primitive-type* any-reg-sc-number rcx-offset)) ;;; Make a TN to hold the number-stack frame pointer. This is allocated ;;; once per component, and is component-live. -(!def-vm-support-routine make-nfp-tn () +(defun make-nfp-tn () (make-restricted-tn *fixnum-primitive-type* ignore-me-sc-number)) -(!def-vm-support-routine make-stack-pointer-tn () +(defun make-stack-pointer-tn () (make-normal-tn *fixnum-primitive-type*)) -(!def-vm-support-routine make-number-stack-pointer-tn () +(defun make-number-stack-pointer-tn () (make-restricted-tn *fixnum-primitive-type* ignore-me-sc-number)) ;;; Return a list of TNs that can be used to represent an unknown-values ;;; continuation within a function. -(!def-vm-support-routine make-unknown-values-locations () +(defun make-unknown-values-locations () (list (make-stack-pointer-tn) (make-normal-tn *fixnum-primitive-type*))) @@ -87,19 +87,9 @@ ;;; VM-dependent initialization of the IR2-COMPONENT structure. We ;;; push placeholder entries in the CONSTANTS to leave room for ;;; additional noise in the code object header. -(!def-vm-support-routine select-component-format (component) +(defun select-component-format (component) (declare (type component component)) - ;; The 1+ here is because for the x86 the first constant is a - ;; pointer to a list of fixups, or NIL if the code object has none. - ;; (If I understand correctly, the fixups are needed at GC copy - ;; time because the X86 code isn't relocatable.) - ;; - ;; KLUDGE: It'd be cleaner to have the fixups entry be a named - ;; element of the CODE (aka component) primitive object. However, - ;; it's currently a large, tricky, error-prone chore to change - ;; the layout of any primitive object, so for the foreseeable future - ;; we'll just live with this ugliness. -- WHN 2002-01-02 - (dotimes (i (1+ code-constants-offset)) + (dotimes (i code-constants-offset) (vector-push-extend nil (ir2-component-constants (component-info component)))) (values)) @@ -139,6 +129,79 @@ (storew value frame-pointer (frame-word-offset (tn-offset variable-home-tn))))) +(macrolet ((define-frame-op + (suffix sc stack-sc instruction + &optional (ea + `(make-ea :qword + :base frame-pointer + :disp (frame-byte-offset + (tn-offset variable-home-tn))))) + (let ((reffer (symbolicate 'ancestor-frame-ref '/ suffix)) + (setter (symbolicate 'ancestor-frame-set '/ suffix))) + `(progn + (define-vop (,reffer ancestor-frame-ref) + (:results (value :scs (,sc))) + (:generator 4 + (aver (sc-is variable-home-tn ,stack-sc)) + (inst ,instruction value + ,ea))) + (define-vop (,setter ancestor-frame-set) + (:args (frame-pointer :scs (descriptor-reg)) + (value :scs (,sc))) + (:generator 4 + (aver (sc-is variable-home-tn ,stack-sc)) + (inst ,instruction ,ea value))))))) + (define-frame-op double-float double-reg double-stack movsd) + (define-frame-op single-float single-reg single-stack movss) + (define-frame-op complex-double-float complex-double-reg complex-double-stack + movupd (ea-for-cdf-data-stack variable-home-tn frame-pointer)) + (define-frame-op complex-single-float complex-single-reg complex-single-stack + movq (ea-for-csf-data-stack variable-home-tn frame-pointer)) + (define-frame-op signed-byte-64 signed-reg signed-stack mov) + (define-frame-op unsigned-byte-64 unsigned-reg unsigned-stack mov) + (define-frame-op system-area-pointer sap-reg sap-stack mov)) + +(defun primitive-type-indirect-cell-type (ptype) + (declare (type primitive-type ptype)) + (macrolet ((foo (&body data) + `(case (primitive-type-name ptype) + ,@(loop for (name stack-sc ref set) in data + collect + `(,name + (load-time-value + (list (primitive-type-or-lose ',name) + (sc-or-lose ',stack-sc) + (lambda (node block fp value res) + (sb!c::vop ,ref node block + fp value res)) + (lambda (node block fp new-val value) + (sb!c::vop ,set node block + fp new-val value))))))))) + (foo (double-float double-stack + ancestor-frame-ref/double-float + ancestor-frame-set/double-float) + (single-float single-stack + ancestor-frame-ref/single-float + ancestor-frame-set/single-float) + (complex-double-float complex-double-stack + ancestor-frame-ref/complex-double-float + ancestor-frame-set/complex-double-float) + (complex-single-float complex-single-stack + ancestor-frame-ref/complex-single-float + ancestor-frame-set/complex-single-float) + (signed-byte-64 signed-stack + ancestor-frame-ref/signed-byte-64 + ancestor-frame-set/signed-byte-64) + (unsigned-byte-64 unsigned-stack + ancestor-frame-ref/unsigned-byte-64 + ancestor-frame-set/unsigned-byte-64) + (unsigned-byte-63 unsigned-stack + ancestor-frame-ref/unsigned-byte-64 + ancestor-frame-set/unsigned-byte-64) + (system-area-pointer sap-stack + ancestor-frame-ref/system-area-pointer + ancestor-frame-set/system-area-pointer)))) + (define-vop (xep-allocate-frame) (:info start-lab copy-more-arg-follows) (:vop-var vop) @@ -370,7 +433,7 @@ :disp (frame-byte-offset (+ sp->fp-offset register-arg-count)))) ;; Do the copy. - (inst shr rcx-tn word-shift) ; make word count + (inst shr rcx-tn n-fixnum-tag-bits) ; make word count (inst std) (inst rep) (inst movs :qword) @@ -382,7 +445,7 @@ ;; If none, then just blow out of here. (inst jmp :le restore-edi) (inst mov rcx-tn rax-tn) - (inst shr rcx-tn word-shift) ; word count + (inst shr rcx-tn n-fixnum-tag-bits) ; word count ;; Load RAX with NIL for fast storing. (inst mov rax-tn nil-value) ;; Do the store. @@ -437,7 +500,15 @@ register-arg-count) (inst cmp nargs (fixnumize register-arg-count)) (inst jmp :g stack-values) + #!+#.(cl:if (cl:= sb!vm:word-shift sb!vm:n-fixnum-tag-bits) '(and) '(or)) (inst sub rsp-tn nargs) + #!-#.(cl:if (cl:= sb!vm:word-shift sb!vm:n-fixnum-tag-bits) '(and) '(or)) + (progn + ;; FIXME: This can't be efficient, but LEA (my first choice) + ;; doesn't do subtraction. + (inst shl nargs (- word-shift n-fixnum-tag-bits)) + (inst sub rsp-tn nargs) + (inst shr nargs (- word-shift n-fixnum-tag-bits))) (emit-label stack-values)) ;; dtc: this writes the registers onto the stack even if they are ;; not needed, only the number specified in rcx are used and have @@ -484,16 +555,18 @@ (= (tn-offset return-pc) return-pc-save-offset)) (error "return-pc not on stack in standard save location?"))) -;;; Instead of JMPing to TARGET, CALL a trampoline that saves the -;;; return pc and jumps. Although this is an incredibly stupid trick -;;; the paired CALL/RET instructions are a big win. -(defun make-local-call (target) - (let ((tramp (gen-label))) - (inst call tramp) - (assemble (*elsewhere*) - (emit-label tramp) - (popw rbp-tn (frame-word-offset return-pc-save-offset)) - (inst jmp target)))) +;;; The local call convention doesn't fit that well with x86-style +;;; calls. Emit a header for local calls to pop the return address +;;; in the right place. +(defun emit-block-header (start-label trampoline-label fall-thru-p alignp) + (when (and fall-thru-p trampoline-label) + (inst jmp start-label)) + (when trampoline-label + (emit-label trampoline-label) + (popw rbp-tn (frame-word-offset return-pc-save-offset))) + (when alignp + (emit-alignment n-lowtag-bits :long-nop)) + (emit-label start-label)) ;;; Non-TR local call for a fixed number of values passed according to ;;; the unknown values convention. @@ -529,7 +602,7 @@ (trace-table-entry trace-table-call-site) (move rbp-tn fp) (note-this-location vop :call-site) - (make-local-call target) + (inst call target) (default-unknown-values vop values nvals node) (trace-table-entry trace-table-normal))) @@ -550,7 +623,7 @@ (trace-table-entry trace-table-call-site) (move rbp-tn fp) (note-this-location vop :call-site) - (make-local-call target) + (inst call target) (note-this-location vop :unknown-return) (receive-unknown-values values-start nvals start count node) (trace-table-entry trace-table-normal))) @@ -577,7 +650,7 @@ (trace-table-entry trace-table-call-site) (move rbp-tn fp) (note-this-location vop :call-site) - (make-local-call target) + (inst call target) (note-this-location vop :known-return) (trace-table-entry trace-table-normal))) @@ -735,6 +808,9 @@ ;; Compute the number of arguments. (noise '(inst mov rcx new-fp)) (noise '(inst sub rcx rsp-tn)) + #.(unless (= word-shift n-fixnum-tag-bits) + '(noise '(inst shr rcx + (- word-shift n-fixnum-tag-bits)))) ;; Move the necessary args to registers, ;; this moves them all even if they are ;; not all needed. @@ -799,11 +875,11 @@ ;; there are at least 3 slots. This hack ;; just adds 3 more. ,(if variable - '(inst sub rsp-tn (fixnumize 3))) + '(inst sub rsp-tn (* 3 n-word-bytes))) ;; Bias the new-fp for use as an fp ,(if variable - '(inst sub new-fp (fixnumize sp->fp-offset))) + '(inst sub new-fp (* sp->fp-offset n-word-bytes))) ;; Save the fp (storew rbp-tn new-fp @@ -865,9 +941,7 @@ (move rsi args) (move rax function) ;; And jump to the assembly routine. - (inst lea call-target - (make-ea :qword - :disp (make-fixup 'tail-call-variable :assembly-routine))) + (inst mov call-target (make-fixup 'tail-call-variable :assembly-routine)) (inst jmp call-target))) ;;;; unknown values return @@ -1013,9 +1087,7 @@ (emit-label not-single))) (move rsi vals) (move rcx nvals) - (inst lea return-asm - (make-ea :qword :disp (make-fixup 'return-multiple - :assembly-routine))) + (inst mov return-asm (make-fixup 'return-multiple :assembly-routine)) (inst jmp return-asm) (trace-table-entry trace-table-normal))) @@ -1053,15 +1125,25 @@ (inst cmp rcx-tn (fixnumize fixed)) (inst jmp :be JUST-ALLOC-FRAME))) + ;; Create a negated copy of the number of arguments to allow us to + ;; use EA calculations in order to do scaled subtraction. + (inst mov temp rcx-tn) + (inst neg temp) + ;; Allocate the space on the stack. ;; stack = rbp + sp->fp-offset - (max 3 frame-size) - (nargs - fixed) - (inst lea rbx-tn + ;; if we'd move SP backward, swap the meaning of rsp and source; + ;; otherwise, we'd be accessing values below SP, and that's no good + ;; if a signal interrupts this code sequence. In that case, store + ;; the final value in rsp after the stack-stack memmove loop. + (inst lea (if (<= fixed (max 3 (sb-allocated-size 'stack))) + rsp-tn + source) (make-ea :qword :base rbp-tn - :disp (* n-word-bytes - (- (+ sp->fp-offset fixed) - (max 3 (sb-allocated-size 'stack)))))) - (inst sub rbx-tn rcx-tn) ; Got the new stack in rbx - (inst mov rsp-tn rbx-tn) + :index temp :scale (ash 1 (- word-shift n-fixnum-tag-bits)) + :disp (* n-word-bytes + (- (+ sp->fp-offset fixed) + (max 3 (sb-allocated-size 'stack)))))) ;; Now: nargs>=1 && nargs>fixed @@ -1069,40 +1151,64 @@ (inst mov rbx-tn rcx-tn) (cond ((< fixed register-arg-count) + ;; the code above only moves the final value of rsp in + ;; rsp directly if that condition is satisfied. Currently, + ;; r-a-c is 3, so the aver is OK. If the calling convention + ;; ever changes, the logic above with LEA will have to be + ;; adjusted. + (aver (<= fixed (max 3 (sb-allocated-size 'stack)))) ;; We must stop when we run out of stack args, not when we ;; run out of more args. ;; Number to copy = nargs-3 - (inst sub rcx-tn (fixnumize register-arg-count)) + (inst sub rbx-tn (fixnumize register-arg-count)) ;; Everything of interest in registers. (inst jmp :be DO-REGS)) (t ;; Number to copy = nargs-fixed - (inst sub rcx-tn (fixnumize fixed)))) + (inst sub rbx-tn (fixnumize fixed)))) ;; Initialize R8 to be the end of args. - (inst lea source (make-ea :qword :base rbp-tn - :disp (* sp->fp-offset n-word-bytes))) - (inst sub source rbx-tn) - - ;; We need to copy from downwards up to avoid overwriting some of - ;; the yet uncopied args. So we need to use R9 as the copy index - ;; and RCX as the loop counter, rather than using RCX for both. - (zeroize copy-index) - - ;; We used to use REP MOVS here, but on modern x86 it performs - ;; much worse than an explicit loop for small blocks. - COPY-LOOP - (inst mov temp (make-ea :qword :base source :index copy-index)) - (inst mov (make-ea :qword :base rsp-tn :index copy-index) temp) - (inst add copy-index n-word-bytes) - (inst sub rcx-tn n-word-bytes) - (inst jmp :nz COPY-LOOP) + ;; Swap with SP if necessary to mirror the previous condition + (inst lea (if (<= fixed (max 3 (sb-allocated-size 'stack))) + source + rsp-tn) + (make-ea :qword :base rbp-tn + :index temp :scale (ash 1 (- word-shift n-fixnum-tag-bits)) + :disp (* sp->fp-offset n-word-bytes))) + ;; src: rbp + temp + sp->fp + ;; dst: rbp + temp + sp->fp + (fixed - (max 3 [stack-size])) + (let ((delta (- fixed (max 3 (sb-allocated-size 'stack)))) + (loop (gen-label)) + (fixnum->word (ash 1 (- word-shift n-fixnum-tag-bits)))) + (cond ((zerop delta)) ; no-op move + ((minusp delta) + ;; dst is lower than src, copy forward + (zeroize copy-index) + ;; We used to use REP MOVS here, but on modern x86 it performs + ;; much worse than an explicit loop for small blocks. + + (emit-label loop) + (inst mov temp (make-ea :qword :base source :index copy-index)) + (inst mov (make-ea :qword :base rsp-tn :index copy-index) temp) + (inst add copy-index n-word-bytes) + (inst sub rbx-tn (fixnumize 1)) + (inst jmp :nz loop)) + ((plusp delta) + ;; dst is higher than src; copy backward + (emit-label loop) + (inst sub rbx-tn (fixnumize 1)) + (inst mov temp (make-ea :qword :base rsp-tn + :index rbx-tn :scale fixnum->word)) + (inst mov (make-ea :qword :base source + :index rbx-tn :scale fixnum->word) + temp) + (inst jmp :nz loop) + ;; done with the stack--stack copy. Reset RSP to its final + ;; value + (inst mov rsp-tn source)))) DO-REGS - ;; Restore RCX - (inst mov rcx-tn rbx-tn) - ;; Here: nargs>=1 && nargs>fixed (when (< fixed register-arg-count) ;; Now we have to deposit any more args that showed up in @@ -1150,12 +1256,14 @@ (keyword :scs (descriptor-reg any-reg))) (:result-types * *) (:generator 4 - (inst mov value (make-ea :qword :base object :index index)) + (inst mov value (make-ea :qword :base object :index index + :scale (ash 1 (- word-shift n-fixnum-tag-bits)))) (inst mov keyword (make-ea :qword :base object :index index + :scale (ash 1 (- word-shift n-fixnum-tag-bits)) :disp n-word-bytes)))) (define-vop (more-arg) - (:translate sb!c::%more-arg) + (:translate sb!c::%more-arg) (:policy :fast-safe) (:args (object :scs (descriptor-reg) :to (:result 1)) (index :scs (any-reg) :to (:result 1) :target value)) @@ -1165,7 +1273,8 @@ (:generator 4 (move value index) (inst neg value) - (inst mov value (make-ea :qword :base object :index value)))) + (inst mov value (make-ea :qword :base object :index value + :scale (ash 1 (- word-shift n-fixnum-tag-bits)))))) ;;; Turn more arg (context, count) into a list. (define-vop (listify-rest-args) @@ -1190,11 +1299,9 @@ ;; Check to see whether there are no args, and just return NIL if so. (inst mov result nil-value) (inst jrcxz done) - (inst lea dst (make-ea :qword :base rcx :index rcx)) + (inst lea dst (make-ea :qword :index rcx :scale (ash 2 (- word-shift n-fixnum-tag-bits)))) (maybe-pseudo-atomic stack-allocate-p (allocation dst dst node stack-allocate-p list-pointer-lowtag) - ;; Set decrement mode (successive args at lower addresses) - (inst std) ;; Set up the result. (move result dst) ;; Jump into the middle of the loop, 'cause that's where we want @@ -1211,11 +1318,10 @@ (inst sub src n-word-bytes) (storew rax dst 0 list-pointer-lowtag) ;; Go back for more. - (inst sub rcx n-word-bytes) + (inst sub rcx (fixnumize 1)) (inst jmp :nz loop) ;; NIL out the last cons. - (storew nil-value dst 1 list-pointer-lowtag) - (inst cld)) + (storew nil-value dst 1 list-pointer-lowtag)) (emit-label done)))) ;;; Return the location and size of the &MORE arg glob created by @@ -1243,8 +1349,9 @@ ;; SP at this point points at the last arg pushed. ;; Point to the first more-arg, not above it. (inst lea context (make-ea :qword :base rsp-tn - :index count :scale 1 - :disp (- (+ (fixnumize fixed) n-word-bytes)))) + :index count + :scale (ash 1 (- word-shift n-fixnum-tag-bits)) + :disp (- (* (1+ fixed) n-word-bytes)))) (unless (zerop fixed) (inst sub count (fixnumize fixed)))))