X-Git-Url: http://repo.macrolet.net/gitweb/?a=blobdiff_plain;f=src%2Fcompiler%2Fx86-64%2Fcell.lisp;h=3d85b6b441969e1ff764551abcf6c082c11b81d2;hb=8a632c14b592472873cfb214239c9387bc1a1ced;hp=929e74f002e36930ee03a35c5d2e325b19aef656;hpb=e188f845350da58ffd0f52f08f21cb06cd57c584;p=sbcl.git diff --git a/src/compiler/x86-64/cell.lisp b/src/compiler/x86-64/cell.lisp index 929e74f..3d85b6b 100644 --- a/src/compiler/x86-64/cell.lisp +++ b/src/compiler/x86-64/cell.lisp @@ -46,11 +46,67 @@ temp)) ;; Else, value not immediate. (storew value object offset lowtag)))) - - +(define-vop (compare-and-swap-slot) + (:args (object :scs (descriptor-reg) :to :eval) + (old :scs (descriptor-reg any-reg) :target rax) + (new :scs (descriptor-reg any-reg))) + (:temporary (:sc descriptor-reg :offset rax-offset + :from (:argument 1) :to :result :target result) + rax) + (:info name offset lowtag) + (:ignore name) + (:results (result :scs (descriptor-reg any-reg))) + (:generator 5 + (move rax old) + #!+sb-thread + (inst lock) + (inst cmpxchg (make-ea :qword :base object + :disp (- (* offset n-word-bytes) lowtag)) + new) + (move result rax))) + ;;;; symbol hacking VOPs +(define-vop (%compare-and-swap-symbol-value) + (:translate %compare-and-swap-symbol-value) + (:args (symbol :scs (descriptor-reg) :to (:result 1)) + (old :scs (descriptor-reg any-reg) :target rax) + (new :scs (descriptor-reg any-reg))) + (:temporary (:sc descriptor-reg :offset rax-offset) rax) + #!+sb-thread + (:temporary (:sc descriptor-reg) tls) + (:results (result :scs (descriptor-reg any-reg))) + (:policy :fast-safe) + (:vop-var vop) + (:generator 15 + ;; This code has to pathological cases: NO-TLS-VALUE-MARKER + ;; or UNBOUND-MARKER as NEW: in either case we would end up + ;; doing possible damage with CMPXCHG -- so don't do that! + (let ((unbound (generate-error-code vop unbound-symbol-error symbol)) + (check (gen-label))) + (move rax old) + #!+sb-thread + (progn + (loadw tls symbol symbol-tls-index-slot other-pointer-lowtag) + ;; Thread-local area, no LOCK needed. + (inst cmpxchg (make-ea :qword :base thread-base-tn + :index tls :scale 1) + new) + (inst cmp rax no-tls-value-marker-widetag) + (inst jmp :ne check) + (move rax old) + (inst lock)) + (inst cmpxchg (make-ea :qword :base symbol + :disp (- (* symbol-value-slot n-word-bytes) + other-pointer-lowtag) + :scale 1) + new) + (emit-label check) + (move result rax) + (inst cmp result unbound-marker-widetag) + (inst jmp :e unbound)))) + ;;; these next two cf the sparc version, by jrd. ;;; FIXME: Deref this ^ reference. @@ -66,8 +122,6 @@ (let ((global-val (gen-label)) (done (gen-label))) (loadw tls symbol symbol-tls-index-slot other-pointer-lowtag) - (inst or tls tls) - (inst jmp :z global-val) (inst cmp (make-ea :qword :base thread-base-tn :scale 1 :index tls) no-tls-value-marker-widetag) (inst jmp :z global-val) @@ -274,7 +328,7 @@ (:args (val :scs (any-reg descriptor-reg)) (symbol :scs (descriptor-reg))) (:temporary (:sc descriptor-reg :offset rax-offset) rax) - (:temporary (:sc unsigned-reg) tls-index temp bsp) + (:temporary (:sc unsigned-reg) tls-index bsp) (:generator 10 (let ((tls-index-valid (gen-label)) (get-tls-index-lock (gen-label)) @@ -286,12 +340,17 @@ (inst or tls-index tls-index) (inst jmp :ne tls-index-valid) + ;; FIXME: + ;; * We should ensure the existence of TLS index for LET-bound specials + ;; at compile/load time, and use write a FAST-BIND for use with those. + ;; * PROGV will need to do this, but even there this should not be inline. + ;; This is probably best moved to C, since dynbind.c also needs to do this. (pseudo-atomic (emit-label get-tls-index-lock) - (inst mov temp 1) + (inst mov tls-index 1) (zeroize rax) (inst lock) - (inst cmpxchg (make-ea-for-symbol-value *tls-index-lock*) temp) + (inst cmpxchg (make-ea-for-symbol-value *tls-index-lock*) tls-index) (inst jmp :ne get-tls-index-lock) ;; now with the lock held, see if the symbol's tls index has ;; been set in the meantime @@ -300,17 +359,14 @@ (inst jmp :ne release-tls-index-lock) ;; allocate a new tls-index (load-symbol-value tls-index *free-tls-index*) - (inst add tls-index 8) ;XXX surely we can do this more - (store-symbol-value tls-index *free-tls-index*) ;succintly - (inst sub tls-index 8) + (inst add (make-ea-for-symbol-value *free-tls-index*) 8) ; fixnum + 1 (storew tls-index symbol symbol-tls-index-slot other-pointer-lowtag) (emit-label release-tls-index-lock) (store-symbol-value 0 *tls-index-lock*)) (emit-label tls-index-valid) - (inst mov temp - (make-ea :qword :base thread-base-tn :scale 1 :index tls-index)) - (storew temp bsp (- binding-value-slot binding-size)) + (inst mov rax (make-ea :qword :base thread-base-tn :scale 1 :index tls-index)) + (storew rax bsp (- binding-value-slot binding-size)) (storew symbol bsp (- binding-symbol-slot binding-size)) (inst mov (make-ea :qword :base thread-base-tn :scale 1 :index tls-index) val)))) @@ -374,6 +430,9 @@ (loadw symbol bsp (- binding-symbol-slot binding-size)) (inst or symbol symbol) (inst jmp :z SKIP) + ;; Bind stack debug sentinels have the unbound marker in the symbol slot + (inst cmp symbol unbound-marker-widetag) + (inst jmp :eq SKIP) (loadw value bsp (- binding-value-slot binding-size)) #!-sb-thread (storew value symbol symbol-value-slot other-pointer-lowtag) @@ -392,6 +451,25 @@ (store-binding-stack-pointer bsp) DONE)) + +(define-vop (bind-sentinel) + (:temporary (:sc unsigned-reg) bsp) + (:generator 1 + (load-binding-stack-pointer bsp) + (inst add bsp (* binding-size n-word-bytes)) + (storew unbound-marker-widetag bsp (- binding-symbol-slot binding-size)) + (storew rbp-tn bsp (- binding-value-slot binding-size)) + (store-binding-stack-pointer bsp))) + +(define-vop (unbind-sentinel) + (:temporary (:sc unsigned-reg) bsp) + (:generator 1 + (load-binding-stack-pointer bsp) + (storew 0 bsp (- binding-value-slot binding-size)) + (storew 0 bsp (- binding-symbol-slot binding-size)) + (inst sub bsp (* binding-size n-word-bytes)) + (store-binding-stack-pointer bsp))) + @@ -441,32 +519,10 @@ (define-full-setter instance-index-set * instance-slots-offset instance-pointer-lowtag (any-reg descriptor-reg) * %instance-set) - -(defknown %instance-set-conditional (instance index t t) t - (unsafe)) - -(define-vop (instance-set-conditional) - (:translate %instance-set-conditional) - (:args (object :scs (descriptor-reg) :to :eval) - (slot :scs (any-reg) :to :result) - (old-value :scs (descriptor-reg any-reg) :target rax) - (new-value :scs (descriptor-reg any-reg))) - (:arg-types instance positive-fixnum * *) - (:temporary (:sc descriptor-reg :offset rax-offset - :from (:argument 2) :to :result :target result) rax) - (:results (result :scs (descriptor-reg any-reg))) - ;(:guard (backend-featurep :i486)) - (:policy :fast-safe) - (:generator 5 - (move rax old-value) - (inst lock) - (inst cmpxchg (make-ea :qword :base object :index slot :scale 1 - :disp (- (* instance-slots-offset n-word-bytes) - instance-pointer-lowtag)) - new-value) - (move result rax))) - - +(define-full-compare-and-swap %compare-and-swap-instance-ref instance + instance-slots-offset instance-pointer-lowtag + (any-reg descriptor-reg) * + %compare-and-swap-instance-ref) ;;;; code object frobbing @@ -475,11 +531,25 @@ (define-full-setter code-header-set * 0 other-pointer-lowtag (any-reg descriptor-reg) * code-header-set) - - ;;;; raw instance slot accessors +(defun make-ea-for-raw-slot (object index instance-length + &optional (adjustment 0)) + (etypecase index + (tn + (make-ea :qword :base object :index instance-length + :disp (+ (* (1- instance-slots-offset) n-word-bytes) + (- instance-pointer-lowtag) + adjustment))) + (integer + (make-ea :qword :base object :index instance-length + :scale 8 + :disp (+ (* (1- instance-slots-offset) n-word-bytes) + (- instance-pointer-lowtag) + adjustment + (- (fixnumize index))))))) + (define-vop (raw-instance-ref/word) (:translate %raw-instance-ref/word) (:policy :fast-safe) @@ -493,13 +563,23 @@ (inst shr tmp n-widetag-bits) (inst shl tmp 3) (inst sub tmp index) - (inst mov - value - (make-ea :qword - :base object - :index tmp - :disp (- (* (1- instance-slots-offset) n-word-bytes) - instance-pointer-lowtag))))) + (inst mov value (make-ea-for-raw-slot object index tmp)))) + +(define-vop (raw-instance-ref-c/word) + (:translate %raw-instance-ref/word) + (:policy :fast-safe) + (:args (object :scs (descriptor-reg))) + (:arg-types * (:constant (load/store-index #.sb!vm:n-word-bytes + #.instance-pointer-lowtag + #.instance-slots-offset))) + (:info index) + (:temporary (:sc unsigned-reg) tmp) + (:results (value :scs (unsigned-reg))) + (:result-types unsigned-num) + (:generator 4 + (loadw tmp object 0 instance-pointer-lowtag) + (inst shr tmp n-widetag-bits) + (inst mov value (make-ea-for-raw-slot object index tmp)))) (define-vop (raw-instance-set/word) (:translate %raw-instance-set/word) @@ -516,13 +596,26 @@ (inst shr tmp n-widetag-bits) (inst shl tmp 3) (inst sub tmp index) - (inst mov - (make-ea :qword - :base object - :index tmp - :disp (- (* (1- instance-slots-offset) n-word-bytes) - instance-pointer-lowtag)) - value) + (inst mov (make-ea-for-raw-slot object index tmp) value) + (move result value))) + +(define-vop (raw-instance-set-c/word) + (:translate %raw-instance-set/word) + (:policy :fast-safe) + (:args (object :scs (descriptor-reg)) + (value :scs (unsigned-reg) :target result)) + (:arg-types * (:constant (load/store-index #.sb!vm:n-word-bytes + #.instance-pointer-lowtag + #.instance-slots-offset)) + unsigned-num) + (:info index) + (:temporary (:sc unsigned-reg) tmp) + (:results (result :scs (unsigned-reg))) + (:result-types unsigned-num) + (:generator 4 + (loadw tmp object 0 instance-pointer-lowtag) + (inst shr tmp n-widetag-bits) + (inst mov (make-ea-for-raw-slot object index tmp) value) (move result value))) (define-vop (raw-instance-ref/single) @@ -539,13 +632,23 @@ (inst shr tmp n-widetag-bits) (inst shl tmp 3) (inst sub tmp index) - (inst movss - value - (make-ea :dword - :base object - :index tmp - :disp (- (* (1- instance-slots-offset) n-word-bytes) - instance-pointer-lowtag))))) + (inst movss value (make-ea-for-raw-slot object index tmp)))) + +(define-vop (raw-instance-ref-c/single) + (:translate %raw-instance-ref/single) + (:policy :fast-safe) + (:args (object :scs (descriptor-reg))) + (:arg-types * (:constant (load/store-index #.sb!vm:n-word-bytes + #.instance-pointer-lowtag + #.instance-slots-offset))) + (:info index) + (:temporary (:sc unsigned-reg) tmp) + (:results (value :scs (single-reg))) + (:result-types single-float) + (:generator 4 + (loadw tmp object 0 instance-pointer-lowtag) + (inst shr tmp n-widetag-bits) + (inst movss value (make-ea-for-raw-slot object index tmp)))) (define-vop (raw-instance-set/single) (:translate %raw-instance-set/single) @@ -562,13 +665,27 @@ (inst shr tmp n-widetag-bits) (inst shl tmp 3) (inst sub tmp index) - (inst movss - (make-ea :dword - :base object - :index tmp - :disp (- (* (1- instance-slots-offset) n-word-bytes) - instance-pointer-lowtag)) - value) + (inst movss (make-ea-for-raw-slot object index tmp) value) + (unless (location= result value) + (inst movss result value)))) + +(define-vop (raw-instance-set-c/single) + (:translate %raw-instance-set/single) + (:policy :fast-safe) + (:args (object :scs (descriptor-reg)) + (value :scs (single-reg) :target result)) + (:arg-types * (:constant (load/store-index #.sb!vm:n-word-bytes + #.instance-pointer-lowtag + #.instance-slots-offset)) + single-float) + (:info index) + (:temporary (:sc unsigned-reg) tmp) + (:results (result :scs (single-reg))) + (:result-types single-float) + (:generator 4 + (loadw tmp object 0 instance-pointer-lowtag) + (inst shr tmp n-widetag-bits) + (inst movss (make-ea-for-raw-slot object index tmp) value) (unless (location= result value) (inst movss result value)))) @@ -586,13 +703,23 @@ (inst shr tmp n-widetag-bits) (inst shl tmp 3) (inst sub tmp index) - (inst movsd - value - (make-ea :dword - :base object - :index tmp - :disp (- (* (1- instance-slots-offset) n-word-bytes) - instance-pointer-lowtag))))) + (inst movsd value (make-ea-for-raw-slot object index tmp)))) + +(define-vop (raw-instance-ref-c/double) + (:translate %raw-instance-ref/double) + (:policy :fast-safe) + (:args (object :scs (descriptor-reg))) + (:arg-types * (:constant (load/store-index #.sb!vm:n-word-bytes + #.instance-pointer-lowtag + #.instance-slots-offset))) + (:info index) + (:temporary (:sc unsigned-reg) tmp) + (:results (value :scs (double-reg))) + (:result-types double-float) + (:generator 4 + (loadw tmp object 0 instance-pointer-lowtag) + (inst shr tmp n-widetag-bits) + (inst movsd value (make-ea-for-raw-slot object index tmp)))) (define-vop (raw-instance-set/double) (:translate %raw-instance-set/double) @@ -609,13 +736,27 @@ (inst shr tmp n-widetag-bits) (inst shl tmp 3) (inst sub tmp index) - (inst movsd - (make-ea :dword - :base object - :index tmp - :disp (- (* (1- instance-slots-offset) n-word-bytes) - instance-pointer-lowtag)) - value) + (inst movsd (make-ea-for-raw-slot object index tmp) value) + (unless (location= result value) + (inst movsd result value)))) + +(define-vop (raw-instance-set-c/double) + (:translate %raw-instance-set/double) + (:policy :fast-safe) + (:args (object :scs (descriptor-reg)) + (value :scs (double-reg) :target result)) + (:arg-types * (:constant (load/store-index #.sb!vm:n-word-bytes + #.instance-pointer-lowtag + #.instance-slots-offset)) + double-float) + (:info index) + (:temporary (:sc unsigned-reg) tmp) + (:results (result :scs (double-reg))) + (:result-types double-float) + (:generator 4 + (loadw tmp object 0 instance-pointer-lowtag) + (inst shr tmp n-widetag-bits) + (inst movsd (make-ea-for-raw-slot object index tmp) value) (unless (location= result value) (inst movsd result value)))) @@ -634,22 +775,28 @@ (inst shl tmp 3) (inst sub tmp index) (let ((real-tn (complex-single-reg-real-tn value))) - (inst movss - real-tn - (make-ea :dword - :base object - :index tmp - :disp (- (* (1- instance-slots-offset) n-word-bytes) - instance-pointer-lowtag)))) + (inst movss real-tn (make-ea-for-raw-slot object index tmp))) (let ((imag-tn (complex-single-reg-imag-tn value))) - (inst movss - imag-tn - (make-ea :dword - :base object - :index tmp - :disp (+ (* (1- instance-slots-offset) n-word-bytes) - 4 - (- instance-pointer-lowtag))))))) + (inst movss imag-tn (make-ea-for-raw-slot object index tmp 4))))) + +(define-vop (raw-instance-ref-c/complex-single) + (:translate %raw-instance-ref/complex-single) + (:policy :fast-safe) + (:args (object :scs (descriptor-reg))) + (:arg-types * (:constant (load/store-index #.sb!vm:n-word-bytes + #.instance-pointer-lowtag + #.instance-slots-offset))) + (:info index) + (:temporary (:sc unsigned-reg) tmp) + (:results (value :scs (complex-single-reg))) + (:result-types complex-single-float) + (:generator 4 + (loadw tmp object 0 instance-pointer-lowtag) + (inst shr tmp n-widetag-bits) + (let ((real-tn (complex-single-reg-real-tn value))) + (inst movss real-tn (make-ea-for-raw-slot object index tmp))) + (let ((imag-tn (complex-single-reg-imag-tn value))) + (inst movss imag-tn (make-ea-for-raw-slot object index tmp 4))))) (define-vop (raw-instance-set/complex-single) (:translate %raw-instance-set/complex-single) @@ -668,23 +815,39 @@ (inst sub tmp index) (let ((value-real (complex-single-reg-real-tn value)) (result-real (complex-single-reg-real-tn result))) - (inst movss (make-ea :dword - :base object - :index tmp - :disp (- (* (1- instance-slots-offset) n-word-bytes) - instance-pointer-lowtag)) - value-real) + (inst movss (make-ea-for-raw-slot object index tmp) value-real) (unless (location= value-real result-real) (inst movss result-real value-real))) (let ((value-imag (complex-single-reg-imag-tn value)) (result-imag (complex-single-reg-imag-tn result))) - (inst movss (make-ea :dword - :base object - :index tmp - :disp (+ (* (1- instance-slots-offset) n-word-bytes) - 4 - (- instance-pointer-lowtag))) - value-imag) + (inst movss (make-ea-for-raw-slot object index tmp 4) value-imag) + (unless (location= value-imag result-imag) + (inst movss result-imag value-imag))))) + +(define-vop (raw-instance-set-c/complex-single) + (:translate %raw-instance-set/complex-single) + (:policy :fast-safe) + (:args (object :scs (descriptor-reg)) + (value :scs (complex-single-reg) :target result)) + (:arg-types * (:constant (load/store-index #.sb!vm:n-word-bytes + #.instance-pointer-lowtag + #.instance-slots-offset)) + complex-single-float) + (:info index) + (:temporary (:sc unsigned-reg) tmp) + (:results (result :scs (complex-single-reg))) + (:result-types complex-single-float) + (:generator 4 + (loadw tmp object 0 instance-pointer-lowtag) + (inst shr tmp n-widetag-bits) + (let ((value-real (complex-single-reg-real-tn value)) + (result-real (complex-single-reg-real-tn result))) + (inst movss (make-ea-for-raw-slot object index tmp) value-real) + (unless (location= value-real result-real) + (inst movss result-real value-real))) + (let ((value-imag (complex-single-reg-imag-tn value)) + (result-imag (complex-single-reg-imag-tn result))) + (inst movss (make-ea-for-raw-slot object index tmp 4) value-imag) (unless (location= value-imag result-imag) (inst movss result-imag value-imag))))) @@ -703,21 +866,28 @@ (inst shl tmp 3) (inst sub tmp index) (let ((real-tn (complex-double-reg-real-tn value))) - (inst movsd - real-tn - (make-ea :dword - :base object - :index tmp - :disp (- (* (- instance-slots-offset 2) n-word-bytes) - instance-pointer-lowtag)))) + (inst movsd real-tn (make-ea-for-raw-slot object index tmp -8))) (let ((imag-tn (complex-double-reg-imag-tn value))) - (inst movsd - imag-tn - (make-ea :dword - :base object - :index tmp - :disp (- (* (1- instance-slots-offset) n-word-bytes) - instance-pointer-lowtag)))))) + (inst movsd imag-tn (make-ea-for-raw-slot object index tmp))))) + +(define-vop (raw-instance-ref-c/complex-double) + (:translate %raw-instance-ref/complex-double) + (:policy :fast-safe) + (:args (object :scs (descriptor-reg))) + (:arg-types * (:constant (load/store-index #.sb!vm:n-word-bytes + #.instance-pointer-lowtag + #.instance-slots-offset))) + (:info index) + (:temporary (:sc unsigned-reg) tmp) + (:results (value :scs (complex-double-reg))) + (:result-types complex-double-float) + (:generator 4 + (loadw tmp object 0 instance-pointer-lowtag) + (inst shr tmp n-widetag-bits) + (let ((real-tn (complex-double-reg-real-tn value))) + (inst movsd real-tn (make-ea-for-raw-slot object index tmp -8))) + (let ((imag-tn (complex-double-reg-imag-tn value))) + (inst movsd imag-tn (make-ea-for-raw-slot object index tmp))))) (define-vop (raw-instance-set/complex-double) (:translate %raw-instance-set/complex-double) @@ -736,21 +906,38 @@ (inst sub tmp index) (let ((value-real (complex-double-reg-real-tn value)) (result-real (complex-double-reg-real-tn result))) - (inst movsd (make-ea :dword - :base object - :index tmp - :disp (- (* (- instance-slots-offset 2) n-word-bytes) - instance-pointer-lowtag)) - value-real) + (inst movsd (make-ea-for-raw-slot object index tmp -8) value-real) + (unless (location= value-real result-real) + (inst movsd result-real value-real))) + (let ((value-imag (complex-double-reg-imag-tn value)) + (result-imag (complex-double-reg-imag-tn result))) + (inst movsd (make-ea-for-raw-slot object index tmp) value-imag) + (unless (location= value-imag result-imag) + (inst movsd result-imag value-imag))))) + +(define-vop (raw-instance-set-c/complex-double) + (:translate %raw-instance-set/complex-double) + (:policy :fast-safe) + (:args (object :scs (descriptor-reg)) + (value :scs (complex-double-reg) :target result)) + (:arg-types * (:constant (load/store-index #.sb!vm:n-word-bytes + #.instance-pointer-lowtag + #.instance-slots-offset)) + complex-double-float) + (:info index) + (:temporary (:sc unsigned-reg) tmp) + (:results (result :scs (complex-double-reg))) + (:result-types complex-double-float) + (:generator 4 + (loadw tmp object 0 instance-pointer-lowtag) + (inst shr tmp n-widetag-bits) + (let ((value-real (complex-double-reg-real-tn value)) + (result-real (complex-double-reg-real-tn result))) + (inst movsd (make-ea-for-raw-slot object index tmp -8) value-real) (unless (location= value-real result-real) (inst movsd result-real value-real))) (let ((value-imag (complex-double-reg-imag-tn value)) (result-imag (complex-double-reg-imag-tn result))) - (inst movsd (make-ea :dword - :base object - :index tmp - :disp (- (* (1- instance-slots-offset) n-word-bytes) - instance-pointer-lowtag)) - value-imag) + (inst movsd (make-ea-for-raw-slot object index tmp) value-imag) (unless (location= value-imag result-imag) (inst movsd result-imag value-imag)))))