X-Git-Url: http://repo.macrolet.net/gitweb/?a=blobdiff_plain;f=src%2Fcompiler%2Fppc%2Fcell.lisp;h=fda746e4edcb4406ff0d9625573a55196423d7d7;hb=988c0c0982e8ebb891da9241c71b5f8350448350;hp=bf175f453d1b3e9d35de49bfe23f695d3754025a;hpb=506253505641855dc8bb87033f7af894904f848b;p=sbcl.git

diff --git a/src/compiler/ppc/cell.lisp b/src/compiler/ppc/cell.lisp
index bf175f4..fda746e 100644
--- a/src/compiler/ppc/cell.lisp
+++ b/src/compiler/ppc/cell.lisp
@@ -1,12 +1,16 @@
-;;; VOPs for the PPC.
-;;;
-;;; Written by Rob MacLachlan
-;;;
-;;; Converted by William Lott.
-;;; 
+;;;; the VM definition of various primitive memory access VOPs for the
+;;;; PPC
+
+;;;; This software is part of the SBCL system. See the README file for
+;;;; more information.
+;;;;
+;;;; This software is derived from the CMU CL system, which was
+;;;; written at Carnegie Mellon University and released into the
+;;;; public domain. The software is in the public domain and is
+;;;; provided with absolutely no warranty. See the COPYING and CREDITS
+;;;; files for more information.
 
 (in-package "SB!VM")
-
 
 ;;;; Data object ref/set stuff.
 
@@ -20,24 +24,81 @@
 
 (define-vop (set-slot)
   (:args (object :scs (descriptor-reg))
-	 (value :scs (descriptor-reg any-reg)))
+         (value :scs (descriptor-reg any-reg)))
   (:info name offset lowtag)
   (:ignore name)
   (:results)
   (:generator 1
     (storew value object offset lowtag)))
 
+#!+compare-and-swap-vops
+(define-vop (compare-and-swap-slot)
+  (:args (object :scs (descriptor-reg))
+         (old :scs (descriptor-reg any-reg))
+         (new :scs (descriptor-reg any-reg)))
+  (:temporary (:sc non-descriptor-reg) temp)
+  (:info name offset lowtag)
+  (:ignore name)
+  (:results (result :scs (descriptor-reg) :from :load))
+  (:generator 5
+    (inst sync)
+    (inst li temp (- (* offset n-word-bytes) lowtag))
+    LOOP
+    (inst lwarx result temp object)
+    (inst cmpw result old)
+    (inst bne EXIT)
+    (inst stwcx. new temp object)
+    (inst bne LOOP)
+    EXIT
+    (inst isync)))
 
 
 ;;;; Symbol hacking VOPs:
 
+#!+compare-and-swap-vops
+(define-vop (%compare-and-swap-symbol-value)
+  (:translate %compare-and-swap-symbol-value)
+  (:args (symbol :scs (descriptor-reg))
+         (old :scs (descriptor-reg any-reg))
+         (new :scs (descriptor-reg any-reg)))
+  (:temporary (:sc non-descriptor-reg) temp)
+  (:results (result :scs (descriptor-reg any-reg) :from :load))
+  (:policy :fast-safe)
+  (:vop-var vop)
+  (:generator 15
+    (inst sync)
+    #!+sb-thread
+    (assemble ()
+      (loadw temp symbol symbol-tls-index-slot other-pointer-lowtag)
+      ;; Thread-local area, no synchronization needed.
+      (inst lwzx result thread-base-tn temp)
+      (inst cmpw result old)
+      (inst bne DONT-STORE-TLS)
+      (inst stwx new thread-base-tn temp)
+      DONT-STORE-TLS
+
+      (inst cmpwi result no-tls-value-marker-widetag)
+      (inst bne CHECK-UNBOUND))
+
+    (inst li temp (- (* symbol-value-slot n-word-bytes)
+                     other-pointer-lowtag))
+    LOOP
+    (inst lwarx result symbol temp)
+    (inst cmpw result old)
+    (inst bne CHECK-UNBOUND)
+    (inst stwcx. new symbol temp)
+    (inst bne LOOP)
+
+    CHECK-UNBOUND
+    (inst isync)
+    (inst cmpwi result unbound-marker-widetag)
+    (inst beq (generate-error-code vop 'unbound-symbol-error symbol))))
+
 ;;; The compiler likes to be able to directly SET symbols.
-;;;
-(define-vop (set cell-set)
+(define-vop (%set-symbol-global-value cell-set)
   (:variant symbol-value-slot other-pointer-lowtag))
 
 ;;; Do a cell ref with an error check for being unbound.
-;;;
 (define-vop (checked-cell-ref)
   (:args (object :scs (descriptor-reg) :target obj-temp))
   (:results (value :scs (descriptor-reg any-reg)))
@@ -46,19 +107,85 @@
   (:save-p :compute-only)
   (:temporary (:scs (descriptor-reg) :from (:argument 0)) obj-temp))
 
-;;; With Symbol-Value, we check that the value isn't the trap object.  So
-;;; Symbol-Value of NIL is NIL.
-;;;
-(define-vop (symbol-value checked-cell-ref)
-  (:translate symbol-value)
+;;; With SYMBOL-VALUE, we check that the value isn't the trap object.
+;;; So SYMBOL-VALUE of NIL is NIL.
+(define-vop (symbol-global-value checked-cell-ref)
+  (:translate symbol-global-value)
   (:generator 9
     (move obj-temp object)
-    (loadw value obj-temp sb!vm:symbol-value-slot sb!vm:other-pointer-lowtag)
-    (let ((err-lab (generate-error-code vop unbound-symbol-error obj-temp)))
-      (inst cmpwi value sb!vm:unbound-marker-widetag)
+    (loadw value obj-temp symbol-value-slot other-pointer-lowtag)
+    (let ((err-lab (generate-error-code vop 'unbound-symbol-error obj-temp)))
+      (inst cmpwi value unbound-marker-widetag)
       (inst beq err-lab))))
 
-;;; Like CHECKED-CELL-REF, only we are a predicate to see if the cell is bound.
+(define-vop (fast-symbol-global-value cell-ref)
+  (:variant symbol-value-slot other-pointer-lowtag)
+  (:policy :fast)
+  (:translate symbol-global-value))
+
+#!+sb-thread
+(progn
+  (define-vop (set)
+    (:args (symbol :scs (descriptor-reg))
+           (value :scs (descriptor-reg any-reg)))
+    (:temporary (:sc any-reg) tls-slot temp)
+    (:generator 4
+      (loadw tls-slot symbol symbol-tls-index-slot other-pointer-lowtag)
+      (inst lwzx temp thread-base-tn tls-slot)
+      (inst cmpwi temp no-tls-value-marker-widetag)
+      (inst beq GLOBAL-VALUE)
+      (inst stwx value thread-base-tn tls-slot)
+      (inst b DONE)
+      GLOBAL-VALUE
+      (storew value symbol symbol-value-slot other-pointer-lowtag)
+      DONE))
+
+  ;; With Symbol-Value, we check that the value isn't the trap object. So
+  ;; Symbol-Value of NIL is NIL.
+  (define-vop (symbol-value)
+    (:translate symbol-value)
+    (:policy :fast-safe)
+    (:args (object :scs (descriptor-reg) :to (:result 1)))
+    (:results (value :scs (descriptor-reg any-reg)))
+    (:vop-var vop)
+    (:save-p :compute-only)
+    (:generator 9
+      (loadw value object symbol-tls-index-slot other-pointer-lowtag)
+      (inst lwzx value thread-base-tn value)
+      (inst cmpwi value no-tls-value-marker-widetag)
+      (inst bne CHECK-UNBOUND)
+      (loadw value object symbol-value-slot other-pointer-lowtag)
+      CHECK-UNBOUND
+      (inst cmpwi value unbound-marker-widetag)
+      (inst beq (generate-error-code vop 'unbound-symbol-error object))))
+
+  (define-vop (fast-symbol-value symbol-value)
+    ;; KLUDGE: not really fast, in fact, because we're going to have to
+    ;; do a full lookup of the thread-local area anyway.  But half of
+    ;; the meaning of FAST-SYMBOL-VALUE is "do not signal an error if
+    ;; unbound", which is used in the implementation of COPY-SYMBOL.  --
+    ;; CSR, 2003-04-22
+    (:policy :fast)
+    (:translate symbol-value)
+    (:generator 8
+      (loadw value object symbol-tls-index-slot other-pointer-lowtag)
+      (inst lwzx value thread-base-tn value)
+      (inst cmpwi value no-tls-value-marker-widetag)
+      (inst bne DONE)
+      (loadw value object symbol-value-slot other-pointer-lowtag)
+      DONE)))
+
+;;; On unithreaded builds these are just copies of the global versions.
+#!-sb-thread
+(progn
+  (define-vop (symbol-value symbol-global-value)
+    (:translate symbol-value))
+  (define-vop (fast-symbol-value fast-symbol-global-value)
+    (:translate symbol-value))
+  (define-vop (set %set-symbol-global-value)))
+
+;;; Like CHECKED-CELL-REF, only we are a predicate to see if the cell
+;;; is bound.
 (define-vop (boundp-frob)
   (:args (object :scs (descriptor-reg)))
   (:conditional)
@@ -66,18 +193,40 @@
   (:policy :fast-safe)
   (:temporary (:scs (descriptor-reg)) value))
 
+#!+sb-thread
 (define-vop (boundp boundp-frob)
   (:translate boundp)
   (:generator 9
-    (loadw value object sb!vm:symbol-value-slot sb!vm:other-pointer-lowtag)
-    (inst cmpwi value sb!vm:unbound-marker-widetag)
+    (loadw value object symbol-tls-index-slot other-pointer-lowtag)
+    (inst lwzx value thread-base-tn value)
+    (inst cmpwi value no-tls-value-marker-widetag)
+    (inst bne CHECK-UNBOUND)
+    (loadw value object symbol-value-slot other-pointer-lowtag)
+    CHECK-UNBOUND
+    (inst cmpwi value unbound-marker-widetag)
     (inst b? (if not-p :eq :ne) target)))
 
-(define-vop (fast-symbol-value cell-ref)
-  (:variant sb!vm:symbol-value-slot sb!vm:other-pointer-lowtag)
-  (:policy :fast)
-  (:translate symbol-value))
+#!-sb-thread
+(define-vop (boundp boundp-frob)
+  (:translate boundp)
+  (:generator 9
+    (loadw value object symbol-value-slot other-pointer-lowtag)
+    (inst cmpwi value unbound-marker-widetag)
+    (inst b? (if not-p :eq :ne) target)))
 
+(define-vop (symbol-hash)
+  (:policy :fast-safe)
+  (:translate symbol-hash)
+  (:args (symbol :scs (descriptor-reg)))
+  (:results (res :scs (any-reg)))
+  (:result-types positive-fixnum)
+  (:generator 2
+    ;; The symbol-hash slot of NIL holds NIL because it is also the
+    ;; cdr slot, so we have to strip off the two low bits to make sure
+    ;; it is a fixnum.  The lowtag selection magic that is required to
+    ;; ensure this is explained in the comment in objdef.lisp
+    (loadw res symbol symbol-hash-slot other-pointer-lowtag)
+    (inst clrrwi res res n-fixnum-tag-bits)))
 
 ;;;; Fdefinition (fdefn) objects.
 
@@ -94,14 +243,14 @@
     (move obj-temp object)
     (loadw value obj-temp fdefn-fun-slot other-pointer-lowtag)
     (inst cmpw value null-tn)
-    (let ((err-lab (generate-error-code vop undefined-fun-error obj-temp)))
+    (let ((err-lab (generate-error-code vop 'undefined-fun-error obj-temp)))
       (inst beq err-lab))))
 
 (define-vop (set-fdefn-fun)
   (:policy :fast-safe)
   (:translate (setf fdefn-fun))
   (:args (function :scs (descriptor-reg) :target result)
-	 (fdefn :scs (descriptor-reg)))
+         (fdefn :scs (descriptor-reg)))
   (:temporary (:scs (interior-reg)) lip)
   (:temporary (:scs (non-descriptor-reg)) type)
   (:results (result :scs (descriptor-reg)))
@@ -111,9 +260,9 @@
       (inst cmpwi type simple-fun-header-widetag)
       ;;(inst mr lip function)
       (inst addi lip function
-	    (- (ash simple-fun-code-offset word-shift) fun-pointer-lowtag))
+            (- (ash simple-fun-code-offset word-shift) fun-pointer-lowtag))
       (inst beq normal-fn)
-      (inst lr lip  (make-fixup (extern-alien-name "closure_tramp") :foreign))
+      (inst lr lip  (make-fixup "closure_tramp" :foreign))
       (emit-label normal-fn)
       (storew lip fdefn fdefn-raw-addr-slot other-pointer-lowtag)
       (storew function fdefn fdefn-fun-slot other-pointer-lowtag)
@@ -127,7 +276,7 @@
   (:results (result :scs (descriptor-reg)))
   (:generator 38
     (storew null-tn fdefn fdefn-fun-slot other-pointer-lowtag)
-    (inst lr temp  (make-fixup (extern-alien-name "undefined_tramp") :foreign))
+    (inst lr temp  (make-fixup "undefined_tramp" :foreign))
     (storew temp fdefn fdefn-raw-addr-slot other-pointer-lowtag)
     (move result fdefn)))
 
@@ -139,26 +288,96 @@
 ;;; the symbol on the binding stack and stuff the new value into the
 ;;; symbol.
 
+#!+sb-thread
 (define-vop (bind)
   (:args (val :scs (any-reg descriptor-reg))
-	 (symbol :scs (descriptor-reg)))
+         (symbol :scs (descriptor-reg)))
+  (:temporary (:sc non-descriptor-reg :offset nl3-offset) pa-flag)
+  (:temporary (:scs (descriptor-reg)) temp tls-index)
+  (:generator 5
+     (loadw tls-index symbol symbol-tls-index-slot other-pointer-lowtag)
+     (inst cmpwi tls-index 0)
+     (inst bne TLS-VALID)
+
+     ;; No TLS slot allocated, so allocate one.
+     (pseudo-atomic (pa-flag)
+       (without-scheduling ()
+         (assemble ()
+           (inst li temp (+ (static-symbol-offset '*tls-index-lock*)
+                            (ash symbol-value-slot word-shift)
+                            (- other-pointer-lowtag)))
+           OBTAIN-LOCK
+           (inst lwarx tls-index null-tn temp)
+           (inst cmpwi tls-index 0)
+           (inst bne OBTAIN-LOCK)
+           (inst stwcx. thread-base-tn null-tn temp)
+           (inst bne OBTAIN-LOCK)
+           (inst isync)
+
+           ;; Check to see if the TLS index was set while we were waiting.
+           (loadw tls-index symbol symbol-tls-index-slot other-pointer-lowtag)
+           (inst cmpwi tls-index 0)
+           (inst bne RELEASE-LOCK)
+
+           (load-symbol-value tls-index *free-tls-index*)
+           ;; FIXME: Check for TLS index overflow.
+           (inst addi tls-index tls-index n-word-bytes)
+           (store-symbol-value tls-index *free-tls-index*)
+           (inst addi tls-index tls-index (- n-word-bytes))
+           (storew tls-index symbol symbol-tls-index-slot other-pointer-lowtag)
+
+           ;; The sync instruction doesn't need to happen if we branch
+           ;; directly to RELEASE-LOCK as we didn't do any stores in that
+           ;; case.
+           (inst sync)
+           RELEASE-LOCK
+           (inst stwx zero-tn null-tn temp)
+
+           ;; temp is a boxed register, but we've been storing crap in it.
+           ;; fix it before we leave pseudo-atomic.
+           (inst li temp 0))))
+
+     TLS-VALID
+     (inst lwzx temp thread-base-tn tls-index)
+     (inst addi bsp-tn bsp-tn (* 2 n-word-bytes))
+     (storew temp bsp-tn (- binding-value-slot binding-size))
+     (storew symbol bsp-tn (- binding-symbol-slot binding-size))
+     (inst stwx val thread-base-tn tls-index)))
+
+#!-sb-thread
+(define-vop (bind)
+  (:args (val :scs (any-reg descriptor-reg))
+         (symbol :scs (descriptor-reg)))
   (:temporary (:scs (descriptor-reg)) temp)
   (:generator 5
-    (loadw temp symbol sb!vm:symbol-value-slot sb!vm:other-pointer-lowtag)
-    (inst addi bsp-tn bsp-tn (* 2 sb!vm:n-word-bytes))
-    (storew temp bsp-tn (- sb!vm:binding-value-slot sb!vm:binding-size))
-    (storew symbol bsp-tn (- sb!vm:binding-symbol-slot sb!vm:binding-size))
-    (storew val symbol sb!vm:symbol-value-slot sb!vm:other-pointer-lowtag)))
-
+    (loadw temp symbol symbol-value-slot other-pointer-lowtag)
+    (inst addi bsp-tn bsp-tn (* 2 n-word-bytes))
+    (storew temp bsp-tn (- binding-value-slot binding-size))
+    (storew symbol bsp-tn (- binding-symbol-slot binding-size))
+    (storew val symbol symbol-value-slot other-pointer-lowtag)))
 
+#!+sb-thread
+(define-vop (unbind)
+  (:temporary (:scs (descriptor-reg)) tls-index value)
+  (:generator 0
+    (loadw tls-index bsp-tn (- binding-symbol-slot binding-size))
+    (loadw tls-index tls-index symbol-tls-index-slot other-pointer-lowtag)
+    (loadw value bsp-tn (- binding-value-slot binding-size))
+    (inst stwx value thread-base-tn tls-index)
+    (storew zero-tn bsp-tn (- binding-symbol-slot binding-size))
+    (storew zero-tn bsp-tn (- binding-value-slot binding-size))
+    (inst subi bsp-tn bsp-tn (* 2 n-word-bytes))))
+
+#!-sb-thread
 (define-vop (unbind)
   (:temporary (:scs (descriptor-reg)) symbol value)
   (:generator 0
-    (loadw symbol bsp-tn (- sb!vm:binding-symbol-slot sb!vm:binding-size))
-    (loadw value bsp-tn (- sb!vm:binding-value-slot sb!vm:binding-size))
-    (storew value symbol sb!vm:symbol-value-slot sb!vm:other-pointer-lowtag)
-    (storew zero-tn bsp-tn (- sb!vm:binding-symbol-slot sb!vm:binding-size))
-    (inst subi bsp-tn bsp-tn (* 2 sb!vm:n-word-bytes))))
+    (loadw symbol bsp-tn (- binding-symbol-slot binding-size))
+    (loadw value bsp-tn (- binding-value-slot binding-size))
+    (storew value symbol symbol-value-slot other-pointer-lowtag)
+    (storew zero-tn bsp-tn (- binding-symbol-slot binding-size))
+    (storew zero-tn bsp-tn (- binding-value-slot binding-size))
+    (inst subi bsp-tn bsp-tn (* 2 n-word-bytes))))
 
 
 (define-vop (unbind-to-here)
@@ -167,22 +386,28 @@
   (:temporary (:scs (descriptor-reg)) symbol value)
   (:generator 0
     (let ((loop (gen-label))
-	  (skip (gen-label))
-	  (done (gen-label)))
+          (skip (gen-label))
+          (done (gen-label)))
       (move where arg)
       (inst cmpw where bsp-tn)
       (inst beq done)
 
       (emit-label loop)
-      (loadw symbol bsp-tn (- sb!vm:binding-symbol-slot sb!vm:binding-size))
+      (loadw symbol bsp-tn (- binding-symbol-slot binding-size))
       (inst cmpwi symbol 0)
       (inst beq skip)
-      (loadw value bsp-tn (- sb!vm:binding-value-slot sb!vm:binding-size))
-      (storew value symbol sb!vm:symbol-value-slot sb!vm:other-pointer-lowtag)
-      (storew zero-tn bsp-tn (- sb!vm:binding-symbol-slot sb!vm:binding-size))
+      (loadw value bsp-tn (- binding-value-slot binding-size))
+      #!+sb-thread
+      (loadw symbol symbol symbol-tls-index-slot other-pointer-lowtag)
+      #!+sb-thread
+      (inst stwx value thread-base-tn symbol)
+      #!-sb-thread
+      (storew value symbol symbol-value-slot other-pointer-lowtag)
+      (storew zero-tn bsp-tn (- binding-symbol-slot binding-size))
 
       (emit-label skip)
-      (inst subi bsp-tn bsp-tn (* 2 sb!vm:n-word-bytes))
+      (storew zero-tn bsp-tn (- binding-value-slot binding-size))
+      (inst subi bsp-tn bsp-tn (* 2 n-word-bytes))
       (inst cmpw where bsp-tn)
       (inst bne loop)
 
@@ -193,21 +418,17 @@
 ;;;; Closure indexing.
 
 (define-vop (closure-index-ref word-index-ref)
-  (:variant sb!vm:closure-info-offset sb!vm:fun-pointer-lowtag)
+  (:variant closure-info-offset fun-pointer-lowtag)
   (:translate %closure-index-ref))
 
 (define-vop (funcallable-instance-info word-index-ref)
-  (:variant funcallable-instance-info-offset sb!vm:fun-pointer-lowtag)
+  (:variant funcallable-instance-info-offset fun-pointer-lowtag)
   (:translate %funcallable-instance-info))
 
 (define-vop (set-funcallable-instance-info word-index-set)
   (:variant funcallable-instance-info-offset fun-pointer-lowtag)
   (:translate %set-funcallable-instance-info))
 
-(define-vop (funcallable-instance-lexenv cell-ref)
-  (:variant funcallable-instance-lexenv-slot fun-pointer-lowtag))
-
-
 (define-vop (closure-ref slot-ref)
   (:variant closure-info-offset fun-pointer-lowtag))
 
@@ -236,34 +457,26 @@
   (:result-types positive-fixnum)
   (:generator 4
     (loadw temp struct 0 instance-pointer-lowtag)
-    (inst srwi res temp sb!vm:n-widetag-bits)))
-
-(define-vop (instance-ref slot-ref)
-  (:variant instance-slots-offset instance-pointer-lowtag)
-  (:policy :fast-safe)
-  (:translate %instance-ref)
-  (:arg-types * (:constant index)))
-
-#+nil
-(define-vop (instance-set slot-set)
-  (:policy :fast-safe)
-  (:translate %instance-set)
-  (:variant instance-slots-offset instance-pointer-lowtag)
-  (:arg-types instance (:constant index) *))
+    (inst srwi res temp n-widetag-bits)))
 
 (define-vop (instance-index-ref word-index-ref)
-  (:policy :fast-safe) 
+  (:policy :fast-safe)
   (:translate %instance-ref)
   (:variant instance-slots-offset instance-pointer-lowtag)
   (:arg-types instance positive-fixnum))
 
 (define-vop (instance-index-set word-index-set)
-  (:policy :fast-safe) 
+  (:policy :fast-safe)
   (:translate %instance-set)
   (:variant instance-slots-offset instance-pointer-lowtag)
   (:arg-types instance positive-fixnum *))
 
-
+#!+compare-and-swap-vops
+(define-vop (%compare-and-swap-instance-ref word-index-cas)
+  (:policy :fast-safe)
+  (:translate %compare-and-swap-instance-ref)
+  (:variant instance-slots-offset instance-pointer-lowtag)
+  (:arg-types instance tagged-num * *))
 
 
 ;;;; Code object frobbing.
@@ -278,3 +491,333 @@
   (:policy :fast-safe)
   (:variant 0 other-pointer-lowtag))
 
+
+
+;;;; raw instance slot accessors
+
+(defun offset-for-raw-slot (instance-length index n-words)
+  (+ (* (- instance-length instance-slots-offset index (1- n-words))
+        n-word-bytes)
+     (- instance-pointer-lowtag)))
+
+(define-vop (raw-instance-init/word)
+  (:args (object :scs (descriptor-reg))
+         (value :scs (unsigned-reg)))
+  (:arg-types * unsigned-num)
+  (:info instance-length index)
+  (:generator 4
+    (inst stw value object (offset-for-raw-slot instance-length index 1))))
+
+(define-vop (raw-instance-atomic-incf/word)
+  (:translate %raw-instance-atomic-incf/word)
+  (:policy :fast-safe)
+  (:args (object :scs (descriptor-reg))
+         (index :scs (any-reg))
+         (diff :scs (unsigned-reg)))
+  (:arg-types * positive-fixnum unsigned-num)
+  (:temporary (:sc unsigned-reg) offset)
+  (:temporary (:sc non-descriptor-reg) sum)
+  (:results (result :scs (unsigned-reg) :from :load))
+  (:result-types unsigned-num)
+  (:generator 4
+    (loadw offset object 0 instance-pointer-lowtag)
+    ;; offset = (offset >> n-widetag-bits) << 2
+    (inst rlwinm offset offset (- 32 (- n-widetag-bits 2)) (- n-widetag-bits 2) 29)
+    (inst subf offset index offset)
+    (inst addi
+          offset
+          offset
+          (- (* (1- instance-slots-offset) n-word-bytes)
+             instance-pointer-lowtag))
+    ;; load the slot value, add DIFF, write the sum back, and return
+    ;; the original slot value, atomically, and include a memory
+    ;; barrier.
+    (inst sync)
+    LOOP
+    (inst lwarx result offset object)
+    (inst add sum result diff)
+    (inst stwcx. sum offset object)
+    (inst bne LOOP)
+    (inst isync)))
+
+(define-vop (raw-instance-ref/word)
+  (:translate %raw-instance-ref/word)
+  (:policy :fast-safe)
+  (:args (object :scs (descriptor-reg))
+         (index :scs (any-reg)))
+  (:arg-types * positive-fixnum)
+  (:results (value :scs (unsigned-reg)))
+  (:temporary (:scs (non-descriptor-reg)) offset)
+  (:result-types unsigned-num)
+  (:generator 5
+    (loadw offset object 0 instance-pointer-lowtag)
+    ;; offset = (offset >> n-widetag-bits) << 2
+    (inst rlwinm offset offset (- 32 (- n-widetag-bits 2)) (- n-widetag-bits 2) 29)
+    (inst subf offset index offset)
+    (inst addi
+          offset
+          offset
+          (- (* (1- instance-slots-offset) n-word-bytes)
+             instance-pointer-lowtag))
+    (inst lwzx value object offset)))
+
+(define-vop (raw-instance-set/word)
+  (:translate %raw-instance-set/word)
+  (:policy :fast-safe)
+  (:args (object :scs (descriptor-reg))
+         (index :scs (any-reg))
+         (value :scs (unsigned-reg)))
+  (:arg-types * positive-fixnum unsigned-num)
+  (:results (result :scs (unsigned-reg)))
+  (:temporary (:scs (non-descriptor-reg)) offset)
+  (:result-types unsigned-num)
+  (:generator 5
+    (loadw offset object 0 instance-pointer-lowtag)
+    ;; offset = (offset >> n-widetag-bits) << 2
+    (inst rlwinm offset offset (- 32 (- n-widetag-bits 2)) (- n-widetag-bits 2) 29)
+    (inst subf offset index offset)
+    (inst addi
+          offset
+          offset
+          (- (* (1- instance-slots-offset) n-word-bytes)
+             instance-pointer-lowtag))
+    (inst stwx value object offset)
+    (move result value)))
+
+(define-vop (raw-instance-init/single)
+  (:args (object :scs (descriptor-reg))
+         (value :scs (single-reg)))
+  (:arg-types * single-float)
+  (:info instance-length index)
+  (:generator 4
+    (inst stfs value object (offset-for-raw-slot instance-length index 1))))
+
+(define-vop (raw-instance-ref/single)
+  (:translate %raw-instance-ref/single)
+  (:policy :fast-safe)
+  (:args (object :scs (descriptor-reg))
+         (index :scs (any-reg)))
+  (:arg-types * positive-fixnum)
+  (:results (value :scs (single-reg)))
+  (:temporary (:scs (non-descriptor-reg)) offset)
+  (:result-types single-float)
+  (:generator 5
+    (loadw offset object 0 instance-pointer-lowtag)
+    ;; offset = (offset >> n-widetag-bits) << 2
+    (inst rlwinm offset offset (- 32 (- n-widetag-bits 2)) (- n-widetag-bits 2) 29)
+    (inst subf offset index offset)
+    (inst addi
+          offset
+          offset
+          (- (* (1- instance-slots-offset) n-word-bytes)
+             instance-pointer-lowtag))
+    (inst lfsx value object offset)))
+
+(define-vop (raw-instance-set/single)
+  (:translate %raw-instance-set/single)
+  (:policy :fast-safe)
+  (:args (object :scs (descriptor-reg))
+         (index :scs (any-reg))
+         (value :scs (single-reg) :target result))
+  (:arg-types * positive-fixnum single-float)
+  (:results (result :scs (single-reg)))
+  (:result-types single-float)
+  (:temporary (:scs (non-descriptor-reg)) offset)
+  (:generator 5
+    (loadw offset object 0 instance-pointer-lowtag)
+    ;; offset = (offset >> n-widetag-bits) << 2
+    (inst rlwinm offset offset (- 32 (- n-widetag-bits 2)) (- n-widetag-bits 2) 29)
+    (inst subf offset index offset)
+    (inst addi
+          offset
+          offset
+          (- (* (1- instance-slots-offset) n-word-bytes)
+             instance-pointer-lowtag))
+    (inst stfsx value object offset)
+    (unless (location= result value)
+      (inst frsp result value))))
+
+(define-vop (raw-instance-init/double)
+  (:args (object :scs (descriptor-reg))
+         (value :scs (double-reg)))
+  (:arg-types * double-float)
+  (:info instance-length index)
+  (:generator 4
+    (inst stfd value object (offset-for-raw-slot instance-length index 2))))
+
+(define-vop (raw-instance-ref/double)
+  (:translate %raw-instance-ref/double)
+  (:policy :fast-safe)
+  (:args (object :scs (descriptor-reg))
+         (index :scs (any-reg)))
+  (:arg-types * positive-fixnum)
+  (:results (value :scs (double-reg)))
+  (:temporary (:scs (non-descriptor-reg)) offset)
+  (:result-types double-float)
+  (:generator 5
+    (loadw offset object 0 instance-pointer-lowtag)
+    ;; offset = (offset >> n-widetag-bits) << 2
+    (inst rlwinm offset offset (- 32 (- n-widetag-bits 2)) (- n-widetag-bits 2) 29)
+    (inst subf offset index offset)
+    (inst addi
+          offset
+          offset
+          (- (* (- instance-slots-offset 2) n-word-bytes)
+             instance-pointer-lowtag))
+    (inst lfdx value object offset)))
+
+(define-vop (raw-instance-set/double)
+  (:translate %raw-instance-set/double)
+  (:policy :fast-safe)
+  (:args (object :scs (descriptor-reg))
+         (index :scs (any-reg))
+         (value :scs (double-reg) :target result))
+  (:arg-types * positive-fixnum double-float)
+  (:results (result :scs (double-reg)))
+  (:result-types double-float)
+  (:temporary (:scs (non-descriptor-reg)) offset)
+  (:generator 5
+    (loadw offset object 0 instance-pointer-lowtag)
+    ;; offset = (offset >> n-widetag-bits) << 2
+    (inst rlwinm offset offset (- 32 (- n-widetag-bits 2)) (- n-widetag-bits 2) 29)
+    (inst subf offset index offset)
+    (inst addi
+          offset
+          offset
+          (- (* (- instance-slots-offset 2) n-word-bytes)
+             instance-pointer-lowtag))
+    (inst stfdx value object offset)
+    (unless (location= result value)
+      (inst fmr result value))))
+
+(define-vop (raw-instance-init/complex-single)
+  (:args (object :scs (descriptor-reg))
+         (value :scs (complex-single-reg)))
+  (:arg-types * complex-single-float)
+  (:info instance-length index)
+  (:generator 4
+    (inst stfs (complex-single-reg-real-tn value)
+          object (offset-for-raw-slot instance-length index 2))
+    (inst stfs (complex-single-reg-imag-tn value)
+          object (offset-for-raw-slot instance-length index 1))))
+
+(define-vop (raw-instance-ref/complex-single)
+  (:translate %raw-instance-ref/complex-single)
+  (:policy :fast-safe)
+  (:args (object :scs (descriptor-reg))
+         (index :scs (any-reg)))
+  (:arg-types * positive-fixnum)
+  (:results (value :scs (complex-single-reg)))
+  (:temporary (:scs (non-descriptor-reg)) offset)
+  (:result-types complex-single-float)
+  (:generator 5
+    (loadw offset object 0 instance-pointer-lowtag)
+    ;; offset = (offset >> n-widetag-bits) << 2
+    (inst rlwinm offset offset (- 32 (- n-widetag-bits 2)) (- n-widetag-bits 2) 29)
+    (inst subf offset index offset)
+    (inst addi
+          offset
+          offset
+          (- (* (- instance-slots-offset 2) n-word-bytes)
+             instance-pointer-lowtag))
+    (inst lfsx (complex-single-reg-real-tn value) object offset)
+    (inst addi offset offset n-word-bytes)
+    (inst lfsx (complex-single-reg-imag-tn value) object offset)))
+
+(define-vop (raw-instance-set/complex-single)
+  (:translate %raw-instance-set/complex-single)
+  (:policy :fast-safe)
+  (:args (object :scs (descriptor-reg))
+         (index :scs (any-reg))
+         (value :scs (complex-single-reg) :target result))
+  (:arg-types * positive-fixnum complex-single-float)
+  (:results (result :scs (complex-single-reg)))
+  (:result-types complex-single-float)
+  (:temporary (:scs (non-descriptor-reg)) offset)
+  (:generator 5
+    (loadw offset object 0 instance-pointer-lowtag)
+    ;; offset = (offset >> n-widetag-bits) << 2
+    (inst rlwinm offset offset (- 32 (- n-widetag-bits 2)) (- n-widetag-bits 2) 29)
+    (inst subf offset index offset)
+    (inst addi
+          offset
+          offset
+          (- (* (- instance-slots-offset 2) n-word-bytes)
+             instance-pointer-lowtag))
+    (let ((value-real (complex-single-reg-real-tn value))
+          (result-real (complex-single-reg-real-tn result)))
+      (inst stfsx value-real object offset)
+      (unless (location= result-real value-real)
+        (inst frsp result-real value-real)))
+    (inst addi offset offset n-word-bytes)
+    (let ((value-imag (complex-single-reg-imag-tn value))
+          (result-imag (complex-single-reg-imag-tn result)))
+      (inst stfsx value-imag object offset)
+      (unless (location= result-imag value-imag)
+        (inst frsp result-imag value-imag)))))
+
+(define-vop (raw-instance-init/complex-double)
+  (:args (object :scs (descriptor-reg))
+         (value :scs (complex-double-reg)))
+  (:arg-types * complex-double-float)
+  (:info instance-length index)
+  (:generator 4
+    (inst stfd (complex-single-reg-real-tn value)
+          object (offset-for-raw-slot instance-length index 4))
+    (inst stfd (complex-double-reg-imag-tn value)
+          object (offset-for-raw-slot instance-length index 2))))
+
+(define-vop (raw-instance-ref/complex-double)
+  (:translate %raw-instance-ref/complex-double)
+  (:policy :fast-safe)
+  (:args (object :scs (descriptor-reg))
+         (index :scs (any-reg)))
+  (:arg-types * positive-fixnum)
+  (:results (value :scs (complex-double-reg)))
+  (:temporary (:scs (non-descriptor-reg)) offset)
+  (:result-types complex-double-float)
+  (:generator 5
+    (loadw offset object 0 instance-pointer-lowtag)
+    ;; offset = (offset >> n-widetag-bits) << 2
+    (inst rlwinm offset offset (- 32 (- n-widetag-bits 2)) (- n-widetag-bits 2) 29)
+    (inst subf offset index offset)
+    (inst addi
+          offset
+          offset
+          (- (* (- instance-slots-offset 4) n-word-bytes)
+             instance-pointer-lowtag))
+    (inst lfdx (complex-double-reg-real-tn value) object offset)
+    (inst addi offset offset (* 2 n-word-bytes))
+    (inst lfdx (complex-double-reg-imag-tn value) object offset)))
+
+(define-vop (raw-instance-set/complex-double)
+  (:translate %raw-instance-set/complex-double)
+  (:policy :fast-safe)
+  (:args (object :scs (descriptor-reg))
+         (index :scs (any-reg))
+         (value :scs (complex-double-reg) :target result))
+  (:arg-types * positive-fixnum complex-double-float)
+  (:results (result :scs (complex-double-reg)))
+  (:result-types complex-double-float)
+  (:temporary (:scs (non-descriptor-reg)) offset)
+  (:generator 5
+    (loadw offset object 0 instance-pointer-lowtag)
+    ;; offset = (offset >> n-widetag-bits) << 2
+    (inst rlwinm offset offset (- 32 (- n-widetag-bits 2)) (- n-widetag-bits 2) 29)
+    (inst subf offset index offset)
+    (inst addi
+          offset
+          offset
+          (- (* (- instance-slots-offset 4) n-word-bytes)
+             instance-pointer-lowtag))
+    (let ((value-real (complex-double-reg-real-tn value))
+          (result-real (complex-double-reg-real-tn result)))
+      (inst stfdx value-real object offset)
+      (unless (location= result-real value-real)
+        (inst fmr result-real value-real)))
+    (inst addi offset offset (* 2 n-word-bytes))
+    (let ((value-imag (complex-double-reg-imag-tn value))
+          (result-imag (complex-double-reg-imag-tn result)))
+      (inst stfdx value-imag object offset)
+      (unless (location= result-imag value-imag)
+        (inst fmr result-imag value-imag)))))