1.0.41.36: ppc: Implement atomic-{incf,decf} as atomic operations.

[sbcl.git] / src / compiler / ppc / cell.lisp

diff --git a/src/compiler/ppc/cell.lisp b/src/compiler/ppc/cell.lisp
index d1dc086..fda746e 100644 (file)
--- a/src/compiler/ppc/cell.lisp
+++ b/src/compiler/ppc/cell.lisp
@@ -24,18 +24,78 @@
 
 (define-vop (set-slot)
   (:args (object :scs (descriptor-reg))
 
 (define-vop (set-slot)
   (:args (object :scs (descriptor-reg))

-        (value :scs (descriptor-reg any-reg)))
+         (value :scs (descriptor-reg any-reg)))

   (:info name offset lowtag)
   (:ignore name)
   (:results)
   (:generator 1
     (storew value object offset lowtag)))
 
   (:info name offset lowtag)
   (:ignore name)
   (:results)
   (:generator 1
     (storew value object offset lowtag)))
 

+#!+compare-and-swap-vops
+(define-vop (compare-and-swap-slot)
+  (:args (object :scs (descriptor-reg))
+         (old :scs (descriptor-reg any-reg))
+         (new :scs (descriptor-reg any-reg)))
+  (:temporary (:sc non-descriptor-reg) temp)
+  (:info name offset lowtag)
+  (:ignore name)
+  (:results (result :scs (descriptor-reg) :from :load))
+  (:generator 5
+    (inst sync)
+    (inst li temp (- (* offset n-word-bytes) lowtag))
+    LOOP
+    (inst lwarx result temp object)
+    (inst cmpw result old)
+    (inst bne EXIT)
+    (inst stwcx. new temp object)
+    (inst bne LOOP)
+    EXIT
+    (inst isync)))
+

 \f
 ;;;; Symbol hacking VOPs:
 
 \f
 ;;;; Symbol hacking VOPs:
 

+#!+compare-and-swap-vops
+(define-vop (%compare-and-swap-symbol-value)
+  (:translate %compare-and-swap-symbol-value)
+  (:args (symbol :scs (descriptor-reg))
+         (old :scs (descriptor-reg any-reg))
+         (new :scs (descriptor-reg any-reg)))
+  (:temporary (:sc non-descriptor-reg) temp)
+  (:results (result :scs (descriptor-reg any-reg) :from :load))
+  (:policy :fast-safe)
+  (:vop-var vop)
+  (:generator 15
+    (inst sync)
+    #!+sb-thread
+    (assemble ()
+      (loadw temp symbol symbol-tls-index-slot other-pointer-lowtag)
+      ;; Thread-local area, no synchronization needed.
+      (inst lwzx result thread-base-tn temp)
+      (inst cmpw result old)
+      (inst bne DONT-STORE-TLS)
+      (inst stwx new thread-base-tn temp)
+      DONT-STORE-TLS
+
+      (inst cmpwi result no-tls-value-marker-widetag)
+      (inst bne CHECK-UNBOUND))
+
+    (inst li temp (- (* symbol-value-slot n-word-bytes)
+                     other-pointer-lowtag))
+    LOOP
+    (inst lwarx result symbol temp)
+    (inst cmpw result old)
+    (inst bne CHECK-UNBOUND)
+    (inst stwcx. new symbol temp)
+    (inst bne LOOP)
+
+    CHECK-UNBOUND
+    (inst isync)
+    (inst cmpwi result unbound-marker-widetag)
+    (inst beq (generate-error-code vop 'unbound-symbol-error symbol))))
+

 ;;; The compiler likes to be able to directly SET symbols.
 ;;; The compiler likes to be able to directly SET symbols.

-(define-vop (set cell-set)
+(define-vop (%set-symbol-global-value cell-set)

   (:variant symbol-value-slot other-pointer-lowtag))
 
 ;;; Do a cell ref with an error check for being unbound.
   (:variant symbol-value-slot other-pointer-lowtag))
 
 ;;; Do a cell ref with an error check for being unbound.

@@ -49,15 +109,81 @@
 
 ;;; With SYMBOL-VALUE, we check that the value isn't the trap object.
 ;;; So SYMBOL-VALUE of NIL is NIL.
 
 ;;; With SYMBOL-VALUE, we check that the value isn't the trap object.
 ;;; So SYMBOL-VALUE of NIL is NIL.

-(define-vop (symbol-value checked-cell-ref)
-  (:translate symbol-value)
+(define-vop (symbol-global-value checked-cell-ref)
+  (:translate symbol-global-value)

   (:generator 9
     (move obj-temp object)
     (loadw value obj-temp symbol-value-slot other-pointer-lowtag)
   (:generator 9
     (move obj-temp object)
     (loadw value obj-temp symbol-value-slot other-pointer-lowtag)

-    (let ((err-lab (generate-error-code vop unbound-symbol-error obj-temp)))
+    (let ((err-lab (generate-error-code vop 'unbound-symbol-error obj-temp)))

       (inst cmpwi value unbound-marker-widetag)
       (inst beq err-lab))))
 
       (inst cmpwi value unbound-marker-widetag)
       (inst beq err-lab))))
 

+(define-vop (fast-symbol-global-value cell-ref)
+  (:variant symbol-value-slot other-pointer-lowtag)
+  (:policy :fast)
+  (:translate symbol-global-value))
+
+#!+sb-thread
+(progn
+  (define-vop (set)
+    (:args (symbol :scs (descriptor-reg))
+           (value :scs (descriptor-reg any-reg)))
+    (:temporary (:sc any-reg) tls-slot temp)
+    (:generator 4
+      (loadw tls-slot symbol symbol-tls-index-slot other-pointer-lowtag)
+      (inst lwzx temp thread-base-tn tls-slot)
+      (inst cmpwi temp no-tls-value-marker-widetag)
+      (inst beq GLOBAL-VALUE)
+      (inst stwx value thread-base-tn tls-slot)
+      (inst b DONE)
+      GLOBAL-VALUE
+      (storew value symbol symbol-value-slot other-pointer-lowtag)
+      DONE))
+
+  ;; With Symbol-Value, we check that the value isn't the trap object. So
+  ;; Symbol-Value of NIL is NIL.
+  (define-vop (symbol-value)
+    (:translate symbol-value)
+    (:policy :fast-safe)
+    (:args (object :scs (descriptor-reg) :to (:result 1)))
+    (:results (value :scs (descriptor-reg any-reg)))
+    (:vop-var vop)
+    (:save-p :compute-only)
+    (:generator 9
+      (loadw value object symbol-tls-index-slot other-pointer-lowtag)
+      (inst lwzx value thread-base-tn value)
+      (inst cmpwi value no-tls-value-marker-widetag)
+      (inst bne CHECK-UNBOUND)
+      (loadw value object symbol-value-slot other-pointer-lowtag)
+      CHECK-UNBOUND
+      (inst cmpwi value unbound-marker-widetag)
+      (inst beq (generate-error-code vop 'unbound-symbol-error object))))
+
+  (define-vop (fast-symbol-value symbol-value)
+    ;; KLUDGE: not really fast, in fact, because we're going to have to
+    ;; do a full lookup of the thread-local area anyway.  But half of
+    ;; the meaning of FAST-SYMBOL-VALUE is "do not signal an error if
+    ;; unbound", which is used in the implementation of COPY-SYMBOL.  --
+    ;; CSR, 2003-04-22
+    (:policy :fast)
+    (:translate symbol-value)
+    (:generator 8
+      (loadw value object symbol-tls-index-slot other-pointer-lowtag)
+      (inst lwzx value thread-base-tn value)
+      (inst cmpwi value no-tls-value-marker-widetag)
+      (inst bne DONE)
+      (loadw value object symbol-value-slot other-pointer-lowtag)
+      DONE)))
+
+;;; On unithreaded builds these are just copies of the global versions.
+#!-sb-thread
+(progn
+  (define-vop (symbol-value symbol-global-value)
+    (:translate symbol-value))
+  (define-vop (fast-symbol-value fast-symbol-global-value)
+    (:translate symbol-value))
+  (define-vop (set %set-symbol-global-value)))
+

 ;;; Like CHECKED-CELL-REF, only we are a predicate to see if the cell
 ;;; is bound.
 (define-vop (boundp-frob)
 ;;; Like CHECKED-CELL-REF, only we are a predicate to see if the cell
 ;;; is bound.
 (define-vop (boundp-frob)

@@ -67,17 +193,26 @@
   (:policy :fast-safe)
   (:temporary (:scs (descriptor-reg)) value))
 
   (:policy :fast-safe)
   (:temporary (:scs (descriptor-reg)) value))
 

+#!+sb-thread

 (define-vop (boundp boundp-frob)
   (:translate boundp)
   (:generator 9
 (define-vop (boundp boundp-frob)
   (:translate boundp)
   (:generator 9

+    (loadw value object symbol-tls-index-slot other-pointer-lowtag)
+    (inst lwzx value thread-base-tn value)
+    (inst cmpwi value no-tls-value-marker-widetag)
+    (inst bne CHECK-UNBOUND)

     (loadw value object symbol-value-slot other-pointer-lowtag)
     (loadw value object symbol-value-slot other-pointer-lowtag)

+    CHECK-UNBOUND

     (inst cmpwi value unbound-marker-widetag)
     (inst b? (if not-p :eq :ne) target)))
 
     (inst cmpwi value unbound-marker-widetag)
     (inst b? (if not-p :eq :ne) target)))
 

-(define-vop (fast-symbol-value cell-ref)
-  (:variant symbol-value-slot other-pointer-lowtag)
-  (:policy :fast)
-  (:translate symbol-value))
+#!-sb-thread
+(define-vop (boundp boundp-frob)
+  (:translate boundp)
+  (:generator 9
+    (loadw value object symbol-value-slot other-pointer-lowtag)
+    (inst cmpwi value unbound-marker-widetag)
+    (inst b? (if not-p :eq :ne) target)))

 
 (define-vop (symbol-hash)
   (:policy :fast-safe)
 
 (define-vop (symbol-hash)
   (:policy :fast-safe)

@@ -91,7 +226,7 @@
     ;; it is a fixnum.  The lowtag selection magic that is required to
     ;; ensure this is explained in the comment in objdef.lisp
     (loadw res symbol symbol-hash-slot other-pointer-lowtag)
     ;; it is a fixnum.  The lowtag selection magic that is required to
     ;; ensure this is explained in the comment in objdef.lisp
     (loadw res symbol symbol-hash-slot other-pointer-lowtag)

-    (inst clrrwi res res (1- n-lowtag-bits))))
+    (inst clrrwi res res n-fixnum-tag-bits)))

 \f
 ;;;; Fdefinition (fdefn) objects.
 
 \f
 ;;;; Fdefinition (fdefn) objects.
 

@@ -108,14 +243,14 @@
     (move obj-temp object)
     (loadw value obj-temp fdefn-fun-slot other-pointer-lowtag)
     (inst cmpw value null-tn)
     (move obj-temp object)
     (loadw value obj-temp fdefn-fun-slot other-pointer-lowtag)
     (inst cmpw value null-tn)

-    (let ((err-lab (generate-error-code vop undefined-fun-error obj-temp)))
+    (let ((err-lab (generate-error-code vop 'undefined-fun-error obj-temp)))

       (inst beq err-lab))))
 
 (define-vop (set-fdefn-fun)
   (:policy :fast-safe)
   (:translate (setf fdefn-fun))
   (:args (function :scs (descriptor-reg) :target result)
       (inst beq err-lab))))
 
 (define-vop (set-fdefn-fun)
   (:policy :fast-safe)
   (:translate (setf fdefn-fun))
   (:args (function :scs (descriptor-reg) :target result)

-        (fdefn :scs (descriptor-reg)))
+         (fdefn :scs (descriptor-reg)))

   (:temporary (:scs (interior-reg)) lip)
   (:temporary (:scs (non-descriptor-reg)) type)
   (:results (result :scs (descriptor-reg)))
   (:temporary (:scs (interior-reg)) lip)
   (:temporary (:scs (non-descriptor-reg)) type)
   (:results (result :scs (descriptor-reg)))

@@ -125,7 +260,7 @@
       (inst cmpwi type simple-fun-header-widetag)
       ;;(inst mr lip function)
       (inst addi lip function
       (inst cmpwi type simple-fun-header-widetag)
       ;;(inst mr lip function)
       (inst addi lip function

-           (- (ash simple-fun-code-offset word-shift) fun-pointer-lowtag))
+            (- (ash simple-fun-code-offset word-shift) fun-pointer-lowtag))

       (inst beq normal-fn)
       (inst lr lip  (make-fixup "closure_tramp" :foreign))
       (emit-label normal-fn)
       (inst beq normal-fn)
       (inst lr lip  (make-fixup "closure_tramp" :foreign))
       (emit-label normal-fn)

@@ -153,9 +288,66 @@
 ;;; the symbol on the binding stack and stuff the new value into the
 ;;; symbol.
 
 ;;; the symbol on the binding stack and stuff the new value into the
 ;;; symbol.
 

+#!+sb-thread
+(define-vop (bind)
+  (:args (val :scs (any-reg descriptor-reg))
+         (symbol :scs (descriptor-reg)))
+  (:temporary (:sc non-descriptor-reg :offset nl3-offset) pa-flag)
+  (:temporary (:scs (descriptor-reg)) temp tls-index)
+  (:generator 5
+     (loadw tls-index symbol symbol-tls-index-slot other-pointer-lowtag)
+     (inst cmpwi tls-index 0)
+     (inst bne TLS-VALID)
+
+     ;; No TLS slot allocated, so allocate one.
+     (pseudo-atomic (pa-flag)
+       (without-scheduling ()
+         (assemble ()
+           (inst li temp (+ (static-symbol-offset '*tls-index-lock*)
+                            (ash symbol-value-slot word-shift)
+                            (- other-pointer-lowtag)))
+           OBTAIN-LOCK
+           (inst lwarx tls-index null-tn temp)
+           (inst cmpwi tls-index 0)
+           (inst bne OBTAIN-LOCK)
+           (inst stwcx. thread-base-tn null-tn temp)
+           (inst bne OBTAIN-LOCK)
+           (inst isync)
+
+           ;; Check to see if the TLS index was set while we were waiting.
+           (loadw tls-index symbol symbol-tls-index-slot other-pointer-lowtag)
+           (inst cmpwi tls-index 0)
+           (inst bne RELEASE-LOCK)
+
+           (load-symbol-value tls-index *free-tls-index*)
+           ;; FIXME: Check for TLS index overflow.
+           (inst addi tls-index tls-index n-word-bytes)
+           (store-symbol-value tls-index *free-tls-index*)
+           (inst addi tls-index tls-index (- n-word-bytes))
+           (storew tls-index symbol symbol-tls-index-slot other-pointer-lowtag)
+
+           ;; The sync instruction doesn't need to happen if we branch
+           ;; directly to RELEASE-LOCK as we didn't do any stores in that
+           ;; case.
+           (inst sync)
+           RELEASE-LOCK
+           (inst stwx zero-tn null-tn temp)
+
+           ;; temp is a boxed register, but we've been storing crap in it.
+           ;; fix it before we leave pseudo-atomic.
+           (inst li temp 0))))
+
+     TLS-VALID
+     (inst lwzx temp thread-base-tn tls-index)
+     (inst addi bsp-tn bsp-tn (* 2 n-word-bytes))
+     (storew temp bsp-tn (- binding-value-slot binding-size))
+     (storew symbol bsp-tn (- binding-symbol-slot binding-size))
+     (inst stwx val thread-base-tn tls-index)))
+
+#!-sb-thread

 (define-vop (bind)
   (:args (val :scs (any-reg descriptor-reg))
 (define-vop (bind)
   (:args (val :scs (any-reg descriptor-reg))

-        (symbol :scs (descriptor-reg)))
+         (symbol :scs (descriptor-reg)))

   (:temporary (:scs (descriptor-reg)) temp)
   (:generator 5
     (loadw temp symbol symbol-value-slot other-pointer-lowtag)
   (:temporary (:scs (descriptor-reg)) temp)
   (:generator 5
     (loadw temp symbol symbol-value-slot other-pointer-lowtag)

@@ -164,7 +356,19 @@
     (storew symbol bsp-tn (- binding-symbol-slot binding-size))
     (storew val symbol symbol-value-slot other-pointer-lowtag)))
 
     (storew symbol bsp-tn (- binding-symbol-slot binding-size))
     (storew val symbol symbol-value-slot other-pointer-lowtag)))
 

+#!+sb-thread
+(define-vop (unbind)
+  (:temporary (:scs (descriptor-reg)) tls-index value)
+  (:generator 0
+    (loadw tls-index bsp-tn (- binding-symbol-slot binding-size))
+    (loadw tls-index tls-index symbol-tls-index-slot other-pointer-lowtag)
+    (loadw value bsp-tn (- binding-value-slot binding-size))
+    (inst stwx value thread-base-tn tls-index)
+    (storew zero-tn bsp-tn (- binding-symbol-slot binding-size))
+    (storew zero-tn bsp-tn (- binding-value-slot binding-size))
+    (inst subi bsp-tn bsp-tn (* 2 n-word-bytes))))

 
 

+#!-sb-thread

 (define-vop (unbind)
   (:temporary (:scs (descriptor-reg)) symbol value)
   (:generator 0
 (define-vop (unbind)
   (:temporary (:scs (descriptor-reg)) symbol value)
   (:generator 0

@@ -172,6 +376,7 @@
     (loadw value bsp-tn (- binding-value-slot binding-size))
     (storew value symbol symbol-value-slot other-pointer-lowtag)
     (storew zero-tn bsp-tn (- binding-symbol-slot binding-size))
     (loadw value bsp-tn (- binding-value-slot binding-size))
     (storew value symbol symbol-value-slot other-pointer-lowtag)
     (storew zero-tn bsp-tn (- binding-symbol-slot binding-size))

+    (storew zero-tn bsp-tn (- binding-value-slot binding-size))

     (inst subi bsp-tn bsp-tn (* 2 n-word-bytes))))
 
 
     (inst subi bsp-tn bsp-tn (* 2 n-word-bytes))))
 
 

@@ -181,8 +386,8 @@
   (:temporary (:scs (descriptor-reg)) symbol value)
   (:generator 0
     (let ((loop (gen-label))
   (:temporary (:scs (descriptor-reg)) symbol value)
   (:generator 0
     (let ((loop (gen-label))

-         (skip (gen-label))
-         (done (gen-label)))
+          (skip (gen-label))
+          (done (gen-label)))

       (move where arg)
       (inst cmpw where bsp-tn)
       (inst beq done)
       (move where arg)
       (inst cmpw where bsp-tn)
       (inst beq done)

@@ -192,10 +397,16 @@
       (inst cmpwi symbol 0)
       (inst beq skip)
       (loadw value bsp-tn (- binding-value-slot binding-size))
       (inst cmpwi symbol 0)
       (inst beq skip)
       (loadw value bsp-tn (- binding-value-slot binding-size))

+      #!+sb-thread
+      (loadw symbol symbol symbol-tls-index-slot other-pointer-lowtag)
+      #!+sb-thread
+      (inst stwx value thread-base-tn symbol)
+      #!-sb-thread

       (storew value symbol symbol-value-slot other-pointer-lowtag)
       (storew zero-tn bsp-tn (- binding-symbol-slot binding-size))
 
       (emit-label skip)
       (storew value symbol symbol-value-slot other-pointer-lowtag)
       (storew zero-tn bsp-tn (- binding-symbol-slot binding-size))
 
       (emit-label skip)

+      (storew zero-tn bsp-tn (- binding-value-slot binding-size))

       (inst subi bsp-tn bsp-tn (* 2 n-word-bytes))
       (inst cmpw where bsp-tn)
       (inst bne loop)
       (inst subi bsp-tn bsp-tn (* 2 n-word-bytes))
       (inst cmpw where bsp-tn)
       (inst bne loop)

@@ -218,10 +429,6 @@
   (:variant funcallable-instance-info-offset fun-pointer-lowtag)
   (:translate %set-funcallable-instance-info))
 
   (:variant funcallable-instance-info-offset fun-pointer-lowtag)
   (:translate %set-funcallable-instance-info))
 

-(define-vop (funcallable-instance-lexenv cell-ref)
-  (:variant funcallable-instance-lexenv-slot fun-pointer-lowtag))
-
-

 (define-vop (closure-ref slot-ref)
   (:variant closure-info-offset fun-pointer-lowtag))
 
 (define-vop (closure-ref slot-ref)
   (:variant closure-info-offset fun-pointer-lowtag))
 

@@ -252,32 +459,24 @@
     (loadw temp struct 0 instance-pointer-lowtag)
     (inst srwi res temp n-widetag-bits)))
 
     (loadw temp struct 0 instance-pointer-lowtag)
     (inst srwi res temp n-widetag-bits)))
 

-(define-vop (instance-ref slot-ref)
-  (:variant instance-slots-offset instance-pointer-lowtag)
-  (:policy :fast-safe)
-  (:translate %instance-ref)
-  (:arg-types * (:constant index)))
-
-#+nil
-(define-vop (instance-set slot-set)
-  (:policy :fast-safe)
-  (:translate %instance-set)
-  (:variant instance-slots-offset instance-pointer-lowtag)
-  (:arg-types instance (:constant index) *))
-

 (define-vop (instance-index-ref word-index-ref)
 (define-vop (instance-index-ref word-index-ref)

-  (:policy :fast-safe) 
+  (:policy :fast-safe)

   (:translate %instance-ref)
   (:variant instance-slots-offset instance-pointer-lowtag)
   (:arg-types instance positive-fixnum))
 
 (define-vop (instance-index-set word-index-set)
   (:translate %instance-ref)
   (:variant instance-slots-offset instance-pointer-lowtag)
   (:arg-types instance positive-fixnum))
 
 (define-vop (instance-index-set word-index-set)

-  (:policy :fast-safe) 
+  (:policy :fast-safe)

   (:translate %instance-set)
   (:variant instance-slots-offset instance-pointer-lowtag)
   (:arg-types instance positive-fixnum *))
 
   (:translate %instance-set)
   (:variant instance-slots-offset instance-pointer-lowtag)
   (:arg-types instance positive-fixnum *))
 

-
+#!+compare-and-swap-vops
+(define-vop (%compare-and-swap-instance-ref word-index-cas)
+  (:policy :fast-safe)
+  (:translate %compare-and-swap-instance-ref)
+  (:variant instance-slots-offset instance-pointer-lowtag)
+  (:arg-types instance tagged-num * *))

 
 \f
 ;;;; Code object frobbing.
 
 \f
 ;;;; Code object frobbing.

@@ -296,19 +495,64 @@
 \f
 ;;;; raw instance slot accessors
 
 \f
 ;;;; raw instance slot accessors
 

+(defun offset-for-raw-slot (instance-length index n-words)
+  (+ (* (- instance-length instance-slots-offset index (1- n-words))
+        n-word-bytes)
+     (- instance-pointer-lowtag)))
+
+(define-vop (raw-instance-init/word)
+  (:args (object :scs (descriptor-reg))
+         (value :scs (unsigned-reg)))
+  (:arg-types * unsigned-num)
+  (:info instance-length index)
+  (:generator 4
+    (inst stw value object (offset-for-raw-slot instance-length index 1))))
+
+(define-vop (raw-instance-atomic-incf/word)
+  (:translate %raw-instance-atomic-incf/word)
+  (:policy :fast-safe)
+  (:args (object :scs (descriptor-reg))
+         (index :scs (any-reg))
+         (diff :scs (unsigned-reg)))
+  (:arg-types * positive-fixnum unsigned-num)
+  (:temporary (:sc unsigned-reg) offset)
+  (:temporary (:sc non-descriptor-reg) sum)
+  (:results (result :scs (unsigned-reg) :from :load))
+  (:result-types unsigned-num)
+  (:generator 4
+    (loadw offset object 0 instance-pointer-lowtag)
+    ;; offset = (offset >> n-widetag-bits) << 2
+    (inst rlwinm offset offset (- 32 (- n-widetag-bits 2)) (- n-widetag-bits 2) 29)
+    (inst subf offset index offset)
+    (inst addi
+          offset
+          offset
+          (- (* (1- instance-slots-offset) n-word-bytes)
+             instance-pointer-lowtag))
+    ;; load the slot value, add DIFF, write the sum back, and return
+    ;; the original slot value, atomically, and include a memory
+    ;; barrier.
+    (inst sync)
+    LOOP
+    (inst lwarx result offset object)
+    (inst add sum result diff)
+    (inst stwcx. sum offset object)
+    (inst bne LOOP)
+    (inst isync)))
+

 (define-vop (raw-instance-ref/word)
   (:translate %raw-instance-ref/word)
   (:policy :fast-safe)
   (:args (object :scs (descriptor-reg))
 (define-vop (raw-instance-ref/word)
   (:translate %raw-instance-ref/word)
   (:policy :fast-safe)
   (:args (object :scs (descriptor-reg))

-        (index :scs (any-reg)))
+         (index :scs (any-reg)))

   (:arg-types * positive-fixnum)
   (:results (value :scs (unsigned-reg)))
   (:temporary (:scs (non-descriptor-reg)) offset)
   (:result-types unsigned-num)
   (:generator 5
     (loadw offset object 0 instance-pointer-lowtag)
   (:arg-types * positive-fixnum)
   (:results (value :scs (unsigned-reg)))
   (:temporary (:scs (non-descriptor-reg)) offset)
   (:result-types unsigned-num)
   (:generator 5
     (loadw offset object 0 instance-pointer-lowtag)

-    (inst srwi offset offset n-widetag-bits)
-    (inst slwi offset offset 2)
+    ;; offset = (offset >> n-widetag-bits) << 2
+    (inst rlwinm offset offset (- 32 (- n-widetag-bits 2)) (- n-widetag-bits 2) 29)

     (inst subf offset index offset)
     (inst addi
           offset
     (inst subf offset index offset)
     (inst addi
           offset

@@ -321,7 +565,7 @@
   (:translate %raw-instance-set/word)
   (:policy :fast-safe)
   (:args (object :scs (descriptor-reg))
   (:translate %raw-instance-set/word)
   (:policy :fast-safe)
   (:args (object :scs (descriptor-reg))

-        (index :scs (any-reg))
+         (index :scs (any-reg))

          (value :scs (unsigned-reg)))
   (:arg-types * positive-fixnum unsigned-num)
   (:results (result :scs (unsigned-reg)))
          (value :scs (unsigned-reg)))
   (:arg-types * positive-fixnum unsigned-num)
   (:results (result :scs (unsigned-reg)))

@@ -329,8 +573,8 @@
   (:result-types unsigned-num)
   (:generator 5
     (loadw offset object 0 instance-pointer-lowtag)
   (:result-types unsigned-num)
   (:generator 5
     (loadw offset object 0 instance-pointer-lowtag)

-    (inst srwi offset offset n-widetag-bits)
-    (inst slwi offset offset 2)
+    ;; offset = (offset >> n-widetag-bits) << 2
+    (inst rlwinm offset offset (- 32 (- n-widetag-bits 2)) (- n-widetag-bits 2) 29)

     (inst subf offset index offset)
     (inst addi
           offset
     (inst subf offset index offset)
     (inst addi
           offset

@@ -340,19 +584,27 @@
     (inst stwx value object offset)
     (move result value)))
 
     (inst stwx value object offset)
     (move result value)))
 

+(define-vop (raw-instance-init/single)
+  (:args (object :scs (descriptor-reg))
+         (value :scs (single-reg)))
+  (:arg-types * single-float)
+  (:info instance-length index)
+  (:generator 4
+    (inst stfs value object (offset-for-raw-slot instance-length index 1))))
+

 (define-vop (raw-instance-ref/single)
   (:translate %raw-instance-ref/single)
   (:policy :fast-safe)
   (:args (object :scs (descriptor-reg))
 (define-vop (raw-instance-ref/single)
   (:translate %raw-instance-ref/single)
   (:policy :fast-safe)
   (:args (object :scs (descriptor-reg))

-        (index :scs (any-reg)))
+         (index :scs (any-reg)))

   (:arg-types * positive-fixnum)
   (:results (value :scs (single-reg)))
   (:temporary (:scs (non-descriptor-reg)) offset)
   (:result-types single-float)
   (:generator 5
     (loadw offset object 0 instance-pointer-lowtag)
   (:arg-types * positive-fixnum)
   (:results (value :scs (single-reg)))
   (:temporary (:scs (non-descriptor-reg)) offset)
   (:result-types single-float)
   (:generator 5
     (loadw offset object 0 instance-pointer-lowtag)

-    (inst srwi offset offset n-widetag-bits)
-    (inst slwi offset offset 2)
+    ;; offset = (offset >> n-widetag-bits) << 2
+    (inst rlwinm offset offset (- 32 (- n-widetag-bits 2)) (- n-widetag-bits 2) 29)

     (inst subf offset index offset)
     (inst addi
           offset
     (inst subf offset index offset)
     (inst addi
           offset

@@ -365,16 +617,16 @@
   (:translate %raw-instance-set/single)
   (:policy :fast-safe)
   (:args (object :scs (descriptor-reg))
   (:translate %raw-instance-set/single)
   (:policy :fast-safe)
   (:args (object :scs (descriptor-reg))

-        (index :scs (any-reg))
-        (value :scs (single-reg) :target result))
+         (index :scs (any-reg))
+         (value :scs (single-reg) :target result))

   (:arg-types * positive-fixnum single-float)
   (:results (result :scs (single-reg)))
   (:result-types single-float)
   (:temporary (:scs (non-descriptor-reg)) offset)
   (:generator 5
     (loadw offset object 0 instance-pointer-lowtag)
   (:arg-types * positive-fixnum single-float)
   (:results (result :scs (single-reg)))
   (:result-types single-float)
   (:temporary (:scs (non-descriptor-reg)) offset)
   (:generator 5
     (loadw offset object 0 instance-pointer-lowtag)

-    (inst srwi offset offset n-widetag-bits)
-    (inst slwi offset offset 2)
+    ;; offset = (offset >> n-widetag-bits) << 2
+    (inst rlwinm offset offset (- 32 (- n-widetag-bits 2)) (- n-widetag-bits 2) 29)

     (inst subf offset index offset)
     (inst addi
           offset
     (inst subf offset index offset)
     (inst addi
           offset

@@ -385,19 +637,27 @@
     (unless (location= result value)
       (inst frsp result value))))
 
     (unless (location= result value)
       (inst frsp result value))))
 

+(define-vop (raw-instance-init/double)
+  (:args (object :scs (descriptor-reg))
+         (value :scs (double-reg)))
+  (:arg-types * double-float)
+  (:info instance-length index)
+  (:generator 4
+    (inst stfd value object (offset-for-raw-slot instance-length index 2))))
+

 (define-vop (raw-instance-ref/double)
   (:translate %raw-instance-ref/double)
   (:policy :fast-safe)
   (:args (object :scs (descriptor-reg))
 (define-vop (raw-instance-ref/double)
   (:translate %raw-instance-ref/double)
   (:policy :fast-safe)
   (:args (object :scs (descriptor-reg))

-        (index :scs (any-reg)))
+         (index :scs (any-reg)))

   (:arg-types * positive-fixnum)
   (:results (value :scs (double-reg)))
   (:temporary (:scs (non-descriptor-reg)) offset)
   (:result-types double-float)
   (:generator 5
     (loadw offset object 0 instance-pointer-lowtag)
   (:arg-types * positive-fixnum)
   (:results (value :scs (double-reg)))
   (:temporary (:scs (non-descriptor-reg)) offset)
   (:result-types double-float)
   (:generator 5
     (loadw offset object 0 instance-pointer-lowtag)

-    (inst srwi offset offset n-widetag-bits)
-    (inst slwi offset offset 2)
+    ;; offset = (offset >> n-widetag-bits) << 2
+    (inst rlwinm offset offset (- 32 (- n-widetag-bits 2)) (- n-widetag-bits 2) 29)

     (inst subf offset index offset)
     (inst addi
           offset
     (inst subf offset index offset)
     (inst addi
           offset

@@ -410,16 +670,16 @@
   (:translate %raw-instance-set/double)
   (:policy :fast-safe)
   (:args (object :scs (descriptor-reg))
   (:translate %raw-instance-set/double)
   (:policy :fast-safe)
   (:args (object :scs (descriptor-reg))

-        (index :scs (any-reg))
-        (value :scs (double-reg) :target result))
+         (index :scs (any-reg))
+         (value :scs (double-reg) :target result))

   (:arg-types * positive-fixnum double-float)
   (:results (result :scs (double-reg)))
   (:result-types double-float)
   (:temporary (:scs (non-descriptor-reg)) offset)
   (:generator 5
     (loadw offset object 0 instance-pointer-lowtag)
   (:arg-types * positive-fixnum double-float)
   (:results (result :scs (double-reg)))
   (:result-types double-float)
   (:temporary (:scs (non-descriptor-reg)) offset)
   (:generator 5
     (loadw offset object 0 instance-pointer-lowtag)

-    (inst srwi offset offset n-widetag-bits)
-    (inst slwi offset offset 2)
+    ;; offset = (offset >> n-widetag-bits) << 2
+    (inst rlwinm offset offset (- 32 (- n-widetag-bits 2)) (- n-widetag-bits 2) 29)

     (inst subf offset index offset)
     (inst addi
           offset
     (inst subf offset index offset)
     (inst addi
           offset

@@ -430,19 +690,30 @@
     (unless (location= result value)
       (inst fmr result value))))
 
     (unless (location= result value)
       (inst fmr result value))))
 

+(define-vop (raw-instance-init/complex-single)
+  (:args (object :scs (descriptor-reg))
+         (value :scs (complex-single-reg)))
+  (:arg-types * complex-single-float)
+  (:info instance-length index)
+  (:generator 4
+    (inst stfs (complex-single-reg-real-tn value)
+          object (offset-for-raw-slot instance-length index 2))
+    (inst stfs (complex-single-reg-imag-tn value)
+          object (offset-for-raw-slot instance-length index 1))))
+

 (define-vop (raw-instance-ref/complex-single)
   (:translate %raw-instance-ref/complex-single)
   (:policy :fast-safe)
   (:args (object :scs (descriptor-reg))
 (define-vop (raw-instance-ref/complex-single)
   (:translate %raw-instance-ref/complex-single)
   (:policy :fast-safe)
   (:args (object :scs (descriptor-reg))

-        (index :scs (any-reg)))
+         (index :scs (any-reg)))

   (:arg-types * positive-fixnum)
   (:results (value :scs (complex-single-reg)))
   (:temporary (:scs (non-descriptor-reg)) offset)
   (:result-types complex-single-float)
   (:generator 5
     (loadw offset object 0 instance-pointer-lowtag)
   (:arg-types * positive-fixnum)
   (:results (value :scs (complex-single-reg)))
   (:temporary (:scs (non-descriptor-reg)) offset)
   (:result-types complex-single-float)
   (:generator 5
     (loadw offset object 0 instance-pointer-lowtag)

-    (inst srwi offset offset n-widetag-bits)
-    (inst slwi offset offset 2)
+    ;; offset = (offset >> n-widetag-bits) << 2
+    (inst rlwinm offset offset (- 32 (- n-widetag-bits 2)) (- n-widetag-bits 2) 29)

     (inst subf offset index offset)
     (inst addi
           offset
     (inst subf offset index offset)
     (inst addi
           offset

@@ -457,16 +728,16 @@
   (:translate %raw-instance-set/complex-single)
   (:policy :fast-safe)
   (:args (object :scs (descriptor-reg))
   (:translate %raw-instance-set/complex-single)
   (:policy :fast-safe)
   (:args (object :scs (descriptor-reg))

-        (index :scs (any-reg))
-        (value :scs (complex-single-reg) :target result))
+         (index :scs (any-reg))
+         (value :scs (complex-single-reg) :target result))

   (:arg-types * positive-fixnum complex-single-float)
   (:results (result :scs (complex-single-reg)))
   (:result-types complex-single-float)
   (:temporary (:scs (non-descriptor-reg)) offset)
   (:generator 5
     (loadw offset object 0 instance-pointer-lowtag)
   (:arg-types * positive-fixnum complex-single-float)
   (:results (result :scs (complex-single-reg)))
   (:result-types complex-single-float)
   (:temporary (:scs (non-descriptor-reg)) offset)
   (:generator 5
     (loadw offset object 0 instance-pointer-lowtag)

-    (inst srwi offset offset n-widetag-bits)
-    (inst slwi offset offset 2)
+    ;; offset = (offset >> n-widetag-bits) << 2
+    (inst rlwinm offset offset (- 32 (- n-widetag-bits 2)) (- n-widetag-bits 2) 29)

     (inst subf offset index offset)
     (inst addi
           offset
     (inst subf offset index offset)
     (inst addi
           offset

@@ -474,30 +745,41 @@
           (- (* (- instance-slots-offset 2) n-word-bytes)
              instance-pointer-lowtag))
     (let ((value-real (complex-single-reg-real-tn value))
           (- (* (- instance-slots-offset 2) n-word-bytes)
              instance-pointer-lowtag))
     (let ((value-real (complex-single-reg-real-tn value))

-         (result-real (complex-single-reg-real-tn result)))
+          (result-real (complex-single-reg-real-tn result)))

       (inst stfsx value-real object offset)
       (unless (location= result-real value-real)
         (inst frsp result-real value-real)))
     (inst addi offset offset n-word-bytes)
     (let ((value-imag (complex-single-reg-imag-tn value))
       (inst stfsx value-real object offset)
       (unless (location= result-real value-real)
         (inst frsp result-real value-real)))
     (inst addi offset offset n-word-bytes)
     (let ((value-imag (complex-single-reg-imag-tn value))

-         (result-imag (complex-single-reg-imag-tn result)))
+          (result-imag (complex-single-reg-imag-tn result)))

       (inst stfsx value-imag object offset)
       (unless (location= result-imag value-imag)
         (inst frsp result-imag value-imag)))))
 
       (inst stfsx value-imag object offset)
       (unless (location= result-imag value-imag)
         (inst frsp result-imag value-imag)))))
 

+(define-vop (raw-instance-init/complex-double)
+  (:args (object :scs (descriptor-reg))
+         (value :scs (complex-double-reg)))
+  (:arg-types * complex-double-float)
+  (:info instance-length index)
+  (:generator 4
+    (inst stfd (complex-single-reg-real-tn value)
+          object (offset-for-raw-slot instance-length index 4))
+    (inst stfd (complex-double-reg-imag-tn value)
+          object (offset-for-raw-slot instance-length index 2))))
+

 (define-vop (raw-instance-ref/complex-double)
   (:translate %raw-instance-ref/complex-double)
   (:policy :fast-safe)
   (:args (object :scs (descriptor-reg))
 (define-vop (raw-instance-ref/complex-double)
   (:translate %raw-instance-ref/complex-double)
   (:policy :fast-safe)
   (:args (object :scs (descriptor-reg))

-        (index :scs (any-reg)))
+         (index :scs (any-reg)))

   (:arg-types * positive-fixnum)
   (:results (value :scs (complex-double-reg)))
   (:temporary (:scs (non-descriptor-reg)) offset)
   (:result-types complex-double-float)
   (:generator 5
     (loadw offset object 0 instance-pointer-lowtag)
   (:arg-types * positive-fixnum)
   (:results (value :scs (complex-double-reg)))
   (:temporary (:scs (non-descriptor-reg)) offset)
   (:result-types complex-double-float)
   (:generator 5
     (loadw offset object 0 instance-pointer-lowtag)

-    (inst srwi offset offset n-widetag-bits)
-    (inst slwi offset offset 2)
+    ;; offset = (offset >> n-widetag-bits) << 2
+    (inst rlwinm offset offset (- 32 (- n-widetag-bits 2)) (- n-widetag-bits 2) 29)

     (inst subf offset index offset)
     (inst addi
           offset
     (inst subf offset index offset)
     (inst addi
           offset

@@ -512,16 +794,16 @@
   (:translate %raw-instance-set/complex-double)
   (:policy :fast-safe)
   (:args (object :scs (descriptor-reg))
   (:translate %raw-instance-set/complex-double)
   (:policy :fast-safe)
   (:args (object :scs (descriptor-reg))

-        (index :scs (any-reg))
-        (value :scs (complex-double-reg) :target result))
+         (index :scs (any-reg))
+         (value :scs (complex-double-reg) :target result))

   (:arg-types * positive-fixnum complex-double-float)
   (:results (result :scs (complex-double-reg)))
   (:result-types complex-double-float)
   (:temporary (:scs (non-descriptor-reg)) offset)
   (:generator 5
     (loadw offset object 0 instance-pointer-lowtag)
   (:arg-types * positive-fixnum complex-double-float)
   (:results (result :scs (complex-double-reg)))
   (:result-types complex-double-float)
   (:temporary (:scs (non-descriptor-reg)) offset)
   (:generator 5
     (loadw offset object 0 instance-pointer-lowtag)

-    (inst srwi offset offset n-widetag-bits)
-    (inst slwi offset offset 2)
+    ;; offset = (offset >> n-widetag-bits) << 2
+    (inst rlwinm offset offset (- 32 (- n-widetag-bits 2)) (- n-widetag-bits 2) 29)

     (inst subf offset index offset)
     (inst addi
           offset
     (inst subf offset index offset)
     (inst addi
           offset

@@ -529,13 +811,13 @@
           (- (* (- instance-slots-offset 4) n-word-bytes)
              instance-pointer-lowtag))
     (let ((value-real (complex-double-reg-real-tn value))
           (- (* (- instance-slots-offset 4) n-word-bytes)
              instance-pointer-lowtag))
     (let ((value-real (complex-double-reg-real-tn value))

-         (result-real (complex-double-reg-real-tn result)))
+          (result-real (complex-double-reg-real-tn result)))

       (inst stfdx value-real object offset)
       (unless (location= result-real value-real)
         (inst fmr result-real value-real)))
     (inst addi offset offset (* 2 n-word-bytes))
     (let ((value-imag (complex-double-reg-imag-tn value))
       (inst stfdx value-real object offset)
       (unless (location= result-real value-real)
         (inst fmr result-real value-real)))
     (inst addi offset offset (* 2 n-word-bytes))
     (let ((value-imag (complex-double-reg-imag-tn value))

-         (result-imag (complex-double-reg-imag-tn result)))
+          (result-imag (complex-double-reg-imag-tn result)))

       (inst stfdx value-imag object offset)
       (unless (location= result-imag value-imag)
         (inst fmr result-imag value-imag)))))
       (inst stfdx value-imag object offset)
       (unless (location= result-imag value-imag)
         (inst fmr result-imag value-imag)))))