;;;; provided with absolutely no warranty. See the COPYING and CREDITS
;;;; files for more information.
-(in-package "SB!C")
+(in-package "SB!REGALLOC")
;;; for debugging: some parameters controlling which optimizations we
;;; attempt
\f
;;;; conflict determination
-;;; Return true if the element at the specified offset in SB has a
-;;; conflict with TN:
+;;; Return true if the element at the specified offset, or in any of
+;;; the [size-1] subsequent offsets, in SB has a conflict with TN:
;;; -- If a component-live TN (:COMPONENT kind), then iterate over
;;; all the blocks. If the element at OFFSET is used anywhere in
;;; any of the component's blocks (always-live /= 0), then there
;;; that block.
;;; -- If TN is local, then we just check for a conflict in the block
;;; it is local to.
-(defun offset-conflicts-in-sb (tn sb offset)
- (declare (type tn tn) (type finite-sb sb) (type index offset))
+;;;
+;;; If there is a conflict, returns the first such conflicting offset.
+(defun offset-conflicts-in-sb (tn sb offset &key (size 1))
+ (declare (type tn tn) (type finite-sb sb) (type index offset size))
(let ((confs (tn-global-conflicts tn))
- (kind (tn-kind tn)))
- (cond
- ((eq kind :component)
- (let ((loc-live (svref (finite-sb-always-live sb) offset)))
- (dotimes (i (ir2-block-count *component-being-compiled*) nil)
- (when (/= (sbit loc-live i) 0)
- (return t)))))
- (confs
- (let ((loc-confs (svref (finite-sb-conflicts sb) offset))
- (loc-live (svref (finite-sb-always-live sb) offset)))
- (do ((conf confs (global-conflicts-next-tnwise conf)))
- ((null conf)
- nil)
- (let* ((block (global-conflicts-block conf))
- (num (ir2-block-number block)))
- (if (eq (global-conflicts-kind conf) :live)
- (when (/= (sbit loc-live num) 0)
- (return t))
- (when (/= (sbit (svref loc-confs num)
- (global-conflicts-number conf))
- 0)
- (return t)))))))
- (t
- (/= (sbit (svref (svref (finite-sb-conflicts sb) offset)
- (ir2-block-number (tn-local tn)))
- (tn-local-number tn))
- 0)))))
+ (kind (tn-kind tn))
+ (sb-conflicts (finite-sb-conflicts sb))
+ (sb-always-live (finite-sb-always-live sb)))
+ (macrolet ((do-offsets (&body body)
+ `(loop repeat size
+ for offset upfrom offset
+ thereis (progn ,@body))))
+ (cond
+ ((eq kind :component)
+ (do-offsets
+ (let ((loc-live (svref sb-always-live offset)))
+ (dotimes (i (ir2-block-count *component-being-compiled*))
+ (when (/= (sbit loc-live i) 0)
+ (return offset))))))
+ (confs
+ ;; TN is global, iterate over the blocks TN is live in.
+ (do ((conf confs (global-conflicts-next-tnwise conf)))
+ ((null conf)
+ nil)
+ (let* ((block (global-conflicts-block conf))
+ (num (ir2-block-number block)))
+ (if (eq (global-conflicts-kind conf) :live)
+ (do-offsets
+ (let ((loc-live (svref sb-always-live offset)))
+ (when (/= (sbit loc-live num) 0)
+ (return-from offset-conflicts-in-sb offset))))
+ (do-offsets
+ (let ((loc-confs (svref sb-conflicts offset)))
+ (when (/= (sbit (svref loc-confs num)
+ (global-conflicts-number conf))
+ 0)
+ (return-from offset-conflicts-in-sb offset))))))))
+ (t
+ (do-offsets
+ (and (/= (sbit (svref (svref sb-conflicts offset)
+ (ir2-block-number (tn-local tn)))
+ (tn-local-number tn))
+ 0)
+ offset)))))))
;;; Return true if TN has a conflict in SC at the specified offset.
+(declaim (ftype (function (tn sc index) (values (or null index) &optional))
+ conflicts-in-sc))
(defun conflicts-in-sc (tn sc offset)
(declare (type tn tn) (type sc sc) (type index offset))
- (let ((sb (sc-sb sc)))
- (dotimes (i (sc-element-size sc) nil)
- (when (offset-conflicts-in-sb tn sb (+ offset i))
- (return t)))))
+ (offset-conflicts-in-sb tn (sc-sb sc) offset
+ :size (sc-element-size sc)))
;;; Add TN's conflicts into the conflicts for the location at OFFSET
;;; in SC. We iterate over each location in TN, adding to the
;;; the bits, otherwise we OR in the local conflict bits.
;;; -- If the TN is local, then we just do the block it is local to,
;;; setting always-live and OR'ing in the local conflicts.
-(defun add-location-conflicts (tn sc offset)
+(defun add-location-conflicts (tn sc offset optimize)
(declare (type tn tn) (type sc sc) (type index offset))
(let ((confs (tn-global-conflicts tn))
- (sb (sc-sb sc))
- (kind (tn-kind tn)))
+ (sb (sc-sb sc))
+ (kind (tn-kind tn)))
(dotimes (i (sc-element-size sc))
(declare (type index i))
(let* ((this-offset (+ offset i))
- (loc-confs (svref (finite-sb-conflicts sb) this-offset))
- (loc-live (svref (finite-sb-always-live sb) this-offset)))
- (cond
- ((eq kind :component)
- (dotimes (num (ir2-block-count *component-being-compiled*))
- (declare (type index num))
- (setf (sbit loc-live num) 1)
- (set-bit-vector (svref loc-confs num))))
- (confs
- (do ((conf confs (global-conflicts-next-tnwise conf)))
- ((null conf))
- (let* ((block (global-conflicts-block conf))
- (num (ir2-block-number block))
- (local-confs (svref loc-confs num)))
- (declare (type local-tn-bit-vector local-confs))
- (setf (sbit loc-live num) 1)
- (if (eq (global-conflicts-kind conf) :live)
- (set-bit-vector local-confs)
- (bit-ior local-confs (global-conflicts-conflicts conf) t)))))
- (t
- (let ((num (ir2-block-number (tn-local tn))))
- (setf (sbit loc-live num) 1)
- (bit-ior (the local-tn-bit-vector (svref loc-confs num))
- (tn-local-conflicts tn) t))))))))
+ (loc-confs (svref (finite-sb-conflicts sb) this-offset))
+ (loc-live (svref (finite-sb-always-live sb) this-offset)))
+ (cond
+ ((eq kind :component)
+ (dotimes (num (ir2-block-count *component-being-compiled*))
+ (declare (type index num))
+ (setf (sbit loc-live num) 1)
+ (set-bit-vector (svref loc-confs num))))
+ (confs
+ (do ((conf confs (global-conflicts-next-tnwise conf)))
+ ((null conf))
+ (let* ((block (global-conflicts-block conf))
+ (num (ir2-block-number block))
+ (local-confs (svref loc-confs num)))
+ (declare (type local-tn-bit-vector local-confs))
+ (setf (sbit loc-live num) 1)
+ (if (eq (global-conflicts-kind conf) :live)
+ (set-bit-vector local-confs)
+ (bit-ior local-confs (global-conflicts-conflicts conf) t)))))
+ (t
+ (let ((num (ir2-block-number (tn-local tn))))
+ (setf (sbit loc-live num) 1)
+ (bit-ior (the local-tn-bit-vector (svref loc-confs num))
+ (tn-local-conflicts tn) t))))
+ ;; Calculating ALWAYS-LIVE-COUNT is moderately expensive, and
+ ;; currently the information isn't used unless (> SPEED
+ ;; COMPILE-SPEED).
+ (when optimize
+ (setf (svref (finite-sb-always-live-count sb) this-offset)
+ (find-location-usage sb this-offset))))))
+ (values))
+
+;; A rought measure of how much a given OFFSET in SB is currently
+;; used. Current implementation counts the amount of blocks where the
+;; offset has been marked as ALWAYS-LIVE.
+(defun find-location-usage (sb offset)
+ (declare (optimize speed))
+ (declare (type sb sb) (type index offset))
+ (let* ((always-live (svref (finite-sb-always-live sb) offset)))
+ (declare (simple-bit-vector always-live))
+ (count 1 always-live)))
;;; Return the total number of IR2-BLOCKs in COMPONENT.
(defun ir2-block-count (component)
(declare (type component component))
(do ((2block (block-info (block-next (component-head component)))
- (ir2-block-next 2block)))
+ (ir2-block-next 2block)))
((null 2block)
(error "What? No ir2 blocks have a non-nil number?"))
(when (ir2-block-number 2block)
(let ((nblocks (ir2-block-count component)))
(dolist (sb *backend-sb-list*)
(unless (eq (sb-kind sb) :non-packed)
- (let* ((conflicts (finite-sb-conflicts sb))
- (always-live (finite-sb-always-live sb))
- (max-locs (length conflicts))
- (last-count (finite-sb-last-block-count sb)))
- (unless (zerop max-locs)
- (let ((current-size (length (the simple-vector
- (svref conflicts 0)))))
- (cond
- ((> nblocks current-size)
- (let ((new-size (max nblocks (* current-size 2))))
- (declare (type index new-size))
- (dotimes (i max-locs)
- (declare (type index i))
- (let ((new-vec (make-array new-size)))
- (let ((old (svref conflicts i)))
- (declare (simple-vector old))
- (dotimes (j current-size)
- (declare (type index j))
- (setf (svref new-vec j)
- (clear-bit-vector (svref old j)))))
-
- (do ((j current-size (1+ j)))
- ((= j new-size))
- (declare (type index j))
- (setf (svref new-vec j)
- (make-array local-tn-limit :element-type 'bit
- :initial-element 0)))
- (setf (svref conflicts i) new-vec))
- (setf (svref always-live i)
- (make-array new-size :element-type 'bit
- :initial-element 0)))))
- (t
- (dotimes (i (finite-sb-current-size sb))
- (declare (type index i))
- (let ((conf (svref conflicts i)))
- (declare (simple-vector conf))
- (dotimes (j last-count)
- (declare (type index j))
- (clear-bit-vector (svref conf j))))
- (clear-bit-vector (svref always-live i)))))))
-
- (setf (finite-sb-last-block-count sb) nblocks)
- (setf (finite-sb-current-size sb) (sb-size sb))
- (setf (finite-sb-last-offset sb) 0))))))
+ (let* ((conflicts (finite-sb-conflicts sb))
+ (always-live (finite-sb-always-live sb))
+ (always-live-count (finite-sb-always-live-count sb))
+ (max-locs (length conflicts))
+ (last-count (finite-sb-last-block-count sb)))
+ (unless (zerop max-locs)
+ (let ((current-size (length (the simple-vector
+ (svref conflicts 0)))))
+ (cond
+ ((> nblocks current-size)
+ (let ((new-size (max nblocks (* current-size 2))))
+ (declare (type index new-size))
+ (dotimes (i max-locs)
+ (declare (type index i))
+ (let ((new-vec (make-array new-size)))
+ (let ((old (svref conflicts i)))
+ (declare (simple-vector old))
+ (dotimes (j current-size)
+ (declare (type index j))
+ (setf (svref new-vec j)
+ (clear-bit-vector (svref old j)))))
+
+ (do ((j current-size (1+ j)))
+ ((= j new-size))
+ (declare (type index j))
+ (setf (svref new-vec j)
+ (make-array local-tn-limit :element-type 'bit
+ :initial-element 0)))
+ (setf (svref conflicts i) new-vec))
+ (setf (svref always-live i)
+ (make-array new-size :element-type 'bit
+ :initial-element 0))
+ (setf (svref always-live-count i) 0))))
+ (t
+ (dotimes (i (finite-sb-current-size sb))
+ (declare (type index i))
+ (let ((conf (svref conflicts i)))
+ (declare (simple-vector conf))
+ (dotimes (j last-count)
+ (declare (type index j))
+ (clear-bit-vector (svref conf j))))
+ (clear-bit-vector (svref always-live i))
+ (setf (svref always-live-count i) 0))))))
+
+ (setf (finite-sb-last-block-count sb) nblocks)
+ (setf (finite-sb-current-size sb) (sb-size sb))
+ (setf (finite-sb-last-offset sb) 0))))))
;;; Expand the :UNBOUNDED SB backing SC by either the initial size or
;;; the SC element size, whichever is larger. If NEEDED-SIZE is
(defun grow-sc (sc &optional (needed-size 0))
(declare (type sc sc) (type index needed-size))
(let* ((sb (sc-sb sc))
- (size (finite-sb-current-size sb))
- (align-mask (1- (sc-alignment sc)))
- (inc (max (sb-size sb)
- (+ (sc-element-size sc)
- (- (logandc2 (+ size align-mask) align-mask)
- size))
- (- needed-size size)))
- (new-size (+ size inc))
- (conflicts (finite-sb-conflicts sb))
- (block-size (if (zerop (length conflicts))
- (ir2-block-count *component-being-compiled*)
- (length (the simple-vector (svref conflicts 0))))))
- (declare (type index inc new-size))
+ (size (finite-sb-current-size sb))
+ (align-mask (1- (sc-alignment sc)))
+ (inc (max (finite-sb-size-increment sb)
+ (+ (sc-element-size sc)
+ (- (logandc2 (+ size align-mask) align-mask)
+ size))
+ (- needed-size size)))
+ (new-size (let ((align-mask (1- (finite-sb-size-alignment sb))))
+ (logandc2 (+ size inc align-mask) align-mask)))
+ (conflicts (finite-sb-conflicts sb))
+ (block-size (if (zerop (length conflicts))
+ (ir2-block-count *component-being-compiled*)
+ (length (the simple-vector (svref conflicts 0)))))
+ (padded-size (ash 1 (integer-length (1- new-size)))))
+ (declare (type index inc new-size padded-size))
(aver (eq (sb-kind sb) :unbounded))
- (when (> new-size (length conflicts))
- (let ((new-conf (make-array new-size)))
- (replace new-conf conflicts)
- (do ((i size (1+ i)))
- ((= i new-size))
- (declare (type index i))
- (let ((loc-confs (make-array block-size)))
- (dotimes (j block-size)
- (setf (svref loc-confs j)
- (make-array local-tn-limit
- :initial-element 0
- :element-type 'bit)))
- (setf (svref new-conf i) loc-confs)))
- (setf (finite-sb-conflicts sb) new-conf))
-
- (let ((new-live (make-array new-size)))
- (replace new-live (finite-sb-always-live sb))
- (do ((i size (1+ i)))
- ((= i new-size))
- (setf (svref new-live i)
- (make-array block-size
- :initial-element 0
- :element-type 'bit)))
- (setf (finite-sb-always-live sb) new-live))
-
- (let ((new-tns (make-array new-size :initial-element nil)))
- (replace new-tns (finite-sb-live-tns sb))
- (fill (finite-sb-live-tns sb) nil)
- (setf (finite-sb-live-tns sb) new-tns)))
+ (when (> padded-size (length conflicts))
+ (let ((new-conf (make-array padded-size)))
+ (replace new-conf conflicts)
+ (do ((i size (1+ i)))
+ ((= i padded-size))
+ (declare (type index i))
+ (let ((loc-confs (make-array block-size)))
+ (dotimes (j block-size)
+ (setf (svref loc-confs j)
+ (make-array local-tn-limit
+ :initial-element 0
+ :element-type 'bit)))
+ (setf (svref new-conf i) loc-confs)))
+ (setf (finite-sb-conflicts sb) new-conf))
+
+ (let ((new-live (make-array padded-size)))
+ (replace new-live (finite-sb-always-live sb))
+ (do ((i size (1+ i)))
+ ((= i padded-size))
+ (setf (svref new-live i)
+ (make-array block-size
+ :initial-element 0
+ :element-type 'bit)))
+ (setf (finite-sb-always-live sb) new-live))
+
+ (let ((new-live-count (make-array padded-size)))
+ (declare (optimize speed)) ;; FILL deftransform
+ (replace new-live-count (finite-sb-always-live-count sb))
+ (fill new-live-count 0 :start size)
+ (setf (finite-sb-always-live-count sb) new-live-count))
+
+ (let ((new-tns (make-array padded-size :initial-element nil)))
+ (replace new-tns (finite-sb-live-tns sb))
+ (fill (finite-sb-live-tns sb) nil)
+ (setf (finite-sb-live-tns sb) new-tns)))
(setf (finite-sb-current-size sb) new-size))
(values))
-;;; This variable is true whenever we are in pack (and thus the per-SB
-;;; conflicts information is in use.)
-(defvar *in-pack* nil)
-
-;;; In order to prevent the conflict data structures from growing
-;;; arbitrarily large, we clear them whenever a GC happens and we
-;;; aren't currently in pack. We revert to the initial number of
-;;; locations and 0 blocks.
-(defun pack-before-gc-hook ()
- (unless *in-pack*
- (dolist (sb *backend-sb-list*)
- (unless (eq (sb-kind sb) :non-packed)
- (let ((size (sb-size sb)))
- (fill (finite-sb-always-live sb) nil)
- (setf (finite-sb-always-live sb)
- (make-array size
- :initial-element
- #-sb-xc #*
- ;; The cross-compiler isn't very good at
- ;; dumping specialized arrays, so we delay
- ;; construction of this SIMPLE-BIT-VECTOR
- ;; until runtime.
- #+sb-xc (make-array 0 :element-type 'bit)))
-
- (fill (finite-sb-conflicts sb) nil)
- (setf (finite-sb-conflicts sb)
- (make-array size :initial-element '#()))
-
- (fill (finite-sb-live-tns sb) nil)
- (setf (finite-sb-live-tns sb)
- (make-array size :initial-element nil))))))
- (values))
-
-(pushnew 'pack-before-gc-hook sb!ext:*before-gc-hooks*)
\f
;;;; internal errors
;;; defined to move from SRC to DEST.
(defun no-load-fun-error (src dest)
(let* ((src-sc (tn-sc src))
- (src-name (sc-name src-sc))
- (dest-sc (tn-sc dest))
- (dest-name (sc-name dest-sc)))
+ (src-name (sc-name src-sc))
+ (dest-sc (tn-sc dest))
+ (dest-name (sc-name dest-sc)))
(cond ((eq (sb-kind (sc-sb src-sc)) :non-packed)
- (unless (member src-sc (sc-constant-scs dest-sc))
- (error "loading from an invalid constant SC?~@
- VM definition inconsistent, try recompiling."))
- (error "no load function defined to load SC ~S ~
- from its constant SC ~S"
- dest-name src-name))
- ((member src-sc (sc-alternate-scs dest-sc))
- (error "no load function defined to load SC ~S from its ~
- alternate SC ~S"
- dest-name src-name))
- ((member dest-sc (sc-alternate-scs src-sc))
- (error "no load function defined to save SC ~S in its ~
- alternate SC ~S"
- src-name dest-name))
- (t
- ;; FIXME: "VM definition is inconsistent" shouldn't be a
- ;; possibility in SBCL.
- (error "loading to/from SCs that aren't alternates?~@
- VM definition is inconsistent, try recompiling.")))))
+ (unless (member src-sc (sc-constant-scs dest-sc))
+ (error "loading from an invalid constant SC?~@
+ VM definition inconsistent, try recompiling."))
+ (error "no load function defined to load SC ~S ~
+ from its constant SC ~S"
+ dest-name src-name))
+ ((member src-sc (sc-alternate-scs dest-sc))
+ (error "no load function defined to load SC ~S from its ~
+ alternate SC ~S"
+ dest-name src-name))
+ ((member dest-sc (sc-alternate-scs src-sc))
+ (error "no load function defined to save SC ~S in its ~
+ alternate SC ~S"
+ src-name dest-name))
+ (t
+ ;; FIXME: "VM definition is inconsistent" shouldn't be a
+ ;; possibility in SBCL.
+ (error "loading to/from SCs that aren't alternates?~@
+ VM definition is inconsistent, try recompiling.")))))
;;; Called when we failed to pack TN. If RESTRICTED is true, then we
;;; are restricted to pack TN in its SC.
(defun failed-to-pack-error (tn restricted)
(declare (type tn tn))
(let* ((sc (tn-sc tn))
- (scs (cons sc (sc-alternate-scs sc))))
+ (scs (cons sc (sc-alternate-scs sc))))
(cond
(restricted
(error "failed to pack restricted TN ~S in its SC ~S"
- tn (sc-name sc)))
+ tn (sc-name sc)))
(t
(aver (not (find :unbounded scs
- :key (lambda (x) (sb-kind (sc-sb x))))))
+ :key (lambda (x) (sb-kind (sc-sb x))))))
(let ((ptype (tn-primitive-type tn)))
- (cond
- (ptype
- (aver (member (sc-number sc) (primitive-type-scs ptype)))
- (error "SC ~S doesn't have any :UNBOUNDED alternate SCs, but is~@
- a SC for primitive-type ~S."
- (sc-name sc) (primitive-type-name ptype)))
- (t
- (error "SC ~S doesn't have any :UNBOUNDED alternate SCs."
- (sc-name sc)))))))))
+ (cond
+ (ptype
+ (aver (member (sc-number sc) (primitive-type-scs ptype)))
+ (error "SC ~S doesn't have any :UNBOUNDED alternate SCs, but is~@
+ a SC for primitive-type ~S."
+ (sc-name sc) (primitive-type-name ptype)))
+ (t
+ (error "SC ~S doesn't have any :UNBOUNDED alternate SCs."
+ (sc-name sc)))))))))
;;; Return a list of format arguments describing how TN is used in
;;; OP's VOP.
(defun describe-tn-use (loc tn op)
(let* ((vop (tn-ref-vop op))
- (args (vop-args vop))
- (results (vop-results vop))
- (name (with-output-to-string (stream)
- (print-tn-guts tn stream)))
- (2comp (component-info *component-being-compiled*))
- temp)
+ (args (vop-args vop))
+ (results (vop-results vop))
+ (name (with-output-to-string (stream)
+ (print-tn-guts tn stream)))
+ (2comp (component-info *component-being-compiled*))
+ temp)
(cond
((setq temp (position-in #'tn-ref-across tn args :key #'tn-ref-tn))
`("~2D: ~A (~:R argument)" ,loc ,name ,(1+ temp)))
((setq temp (position-in #'tn-ref-across tn args :key #'tn-ref-load-tn))
`("~2D: ~A (~:R argument load TN)" ,loc ,name ,(1+ temp)))
((setq temp (position-in #'tn-ref-across tn results :key
- #'tn-ref-load-tn))
+ #'tn-ref-load-tn))
`("~2D: ~A (~:R result load TN)" ,loc ,name ,(1+ temp)))
((setq temp (position-in #'tn-ref-across tn (vop-temps vop)
- :key #'tn-ref-tn))
+ :key #'tn-ref-tn))
`("~2D: ~A (temporary ~A)" ,loc ,name
- ,(operand-parse-name (elt (vop-parse-temps
- (vop-parse-or-lose
- (vop-info-name (vop-info vop))))
- temp))))
+ ,(operand-parse-name (elt (vop-parse-temps
+ (vop-parse-or-lose
+ (vop-info-name (vop-info vop))))
+ temp))))
((eq (tn-kind tn) :component)
`("~2D: ~A (component live)" ,loc ,name))
((position-in #'tn-next tn (ir2-component-wired-tns 2comp))
(defun failed-to-pack-load-tn-error (scs op)
(declare (list scs) (type tn-ref op))
(collect ((used)
- (unused))
+ (unused))
(dolist (sc scs)
(let* ((sb (sc-sb sc))
- (confs (finite-sb-live-tns sb)))
- (aver (eq (sb-kind sb) :finite))
- (dolist (el (sc-locations sc))
- (declare (type index el))
- (let ((conf (load-tn-conflicts-in-sc op sc el t)))
- (if conf
- (used (describe-tn-use el conf op))
- (do ((i el (1+ i))
- (end (+ el (sc-element-size sc))))
- ((= i end)
- (unused el))
- (declare (type index i end))
- (let ((victim (svref confs i)))
- (when victim
- (used (describe-tn-use el victim op))
- (return t)))))))))
+ (confs (finite-sb-live-tns sb)))
+ (aver (eq (sb-kind sb) :finite))
+ (dolist (el (sc-locations sc))
+ (declare (type index el))
+ (let ((conf (load-tn-conflicts-in-sc op sc el t)))
+ (if conf
+ (used (describe-tn-use el conf op))
+ (do ((i el (1+ i))
+ (end (+ el (sc-element-size sc))))
+ ((= i end)
+ (unused el))
+ (declare (type index i end))
+ (let ((victim (svref confs i)))
+ (when victim
+ (used (describe-tn-use el victim op))
+ (return t)))))))))
(multiple-value-bind (arg-p n more-p costs load-scs incon)
- (get-operand-info op)
+ (get-operand-info op)
(declare (ignore costs load-scs))
- (aver (not more-p))
- (error "unable to pack a Load-TN in SC ~{~A~#[~^~;, or ~:;,~]~} ~
- for the ~:R ~:[result~;argument~] to~@
- the ~S VOP,~@
- ~:[since all SC elements are in use:~:{~%~@?~}~%~;~
- ~:*but these SC elements are not in use:~% ~S~%Bug?~*~]~
- ~:[~;~@
- Current cost info inconsistent with that in effect at compile ~
- time. Recompile.~%Compilation order may be incorrect.~]"
- (mapcar #'sc-name scs)
- n arg-p
- (vop-info-name (vop-info (tn-ref-vop op)))
- (unused) (used)
- incon))))
+ (aver (not more-p))
+ (error "unable to pack a Load-TN in SC ~{~A~#[~^~;, or ~:;,~]~} ~
+ for the ~:R ~:[result~;argument~] to~@
+ the ~S VOP,~@
+ ~:[since all SC elements are in use:~:{~%~@?~}~%~;~
+ ~:*but these SC elements are not in use:~% ~S~%Bug?~*~]~
+ ~:[~;~@
+ Current cost info inconsistent with that in effect at compile ~
+ time. Recompile.~%Compilation order may be incorrect.~]"
+ (mapcar #'sc-name scs)
+ n arg-p
+ (vop-info-name (vop-info (tn-ref-vop op)))
+ (unused) (used)
+ incon))))
;;; This is called when none of the SCs that we can load OP into are
;;; allowed by OP's primitive-type.
(defun no-load-scs-allowed-by-primitive-type-error (ref)
(declare (type tn-ref ref))
(let* ((tn (tn-ref-tn ref))
- (ptype (tn-primitive-type tn)))
+ (ptype (tn-primitive-type tn)))
(multiple-value-bind (arg-p pos more-p costs load-scs incon)
- (get-operand-info ref)
+ (get-operand-info ref)
(declare (ignore costs))
(aver (not more-p))
(error "~S is not valid as the ~:R ~:[result~;argument~] to VOP:~
- ~% ~S,~@
- since the TN's primitive type ~S doesn't allow any of the SCs~@
- allowed by the operand restriction:~% ~S~
- ~:[~;~@
- Current cost info inconsistent with that in effect at compile ~
- time. Recompile.~%Compilation order may be incorrect.~]"
- tn pos arg-p
- (template-name (vop-info (tn-ref-vop ref)))
- (primitive-type-name ptype)
- (mapcar #'sc-name (listify-restrictions load-scs))
- incon))))
+ ~% ~S,~@
+ since the TN's primitive type ~S doesn't allow any of the SCs~@
+ allowed by the operand restriction:~% ~S~
+ ~:[~;~@
+ Current cost info inconsistent with that in effect at compile ~
+ time. Recompile.~%Compilation order may be incorrect.~]"
+ tn pos arg-p
+ (template-name (vop-info (tn-ref-vop ref)))
+ (primitive-type-name ptype)
+ (mapcar #'sc-name (listify-restrictions load-scs))
+ incon))))
\f
;;;; register saving
(declare (type tn tn))
(let ((res (make-tn 0 :save nil nil)))
(dolist (alt (sc-alternate-scs (tn-sc tn))
- (error "no unbounded alternate for SC ~S"
- (sc-name (tn-sc tn))))
+ (error "no unbounded alternate for SC ~S"
+ (sc-name (tn-sc tn))))
(when (eq (sb-kind (sc-sb alt)) :unbounded)
- (setf (tn-save-tn tn) res)
- (setf (tn-save-tn res) tn)
- (setf (tn-sc res) alt)
- (pack-tn res t)
- (return res)))))
+ (setf (tn-save-tn tn) res)
+ (setf (tn-save-tn res) tn)
+ (setf (tn-sc res) alt)
+ (pack-tn res t nil)
+ (return res)))))
;;; Find the load function for moving from SRC to DEST and emit a
;;; MOVE-OPERAND VOP with that function as its info arg.
(defun emit-operand-load (node block src dest before)
(declare (type node node) (type ir2-block block)
- (type tn src dest) (type (or vop null) before))
+ (type tn src dest) (type (or vop null) before))
(emit-load-template node block
- (template-or-lose 'move-operand)
- src dest
- (list (or (svref (sc-move-funs (tn-sc dest))
- (sc-number (tn-sc src)))
- (no-load-fun-error src dest)))
- before)
+ (template-or-lose 'move-operand)
+ src dest
+ (list (or (svref (sc-move-funs (tn-sc dest))
+ (sc-number (tn-sc src)))
+ (no-load-fun-error src dest)))
+ before)
(values))
;;; Find the preceding use of the VOP NAME in the emit order, starting
;;; with VOP. We must find the VOP in the same IR1 block.
(defun reverse-find-vop (name vop)
(do* ((block (vop-block vop) (ir2-block-prev block))
- (last vop (ir2-block-last-vop block)))
+ (last vop (ir2-block-last-vop block)))
(nil)
(aver (eq (ir2-block-block block) (ir2-block-block (vop-block vop))))
(do ((current last (vop-prev current)))
- ((null current))
+ ((null current))
(when (eq (vop-info-name (vop-info current)) name)
- (return-from reverse-find-vop current)))))
+ (return-from reverse-find-vop current)))))
;;; For TNs that have other than one writer, we save the TN before
;;; each call. If a local call (MOVE-ARGS is :LOCAL-CALL), then we
;;; which the values are known to be good.
(defun save-complex-writer-tn (tn vop)
(let ((save (or (tn-save-tn tn)
- (pack-save-tn tn)))
- (node (vop-node vop))
- (block (vop-block vop))
- (next (vop-next vop)))
+ (pack-save-tn tn)))
+ (node (vop-node vop))
+ (block (vop-block vop))
+ (next (vop-next vop)))
(when (eq (tn-kind save) :specified-save)
(setf (tn-kind save) :save))
(aver (eq (tn-kind save) :save))
(emit-operand-load node block tn save
- (if (eq (vop-info-move-args (vop-info vop))
- :local-call)
- (reverse-find-vop 'allocate-frame vop)
- vop))
+ (if (eq (vop-info-move-args (vop-info vop))
+ :local-call)
+ (reverse-find-vop 'allocate-frame vop)
+ vop))
(emit-operand-load node block save tn next)))
;;; Return a VOP after which is an OK place to save the value of TN.
(res nil))
((null write)
(when (and res
- (do ((read (tn-reads tn) (tn-ref-next read)))
- ((not read) t)
- (when (eq (vop-info-move-args
- (vop-info
- (tn-ref-vop read)))
- :local-call)
- (return nil))))
- (tn-ref-vop res)))
+ (do ((read (tn-reads tn) (tn-ref-next read)))
+ ((not read) t)
+ (when (eq (vop-info-move-args
+ (vop-info
+ (tn-ref-vop read)))
+ :local-call)
+ (return nil))))
+ (tn-ref-vop res)))
(unless (eq (vop-info-name (vop-info (tn-ref-vop write)))
- 'move-operand)
+ 'move-operand)
(when res (return nil))
(setq res write))))
(defun save-single-writer-tn (tn)
(declare (type tn tn))
(let* ((old-save (tn-save-tn tn))
- (save (or old-save (pack-save-tn tn)))
- (writer (find-single-writer tn)))
+ (save (or old-save (pack-save-tn tn)))
+ (writer (find-single-writer tn)))
(when (and writer
- (or (not old-save)
- (eq (tn-kind old-save) :specified-save)))
+ (or (not old-save)
+ (eq (tn-kind old-save) :specified-save)))
(emit-operand-load (vop-node writer) (vop-block writer)
- tn save (vop-next writer))
+ tn save (vop-next writer))
(setf (tn-kind save) :save-once)
t)))
(declare (type tn tn) (type vop vop))
(let ((save (tn-save-tn tn)))
(cond ((and save (eq (tn-kind save) :save-once))
- (restore-single-writer-tn tn vop))
- ((save-single-writer-tn tn)
- (restore-single-writer-tn tn vop))
- (t
- (save-complex-writer-tn tn vop))))
+ (restore-single-writer-tn tn vop))
+ ((save-single-writer-tn tn)
+ (restore-single-writer-tn tn vop))
+ (t
+ (save-complex-writer-tn tn vop))))
(values))
;;; Scan over the VOPs in BLOCK, emiting saving code for TNs noted in
((null vop))
(when (eq (vop-info-save-p (vop-info vop)) t)
(do-live-tns (tn (vop-save-set vop) block)
- (when (and (sc-save-p (tn-sc tn))
- (not (eq (tn-kind tn) :component)))
- (basic-save-tn tn vop)))))
+ (when (and (sc-save-p (tn-sc tn))
+ (not (eq (tn-kind tn) :component)))
+ (basic-save-tn tn vop)))))
(values))
\f
(aver (member (tn-kind save) '(:save :save-once)))
(unless (eq (tn-kind save) :save-once)
(or (save-single-writer-tn tn)
- (emit-operand-load (vop-node context) (vop-block context)
- tn save before))))
+ (emit-operand-load (vop-node context) (vop-block context)
+ tn save before))))
(values))
;;; Load the TN from its save location, allocating one if necessary.
-;;; The load is inserted BEFORE the specifier VOP. CONTEXT is a VOP
+;;; The load is inserted BEFORE the specified VOP. CONTEXT is a VOP
;;; used to tell which node/block to use for the new VOP.
(defun restore-tn (tn before context)
(declare (type tn tn) (type (or vop null) before) (type vop context))
(let ((save (or (tn-save-tn tn) (pack-save-tn tn))))
(emit-operand-load (vop-node context) (vop-block context)
- save tn before))
+ save tn before))
(values))
-(eval-when (:compile-toplevel :execute)
-
-;;; Do stuff to note a read of TN, for OPTIMIZED-EMIT-SAVES-BLOCK.
-(defmacro save-note-read (tn)
- `(let* ((tn ,tn)
- (num (tn-number tn)))
- (when (and (sc-save-p (tn-sc tn))
- (zerop (sbit restores num))
- (not (eq (tn-kind tn) :component)))
- (setf (sbit restores num) 1)
- (push tn restores-list))))
-
-) ; EVAL-WHEN
-
;;; Start scanning backward at the end of BLOCK, looking which TNs are
;;; live and looking for places where we have to save. We manipulate
;;; two sets: SAVES and RESTORES.
(defun optimized-emit-saves-block (block saves restores)
(declare (type ir2-block block) (type simple-bit-vector saves restores))
(let ((1block (ir2-block-block block))
- (saves-list ())
- (restores-list ())
- (skipping nil))
+ (saves-list ())
+ (restores-list ())
+ (skipping nil))
(declare (list saves-list restores-list))
(clear-bit-vector saves)
(clear-bit-vector restores)
(do-live-tns (tn (ir2-block-live-in block) block)
(when (and (sc-save-p (tn-sc tn))
- (not (eq (tn-kind tn) :component)))
- (let ((num (tn-number tn)))
- (setf (sbit restores num) 1)
- (push tn restores-list))))
+ (not (eq (tn-kind tn) :component)))
+ (let ((num (tn-number tn)))
+ (setf (sbit restores num) 1)
+ (push tn restores-list))))
(do ((block block (ir2-block-prev block))
- (prev nil block))
- ((not (eq (ir2-block-block block) 1block))
- (aver (not skipping))
- (dolist (save saves-list)
- (let ((start (ir2-block-start-vop prev)))
- (save-if-necessary save start start)))
- prev)
+ (prev nil block))
+ ((not (eq (ir2-block-block block) 1block))
+ (aver (not skipping))
+ (dolist (save saves-list)
+ (let ((start (ir2-block-start-vop prev)))
+ (save-if-necessary save start start)))
+ prev)
(do ((vop (ir2-block-last-vop block) (vop-prev vop)))
- ((null vop))
- (let ((info (vop-info vop)))
- (case (vop-info-name info)
- (allocate-frame
- (aver skipping)
- (setq skipping nil))
- (note-environment-start
- (aver (not skipping))
- (dolist (save saves-list)
- (save-if-necessary save (vop-next vop) vop))
- (return-from optimized-emit-saves-block block)))
-
- (unless skipping
- (do ((write (vop-results vop) (tn-ref-across write)))
- ((null write))
- (let* ((tn (tn-ref-tn write))
- (num (tn-number tn)))
- (unless (zerop (sbit restores num))
- (setf (sbit restores num) 0)
- (setq restores-list
- (delete tn restores-list :test #'eq)))
- (unless (zerop (sbit saves num))
- (setf (sbit saves num) 0)
- (save-if-necessary tn (vop-next vop) vop)
- (setq saves-list
- (delete tn saves-list :test #'eq))))))
-
- (macrolet (;; Do stuff to note a read of TN, for
- ;; OPTIMIZED-EMIT-SAVES-BLOCK.
- (save-note-read (tn)
- `(let* ((tn ,tn)
- (num (tn-number tn)))
- (when (and (sc-save-p (tn-sc tn))
- (zerop (sbit restores num))
- (not (eq (tn-kind tn) :component)))
- (setf (sbit restores num) 1)
- (push tn restores-list)))))
-
- (case (vop-info-save-p info)
- ((t)
- (dolist (tn restores-list)
- (restore-tn tn (vop-next vop) vop)
- (let ((num (tn-number tn)))
- (when (zerop (sbit saves num))
- (push tn saves-list)
- (setf (sbit saves num) 1))))
- (setq restores-list nil)
- (clear-bit-vector restores))
- (:compute-only
- (cond ((policy (vop-node vop) (= speed 3))
- (do-live-tns (tn (vop-save-set vop) block)
- (when (zerop (sbit restores (tn-number tn)))
- (note-spilled-tn tn vop))))
- (t
- (do-live-tns (tn (vop-save-set vop) block)
- (save-note-read tn))))))
-
- (if (eq (vop-info-move-args info) :local-call)
- (setq skipping t)
- (do ((read (vop-args vop) (tn-ref-across read)))
- ((null read))
- (save-note-read (tn-ref-tn read))))))))))
+ ((null vop))
+ (let ((info (vop-info vop)))
+ (case (vop-info-name info)
+ (allocate-frame
+ (aver skipping)
+ (setq skipping nil))
+ (note-environment-start
+ (aver (not skipping))
+ (dolist (save saves-list)
+ (save-if-necessary save (vop-next vop) vop))
+ (return-from optimized-emit-saves-block block)))
+
+ (unless skipping
+ (do ((write (vop-results vop) (tn-ref-across write)))
+ ((null write))
+ (let* ((tn (tn-ref-tn write))
+ (num (tn-number tn)))
+ (unless (zerop (sbit restores num))
+ (setf (sbit restores num) 0)
+ (setq restores-list
+ (delete tn restores-list :test #'eq)))
+ (unless (zerop (sbit saves num))
+ (setf (sbit saves num) 0)
+ (save-if-necessary tn (vop-next vop) vop)
+ (setq saves-list
+ (delete tn saves-list :test #'eq))))))
+
+ (macrolet ((save-note-read (tn)
+ `(let* ((tn ,tn)
+ (num (tn-number tn)))
+ (when (and (sc-save-p (tn-sc tn))
+ (zerop (sbit restores num))
+ (not (eq (tn-kind tn) :component)))
+ (setf (sbit restores num) 1)
+ (push tn restores-list)))))
+
+ (case (vop-info-save-p info)
+ ((t)
+ (dolist (tn restores-list)
+ (restore-tn tn (vop-next vop) vop)
+ (let ((num (tn-number tn)))
+ (when (zerop (sbit saves num))
+ (push tn saves-list)
+ (setf (sbit saves num) 1))))
+ (setq restores-list nil)
+ (clear-bit-vector restores))
+ (:compute-only
+ (cond ((policy (vop-node vop) (= speed 3))
+ (do-live-tns (tn (vop-save-set vop) block)
+ (when (zerop (sbit restores (tn-number tn)))
+ (note-spilled-tn tn vop))))
+ (t
+ (do-live-tns (tn (vop-save-set vop) block)
+ (save-note-read tn))))))
+
+ (if (eq (vop-info-move-args info) :local-call)
+ (setq skipping t)
+ (do ((read (vop-args vop) (tn-ref-across read)))
+ ((null read))
+ (save-note-read (tn-ref-tn read))))))))))
;;; This is like EMIT-SAVES, only different. We avoid redundant saving
;;; within the block, and don't restore values that aren't used before
(defun optimized-emit-saves (component)
(declare (type component component))
(let* ((gtn-count (1+ (ir2-component-global-tn-counter
- (component-info component))))
- (saves (make-array gtn-count :element-type 'bit))
- (restores (make-array gtn-count :element-type 'bit))
- (block (ir2-block-prev (block-info (component-tail component))))
- (head (block-info (component-head component))))
+ (component-info component))))
+ (saves (make-array gtn-count :element-type 'bit))
+ (restores (make-array gtn-count :element-type 'bit))
+ (block (ir2-block-prev (block-info (component-tail component))))
+ (head (block-info (component-head component))))
(loop
(when (eq block head) (return))
(when (do ((vop (ir2-block-start-vop block) (vop-next vop)))
- ((null vop) nil)
- (when (eq (vop-info-save-p (vop-info vop)) t)
- (return t)))
- (setq block (optimized-emit-saves-block block saves restores)))
+ ((null vop) nil)
+ (when (eq (vop-info-save-p (vop-info vop)) t)
+ (return t)))
+ (setq block (optimized-emit-saves-block block saves restores)))
(setq block (ir2-block-prev block)))))
-
+\f
;;; Iterate over the normal TNs, finding the cost of packing on the
-;;; stack in units of the number of references. We count all
-;;; references as +1, and subtract out REGISTER-SAVE-PENALTY for each
-;;; place where we would have to save a register.
+;;; stack in units of the number of references. We count all read
+;;; references as +1, write references as + *tn-write-cost*, and
+;;; subtract out REGISTER-SAVE-PENALTY for each place where we would
+;;; have to save a register.
+;;; The subtraction reflects the fact that having a value in a
+;;; register around a call means that code to spill and unspill must
+;;; be inserted.
+;;;
+;;; The costs also take into account the loop depth at which each
+;;; reference occurs: the penalty or cost is incremented by the depth
+;;; scaled by *tn-loop-depth-multiplier*. The default (NIL) is to let
+;;; this be one more than the max of the cost for reads (1), for write
+;;; references and for being live across a call.
+(defvar *tn-write-cost* 2)
+(defvar *tn-loop-depth-multiplier* nil)
+
(defun assign-tn-costs (component)
- (do-ir2-blocks (block component)
- (do ((vop (ir2-block-start-vop block) (vop-next vop)))
- ((null vop))
- (when (eq (vop-info-save-p (vop-info vop)) t)
- (do-live-tns (tn (vop-save-set vop) block)
- (decf (tn-cost tn) *backend-register-save-penalty*)))))
-
- (do ((tn (ir2-component-normal-tns (component-info component))
- (tn-next tn)))
- ((null tn))
- (let ((cost (tn-cost tn)))
- (declare (fixnum cost))
- (do ((ref (tn-reads tn) (tn-ref-next ref)))
- ((null ref))
- (incf cost))
- (do ((ref (tn-writes tn) (tn-ref-next ref)))
- ((null ref))
- (incf cost))
- (setf (tn-cost tn) cost))))
+ (let* ((save-penalty *backend-register-save-penalty*)
+ (write-cost *tn-write-cost*)
+ (depth-scale (or *tn-loop-depth-multiplier*
+ (1+ (max 1 write-cost save-penalty)))))
+ (flet ((vop-depth-cost (vop)
+ (let ((loop (block-loop
+ (ir2-block-block
+ (vop-block vop)))))
+ (if loop
+ (* depth-scale (loop-depth loop))
+ 0))))
+ (do-ir2-blocks (block component)
+ (do ((vop (ir2-block-start-vop block) (vop-next vop)))
+ ((null vop))
+ (when (eq (vop-info-save-p (vop-info vop)) t)
+ (let ((penalty (+ save-penalty (vop-depth-cost vop))))
+ (do-live-tns (tn (vop-save-set vop) block)
+ (decf (tn-cost tn) penalty))))))
+
+ (do ((tn (ir2-component-normal-tns (component-info component))
+ (tn-next tn)))
+ ((null tn))
+ (let ((cost (tn-cost tn)))
+ (declare (fixnum cost))
+ (do ((ref (tn-reads tn) (tn-ref-next ref)))
+ ((null ref))
+ (incf cost (1+ (vop-depth-cost (tn-ref-vop ref)))))
+ (do ((ref (tn-writes tn) (tn-ref-next ref)))
+ ((null ref))
+ (incf cost (+ write-cost (vop-depth-cost (tn-ref-vop ref)))))
+ (setf (tn-cost tn) cost))))))
+
+;;; Iterate over the normal TNs, folding over the depth of the looops
+;;; that the TN is used in and storing the result in TN-LOOP-DEPTH.
+;;: reducer is the function used to join depth values together. #'max
+;;; gives the maximum depth, #'+ the sum.
+(defun assign-tn-depths (component &key (reducer #'max))
+ (declare (type function reducer))
+ (when *loop-analyze*
+ ;; We only use tn depth for normal TNs
+ (do ((tn (ir2-component-normal-tns (component-info component))
+ (tn-next tn)))
+ ((null tn))
+ (let ((depth 0))
+ (declare (type fixnum depth))
+ (flet ((frob (ref)
+ (declare (type (or null tn-ref) ref))
+ (do ((ref ref (tn-ref-next ref)))
+ ((null ref))
+ (let* ((vop (tn-ref-vop ref))
+ (block (ir2-block-block (vop-block vop)))
+ (loop (block-loop block)))
+ (setf depth (funcall reducer
+ depth
+ (if loop
+ (loop-depth loop)
+ 0)))))))
+ (frob (tn-reads tn))
+ (frob (tn-writes tn))
+ (setf (tn-loop-depth tn) depth))))))
\f
;;;; load TN packing
;;; sticking them in this hash-table. This is initially null. We
;;; create the hashtable if we do any unpacking.
(defvar *repack-blocks*)
-(declaim (type (or hash-table null) *repack-blocks*))
+(declaim (type list *repack-blocks*))
;;; Set the LIVE-TNS vectors in all :FINITE SBs to represent the TNs
;;; live at the end of BLOCK.
(do-live-tns (tn (ir2-block-live-in block) block)
(let* ((sc (tn-sc tn))
- (sb (sc-sb sc)))
+ (sb (sc-sb sc)))
(when (eq (sb-kind sb) :finite)
- (do ((offset (tn-offset tn) (1+ offset))
- (end (+ (tn-offset tn) (sc-element-size sc))))
- ((= offset end))
- (declare (type index offset end))
- (setf (svref (finite-sb-live-tns sb) offset) tn)))))
+ ;; KLUDGE: we can have "live" TNs that are neither read
+ ;; to nor written from, due to more aggressive (type-
+ ;; directed) constant propagation. Such TNs will never
+ ;; be assigned an offset nor be in conflict with anything.
+ ;;
+ ;; Ideally, it seems to me we could make sure these TNs
+ ;; are never allocated in the first place in
+ ;; ASSIGN-LAMBDA-VAR-TNS.
+ (if (tn-offset tn)
+ (do ((offset (tn-offset tn) (1+ offset))
+ (end (+ (tn-offset tn) (sc-element-size sc))))
+ ((= offset end))
+ (declare (type index offset end))
+ (setf (svref (finite-sb-live-tns sb) offset) tn))
+ (aver (and (null (tn-reads tn)) (null (tn-writes tn))))))))
(setq *live-block* block)
(setq *live-vop* (ir2-block-last-vop block))
(do ((current *live-vop* (vop-prev current)))
((eq current vop)
(do ((res (vop-results vop) (tn-ref-across res)))
- ((null res))
- (let* ((tn (tn-ref-tn res))
- (sc (tn-sc tn))
- (sb (sc-sb sc)))
- (when (eq (sb-kind sb) :finite)
- (do ((offset (tn-offset tn) (1+ offset))
- (end (+ (tn-offset tn) (sc-element-size sc))))
- ((= offset end))
- (declare (type index offset end))
- (setf (svref (finite-sb-live-tns sb) offset) nil))))))
+ ((null res))
+ (let* ((tn (tn-ref-tn res))
+ (sc (tn-sc tn))
+ (sb (sc-sb sc)))
+ (when (eq (sb-kind sb) :finite)
+ (do ((offset (tn-offset tn) (1+ offset))
+ (end (+ (tn-offset tn) (sc-element-size sc))))
+ ((= offset end))
+ (declare (type index offset end))
+ (setf (svref (finite-sb-live-tns sb) offset) nil))))))
(do ((ref (vop-refs current) (tn-ref-next-ref ref)))
- ((null ref))
+ ((null ref))
(let ((ltn (tn-ref-load-tn ref)))
- (when ltn
- (let* ((sc (tn-sc ltn))
- (sb (sc-sb sc)))
- (when (eq (sb-kind sb) :finite)
- (let ((tns (finite-sb-live-tns sb)))
- (do ((offset (tn-offset ltn) (1+ offset))
- (end (+ (tn-offset ltn) (sc-element-size sc))))
- ((= offset end))
- (declare (type index offset end))
- (aver (null (svref tns offset)))))))))
+ (when ltn
+ (let* ((sc (tn-sc ltn))
+ (sb (sc-sb sc)))
+ (when (eq (sb-kind sb) :finite)
+ (let ((tns (finite-sb-live-tns sb)))
+ (do ((offset (tn-offset ltn) (1+ offset))
+ (end (+ (tn-offset ltn) (sc-element-size sc))))
+ ((= offset end))
+ (declare (type index offset end))
+ (aver (null (svref tns offset)))))))))
(let* ((tn (tn-ref-tn ref))
- (sc (tn-sc tn))
- (sb (sc-sb sc)))
- (when (eq (sb-kind sb) :finite)
- (let ((tns (finite-sb-live-tns sb)))
- (do ((offset (tn-offset tn) (1+ offset))
- (end (+ (tn-offset tn) (sc-element-size sc))))
- ((= offset end))
- (declare (type index offset end))
- (if (tn-ref-write-p ref)
- (setf (svref tns offset) nil)
- (let ((old (svref tns offset)))
- (aver (or (null old) (eq old tn)))
- (setf (svref tns offset) tn)))))))))
+ (sc (tn-sc tn))
+ (sb (sc-sb sc)))
+ (when (eq (sb-kind sb) :finite)
+ (let ((tns (finite-sb-live-tns sb)))
+ (do ((offset (tn-offset tn) (1+ offset))
+ (end (+ (tn-offset tn) (sc-element-size sc))))
+ ((= offset end))
+ (declare (type index offset end))
+ (if (tn-ref-write-p ref)
+ (setf (svref tns offset) nil)
+ (let ((old (svref tns offset)))
+ (aver (or (null old) (eq old tn)))
+ (setf (svref tns offset) tn)))))))))
(setq *live-vop* vop)
(values))
;;; aren't any TN-REFs to represent the implicit reading of results or
;;; writing of arguments.
;;;
-;;; The second bullet corresponds conflicts with temporaries or between
-;;; arguments and results.
+;;; The second bullet corresponds to conflicts with temporaries or
+;;; between arguments and results.
;;;
;;; We consider both the TN-REF-TN and the TN-REF-LOAD-TN (if any) to
;;; be referenced simultaneously and in the same way. This causes
(aver (eq (sb-kind sb) :finite))
(let ((vop (tn-ref-vop op)))
(labels ((tn-overlaps (tn)
- (let ((sc (tn-sc tn))
- (tn-offset (tn-offset tn)))
- (when (and (eq (sc-sb sc) sb)
- (<= tn-offset offset)
- (< offset
- (the index
- (+ tn-offset (sc-element-size sc)))))
- tn)))
- (same (ref)
- (let ((tn (tn-ref-tn ref))
- (ltn (tn-ref-load-tn ref)))
- (or (tn-overlaps tn)
- (and ltn (tn-overlaps ltn)))))
- (is-op (ops)
- (do ((ops ops (tn-ref-across ops)))
- ((null ops) nil)
- (let ((found (same ops)))
- (when (and found (not (eq ops op)))
- (return found)))))
- (is-ref (refs end)
- (do ((refs refs (tn-ref-next-ref refs)))
- ((eq refs end) nil)
- (let ((found (same refs)))
- (when found (return found))))))
+ (let ((sc (tn-sc tn))
+ (tn-offset (tn-offset tn)))
+ (when (and (eq (sc-sb sc) sb)
+ (<= tn-offset offset)
+ (< offset
+ (the index
+ (+ tn-offset (sc-element-size sc)))))
+ tn)))
+ (same (ref)
+ (let ((tn (tn-ref-tn ref))
+ (ltn (tn-ref-load-tn ref)))
+ (or (tn-overlaps tn)
+ (and ltn (tn-overlaps ltn)))))
+ (is-op (ops)
+ (do ((ops ops (tn-ref-across ops)))
+ ((null ops) nil)
+ (let ((found (same ops)))
+ (when (and found (not (eq ops op)))
+ (return found)))))
+ (is-ref (refs end)
+ (do ((refs refs (tn-ref-next-ref refs)))
+ ((eq refs end) nil)
+ (let ((found (same refs)))
+ (when found (return found))))))
(declare (inline is-op is-ref tn-overlaps))
(if (tn-ref-write-p op)
- (or (is-op (vop-results vop))
- (is-ref (vop-refs vop) op))
- (or (is-op (vop-args vop))
- (is-ref (tn-ref-next-ref op) nil))))))
+ (or (is-op (vop-results vop))
+ (is-ref (vop-refs vop) op))
+ (or (is-op (vop-args vop))
+ (is-ref (tn-ref-next-ref op) nil))))))
;;; Iterate over all the elements in the SB that would be allocated by
;;; allocating a TN in SC at Offset, checking for conflict with
;;; We return a conflicting TN, or :OVERFLOW if the TN won't fit.
(defun load-tn-conflicts-in-sc (op sc offset ignore-live)
(let* ((sb (sc-sb sc))
- (size (finite-sb-current-size sb)))
+ (size (finite-sb-current-size sb)))
(do ((i offset (1+ i))
- (end (+ offset (sc-element-size sc))))
- ((= i end) nil)
+ (end (+ offset (sc-element-size sc))))
+ ((= i end) nil)
(declare (type index i end))
(let ((res (or (when (>= i size) :overflow)
- (and (not ignore-live)
- (svref (finite-sb-live-tns sb) i))
- (load-tn-offset-conflicts-in-sb op sb i))))
- (when res (return res))))))
+ (and (not ignore-live)
+ (svref (finite-sb-live-tns sb) i))
+ (load-tn-offset-conflicts-in-sb op sb i))))
+ (when res (return res))))))
;;; If a load-TN for OP is targeted to a legal location in SC, then
;;; return the offset, otherwise return NIL. We see whether the target
(let ((target (tn-ref-target op)))
(when target
(let* ((tn (tn-ref-tn target))
- (loc (tn-offset tn)))
- (if (and (eq (tn-sc tn) sc)
- (member (the index loc) (sc-locations sc))
- (not (load-tn-conflicts-in-sc op sc loc nil)))
- loc
- nil)))))
+ (loc (tn-offset tn)))
+ (if (and (eq (tn-sc tn) sc)
+ (member (the index loc) (sc-locations sc))
+ (not (load-tn-conflicts-in-sc op sc loc nil)))
+ loc
+ nil)))))
;;; Select a legal location for a load TN for Op in SC. We just
;;; iterate over the SC's locations. If we can't find a legal
(let ((target (tn-ref-target op)))
(when target
(let* ((tn (tn-ref-tn target))
- (loc (tn-offset tn)))
- (when (and (eq (sc-sb sc) (sc-sb (tn-sc tn)))
- (member (the index loc) (sc-locations sc))
- (not (load-tn-conflicts-in-sc op sc loc nil)))
- (return-from select-load-tn-location loc)))))
+ (loc (tn-offset tn)))
+ (when (and (eq (sc-sb sc) (sc-sb (tn-sc tn)))
+ (member (the index loc) (sc-locations sc))
+ (not (load-tn-conflicts-in-sc op sc loc nil)))
+ (return-from select-load-tn-location loc)))))
(dolist (loc (sc-locations sc) nil)
(unless (load-tn-conflicts-in-sc op sc loc nil)
(defun unpack-tn (tn)
(event unpack-tn)
(let ((stn (or (tn-save-tn tn)
- (pack-save-tn tn))))
+ (pack-save-tn tn))))
(setf (tn-sc tn) (tn-sc stn))
(setf (tn-offset tn) (tn-offset stn))
(flet ((zot (refs)
- (do ((ref refs (tn-ref-next ref)))
- ((null ref))
- (let ((vop (tn-ref-vop ref)))
- (if (eq (vop-info-name (vop-info vop)) 'move-operand)
- (delete-vop vop)
- (setf (gethash (vop-block vop) *repack-blocks*) t))))))
+ (do ((ref refs (tn-ref-next ref)))
+ ((null ref))
+ (let ((vop (tn-ref-vop ref)))
+ (if (eq (vop-info-name (vop-info vop)) 'move-operand)
+ (delete-vop vop)
+ (pushnew (vop-block vop) *repack-blocks*))))))
(zot (tn-reads tn))
(zot (tn-writes tn))))
;;; This is called by PACK-LOAD-TN where there isn't any location free
;;; that we can pack into. What we do is move some live TN in one of
-;;; the specified SCs to memory, then mark this block all blocks that
-;;; reference the TN as needing repacking. If we succeed, we throw to
-;;; UNPACKED-TN. If we fail, we return NIL.
+;;; the specified SCs to memory, then mark all blocks that reference
+;;; the TN as needing repacking. If we succeed, we throw to UNPACKED-TN.
+;;; If we fail, we return NIL.
;;;
;;; We can unpack any live TN that appears in the NORMAL-TNs list
;;; (isn't wired or restricted.) We prefer to unpack TNs that are not
(defun unpack-for-load-tn (sc op)
(declare (type sc sc) (type tn-ref op))
(let ((sb (sc-sb sc))
- (normal-tns (ir2-component-normal-tns
- (component-info *component-being-compiled*)))
- (node (vop-node (tn-ref-vop op)))
- (fallback nil))
+ (normal-tns (ir2-component-normal-tns
+ (component-info *component-being-compiled*)))
+ (node (vop-node (tn-ref-vop op)))
+ (fallback nil))
(flet ((unpack-em (victims)
- (unless *repack-blocks*
- (setq *repack-blocks* (make-hash-table :test 'eq)))
- (setf (gethash (vop-block (tn-ref-vop op)) *repack-blocks*) t)
- (dolist (victim victims)
- (event unpack-tn node)
- (unpack-tn victim))
- (throw 'unpacked-tn nil)))
+ (pushnew (vop-block (tn-ref-vop op)) *repack-blocks*)
+ (dolist (victim victims)
+ (event unpack-tn node)
+ (unpack-tn victim))
+ (throw 'unpacked-tn nil)))
(dolist (loc (sc-locations sc))
- (declare (type index loc))
- (block SKIP
- (collect ((victims nil adjoin))
- (do ((i loc (1+ i))
- (end (+ loc (sc-element-size sc))))
- ((= i end))
- (declare (type index i end))
- (let ((victim (svref (finite-sb-live-tns sb) i)))
- (when victim
- (unless (find-in #'tn-next victim normal-tns)
- (return-from SKIP))
- (victims victim))))
-
- (let ((conf (load-tn-conflicts-in-sc op sc loc t)))
- (cond ((not conf)
- (unpack-em (victims)))
- ((eq conf :overflow))
- ((not fallback)
- (cond ((find conf (victims))
- (setq fallback (victims)))
- ((find-in #'tn-next conf normal-tns)
- (setq fallback (list conf))))))))))
+ (declare (type index loc))
+ (block SKIP
+ (collect ((victims nil adjoin))
+ (do ((i loc (1+ i))
+ (end (+ loc (sc-element-size sc))))
+ ((= i end))
+ (declare (type index i end))
+ (let ((victim (svref (finite-sb-live-tns sb) i)))
+ (when victim
+ (unless (find-in #'tn-next victim normal-tns)
+ (return-from SKIP))
+ (victims victim))))
+
+ (let ((conf (load-tn-conflicts-in-sc op sc loc t)))
+ (cond ((not conf)
+ (unpack-em (victims)))
+ ((eq conf :overflow))
+ ((not fallback)
+ (cond ((find conf (victims))
+ (setq fallback (victims)))
+ ((find-in #'tn-next conf normal-tns)
+ (setq fallback (list conf))))))))))
(when fallback
- (event unpack-fallback node)
- (unpack-em fallback))))
+ (event unpack-fallback node)
+ (unpack-em fallback))))
nil)
;;; if that location is in a SC not allowed by the primitive type.
;;; (The SC must still be allowed by the operand restriction.) This
;;; makes move VOPs more efficient, since we won't do a move from the
-;;; stack into a non-descriptor any-reg though a descriptor argument
+;;; stack into a non-descriptor any-reg through a descriptor argument
;;; load-TN. This does give targeting some real semantics, making it
;;; not a pure advisory to pack. It allows pack to do some packing it
;;; wouldn't have done before.
(compute-live-tns (vop-block vop) vop))
(let* ((tn (tn-ref-tn op))
- (ptype (tn-primitive-type tn))
- (scs (svref load-scs (sc-number (tn-sc tn)))))
+ (ptype (tn-primitive-type tn))
+ (scs (svref load-scs (sc-number (tn-sc tn)))))
(let ((current-scs scs)
- (allowed ()))
+ (allowed ()))
(loop
- (cond
- ((null current-scs)
- (unless allowed
- (no-load-scs-allowed-by-primitive-type-error op))
- (dolist (sc allowed)
- (unpack-for-load-tn sc op))
- (failed-to-pack-load-tn-error allowed op))
- (t
- (let* ((sc (svref *backend-sc-numbers* (pop current-scs)))
- (target (find-load-tn-target op sc)))
- (when (or target (sc-allowed-by-primitive-type sc ptype))
- (let ((loc (or target
- (select-load-tn-location op sc))))
- (when loc
- (let ((res (make-tn 0 :load nil sc)))
- (setf (tn-offset res) loc)
- (return res))))
- (push sc allowed)))))))))
+ (cond
+ ((null current-scs)
+ (unless allowed
+ (no-load-scs-allowed-by-primitive-type-error op))
+ (dolist (sc allowed)
+ (unpack-for-load-tn sc op))
+ (failed-to-pack-load-tn-error allowed op))
+ (t
+ (let* ((sc (svref *backend-sc-numbers* (pop current-scs)))
+ (target (find-load-tn-target op sc)))
+ (when (or target (sc-allowed-by-primitive-type sc ptype))
+ (let ((loc (or target
+ (select-load-tn-location op sc))))
+ (when loc
+ (let ((res (make-tn 0 :load nil sc)))
+ (setf (tn-offset res) loc)
+ (return res))))
+ (push sc allowed)))))))))
;;; Scan a list of load-SCs vectors and a list of TN-REFS threaded by
;;; TN-REF-ACROSS. When we find a reference whose TN doesn't satisfy
(op ops (tn-ref-across op)))
((null scs))
(let ((target (tn-ref-target op)))
- (when target
- (let* ((load-tn (tn-ref-load-tn op))
- (load-scs (svref (car scs)
- (sc-number
- (tn-sc (or load-tn (tn-ref-tn op)))))))
- (if load-tn
- (aver (eq load-scs t))
- (unless (eq load-scs t)
- (setf (tn-ref-load-tn op)
- (pack-load-tn (car scs) op))))))))
+ (when target
+ (let* ((load-tn (tn-ref-load-tn op))
+ (load-scs (svref (car scs)
+ (sc-number
+ (tn-sc (or load-tn (tn-ref-tn op)))))))
+ (if load-tn
+ (aver (eq load-scs t))
+ (unless (eq load-scs t)
+ (setf (tn-ref-load-tn op)
+ (pack-load-tn (car scs) op))))))))
(do ((scs scs (cdr scs))
(op ops (tn-ref-across op)))
((null scs))
(let ((target (tn-ref-target op)))
- (unless target
- (let* ((load-tn (tn-ref-load-tn op))
- (load-scs (svref (car scs)
- (sc-number
- (tn-sc (or load-tn (tn-ref-tn op)))))))
- (if load-tn
- (aver (eq load-scs t))
- (unless (eq load-scs t)
- (setf (tn-ref-load-tn op)
- (pack-load-tn (car scs) op))))))))
+ (unless target
+ (let* ((load-tn (tn-ref-load-tn op))
+ (load-scs (svref (car scs)
+ (sc-number
+ (tn-sc (or load-tn (tn-ref-tn op)))))))
+ (if load-tn
+ (aver (eq load-scs t))
+ (unless (eq load-scs t)
+ (setf (tn-ref-load-tn op)
+ (pack-load-tn (car scs) op))))))))
(values))
(defun pack-load-tns (block)
(catch 'unpacked-tn
(let ((*live-block* nil)
- (*live-vop* nil))
+ (*live-vop* nil))
(do ((vop (ir2-block-last-vop block) (vop-prev vop)))
- ((null vop))
- (let ((info (vop-info vop)))
- (check-operand-restrictions (vop-info-result-load-scs info)
- (vop-results vop))
- (check-operand-restrictions (vop-info-arg-load-scs info)
- (vop-args vop))))))
+ ((null vop))
+ (let ((info (vop-info vop)))
+ (check-operand-restrictions (vop-info-result-load-scs info)
+ (vop-results vop))
+ (check-operand-restrictions (vop-info-arg-load-scs info)
+ (vop-args vop))))))
(values))
\f
;;;; targeting
(defun check-ok-target (target tn sc)
(declare (type tn target tn) (type sc sc) (inline member))
(let* ((loc (tn-offset target))
- (target-sc (tn-sc target))
- (target-sb (sc-sb target-sc)))
+ (target-sc (tn-sc target))
+ (target-sb (sc-sb target-sc)))
(declare (type index loc))
;; We can honor a preference if:
;; -- TARGET's location is in SC's locations.
;; -- The element sizes of the two SCs are the same.
;; -- TN doesn't conflict with target's location.
(if (and (eq target-sb (sc-sb sc))
- (or (eq (sb-kind target-sb) :unbounded)
- (member loc (sc-locations sc)))
- (= (sc-element-size target-sc) (sc-element-size sc))
- (not (conflicts-in-sc tn sc loc))
- (zerop (mod loc (sc-alignment sc))))
- loc
- nil)))
+ (or (eq (sb-kind target-sb) :unbounded)
+ (member loc (sc-locations sc)))
+ (= (sc-element-size target-sc) (sc-element-size sc))
+ (not (conflicts-in-sc tn sc loc))
+ (zerop (mod loc (sc-alignment sc))))
+ loc
+ nil)))
;;; Scan along the target path from TN, looking at readers or writers.
-;;; When we find a packed TN, return CHECK-OK-TARGET of that TN. If
-;;; there is no target, or if the TN has multiple readers (writers),
-;;; then we return NIL. We also always return NIL after 10 iterations
-;;; to get around potential circularity problems.
+;;; When we find a TN, call CALLEE with that TN, and then resume
+;;; walking down that TN's target. As soon as there is no target, or
+;;; if the TN has multiple readers (writers), we stop walking the
+;;; targetting chain. We also always stop after 10 iterations to get
+;;; around potential circularity problems.
;;;
-;;; FIXME: (30 minutes of reverse engineering?) It'd be nice to
-;;; rewrite the header comment here to explain the interface and its
-;;; motivation, and move remarks about implementation details (like
-;;; 10!) inside.
+;;; Why the single-reader/writer constraint? As far as I can tell,
+;;; this is concerned with straight pipeline of data, e.g. CASTs. In
+;;; that case, limiting to chains of length 10 seems to be more than
+;;; enough.
+(declaim (inline %call-with-target-tns))
+(defun %call-with-target-tns (tn callee
+ &key (limit 10) (reads t) (writes t))
+ (declare (type tn tn) (type function callee) (type index limit))
+ (flet ((frob-slot (slot-function)
+ (declare (type function slot-function))
+ (let ((count limit)
+ (current tn))
+ (declare (type index count))
+ (loop
+ (let ((refs (funcall slot-function current)))
+ (unless (and (plusp count)
+ refs
+ (not (tn-ref-next refs)))
+ (return nil))
+ (let ((target (tn-ref-target refs)))
+ (unless target (return nil))
+ (setq current (tn-ref-tn target))
+ (funcall callee current)
+ (decf count)))))))
+ (when reads
+ (frob-slot #'tn-reads))
+ (when writes
+ (frob-slot #'tn-writes))
+ nil))
+
+(defmacro do-target-tns ((target-variable source-tn
+ &rest keys &key limit reads writes)
+ &body body)
+ (declare (ignore limit reads writes))
+ (let ((callback (gensym "CALLBACK")))
+ `(flet ((,callback (,target-variable)
+ ,@body))
+ (declare (dynamic-extent #',callback))
+ (%call-with-target-tns ,source-tn #',callback ,@keys))))
+
(defun find-ok-target-offset (tn sc)
(declare (type tn tn) (type sc sc))
- (flet ((frob-slot (slot-fun)
- (declare (type function slot-fun))
- (let ((count 10)
- (current tn))
- (declare (type index count))
- (loop
- (let ((refs (funcall slot-fun current)))
- (unless (and (plusp count)
- refs
- (not (tn-ref-next refs)))
- (return nil))
- (let ((target (tn-ref-target refs)))
- (unless target (return nil))
- (setq current (tn-ref-tn target))
- (when (tn-offset current)
- (return (check-ok-target current tn sc)))
- (decf count)))))))
- (declare (inline frob-slot)) ; until DYNAMIC-EXTENT works
- (or (frob-slot #'tn-reads)
- (frob-slot #'tn-writes))))
+ (do-target-tns (target tn)
+ (awhen (and (tn-offset target)
+ (check-ok-target target tn sc))
+ (return-from find-ok-target-offset it))))
\f
;;;; location selection
;;; Select some location for TN in SC, returning the offset if we
-;;; succeed, and NIL if we fail. We start scanning at the Last-Offset
-;;; in an attempt to distribute the TNs across all storage.
-;;;
-;;; We call OFFSET-CONFLICTS-IN-SB directly, rather than using
-;;; CONFLICTS-IN-SC. This allows us to more efficient in packing
-;;; multi-location TNs: we don't have to multiply the number of tests
-;;; by the TN size. This falls out naturally, since we have to be
-;;; aware of TN size anyway so that we don't call CONFLICTS-IN-SC on a
-;;; bogus offset.
-;;;
-;;; We give up on finding a location after our current pointer has
-;;; wrapped twice. This will result in testing some locations twice in
-;;; the case that we fail, but is simpler than trying to figure out
-;;; the soonest failure point.
+;;; succeed, and NIL if we fail.
;;;
-;;; We also give up without bothering to wrap if the current size
-;;; isn't large enough to hold a single element of element-size
-;;; without bothering to wrap. If it doesn't fit this iteration, it
-;;; won't fit next.
+;;; For :UNBOUNDED SCs just find the smallest correctly aligned offset
+;;; where the TN doesn't conflict with the TNs that have already been
+;;; packed. For :FINITE SCs try to pack the TN into the most heavily
+;;; used locations first (as estimated in FIND-LOCATION-USAGE).
;;;
-;;; ### Note that we actually try to pack as many consecutive TNs as
-;;; possible in the same location, since we start scanning at the same
-;;; offset that the last TN was successfully packed in. This is a
-;;; weakening of the scattering hueristic that was put in to prevent
-;;; restricted VOP temps from hogging all of the registers. This way,
-;;; all of these temps probably end up in one register.
-(defun select-location (tn sc &optional use-reserved-locs)
+;;; Historically SELECT-LOCATION tried did the opposite and tried to
+;;; distribute the TNs evenly across the available locations. At least
+;;; on register-starved architectures (x86) this seems to be a bad
+;;; strategy. -- JES 2004-09-11
+(defun select-location (tn sc &key use-reserved-locs optimize)
(declare (type tn tn) (type sc sc) (inline member))
(let* ((sb (sc-sb sc))
- (element-size (sc-element-size sc))
- (alignment (sc-alignment sc))
- (align-mask (1- alignment))
- (size (finite-sb-current-size sb))
- (start-offset (finite-sb-last-offset sb)))
- (let ((current-start
- (logandc2 (the index (+ start-offset align-mask)) align-mask))
- (wrap-p nil))
- (declare (type index current-start))
- (loop
- (when (> (+ current-start element-size) size)
- (cond ((or wrap-p (> element-size size))
- (return nil))
- (t
- (setq current-start 0)
- (setq wrap-p t))))
-
- (if (or (eq (sb-kind sb) :unbounded)
- (and (member current-start (sc-locations sc))
- (or use-reserved-locs
- (not (member current-start
- (sc-reserve-locations sc))))))
- (dotimes (i element-size
- (return-from select-location current-start))
- (declare (type index i))
- (let ((offset (+ current-start i)))
- (when (offset-conflicts-in-sb tn sb offset)
- (setq current-start
- (logandc2 (the index (+ (the index (1+ offset))
- align-mask))
- align-mask))
- (return))))
- (incf current-start alignment))))))
+ (element-size (sc-element-size sc))
+ (alignment (sc-alignment sc))
+ (align-mask (1- alignment))
+ (size (finite-sb-current-size sb)))
+ (flet ((attempt-location (start-offset)
+ (let ((conflict (conflicts-in-sc tn sc start-offset)))
+ (if conflict
+ (logandc2 (+ conflict align-mask 1)
+ align-mask)
+ (return-from select-location start-offset)))))
+ (if (eq (sb-kind sb) :unbounded)
+ (loop with offset = 0
+ until (> (+ offset element-size) size) do
+ (setf offset (attempt-location offset)))
+ (let ((locations (sc-locations sc)))
+ (when optimize
+ (setf locations
+ (schwartzian-stable-sort-list
+ locations '>
+ :key (lambda (location-offset)
+ (loop for offset from location-offset
+ repeat element-size
+ maximize (svref
+ (finite-sb-always-live-count sb)
+ offset))))))
+ (dolist (offset locations)
+ (when (or use-reserved-locs
+ (not (member offset
+ (sc-reserve-locations sc))))
+ (attempt-location offset))))))))
;;; If a save TN, return the saved TN, otherwise return TN. This is
;;; useful for getting the conflicts of a TN that might be a save TN.
\f
;;;; pack interface
+;; Misc. utilities
+(declaim (inline unbounded-sc-p))
+(defun unbounded-sc-p (sc)
+ (eq (sb-kind (sc-sb sc)) :unbounded))
+
+(defun unbounded-tn-p (tn)
+ (unbounded-sc-p (tn-sc tn)))
+(declaim (notinline unbounded-sc-p))
+
;;; Attempt to pack TN in all possible SCs, first in the SC chosen by
;;; representation selection, then in the alternate SCs in the order
;;; they were specified in the SC definition. If the TN-COST is
;;; If we are attempting to pack in the SC of the save TN for a TN
;;; with a :SPECIFIED-SAVE TN, then we pack in that location, instead
;;; of allocating a new stack location.
-(defun pack-tn (tn restricted)
+(defun pack-tn (tn restricted optimize &key (allow-unbounded-sc t))
(declare (type tn tn))
+ (aver (not (tn-offset tn)))
(let* ((original (original-tn tn))
- (fsc (tn-sc tn))
- (alternates (unless restricted (sc-alternate-scs fsc)))
- (save (tn-save-tn tn))
- (specified-save-sc
- (when (and save
- (eq (tn-kind save) :specified-save))
- (tn-sc save))))
-
+ (fsc (tn-sc tn))
+ (alternates (unless restricted (sc-alternate-scs fsc)))
+ (save (tn-save-tn tn))
+ (specified-save-sc
+ (when (and save
+ (eq (tn-kind save) :specified-save))
+ (tn-sc save))))
(do ((sc fsc (pop alternates)))
- ((null sc)
- (failed-to-pack-error tn restricted))
+ ((null sc)
+ (failed-to-pack-error tn restricted))
+ (unless (or allow-unbounded-sc
+ (not (unbounded-sc-p sc)))
+ (return nil))
(when (eq sc specified-save-sc)
- (unless (tn-offset save)
- (pack-tn save nil))
- (setf (tn-offset tn) (tn-offset save))
- (setf (tn-sc tn) (tn-sc save))
- (return))
+ (unless (tn-offset save)
+ (pack-tn save nil optimize))
+ (setf (tn-offset tn) (tn-offset save))
+ (setf (tn-sc tn) (tn-sc save))
+ (return t))
(when (or restricted
- (not (and (minusp (tn-cost tn)) (sc-save-p sc))))
- (let ((loc (or (find-ok-target-offset original sc)
- (select-location original sc)
- (and restricted
- (select-location original sc t))
- (when (eq (sb-kind (sc-sb sc)) :unbounded)
- (grow-sc sc)
- (or (select-location original sc)
- (error "failed to pack after growing SC?"))))))
- (when loc
- (add-location-conflicts original sc loc)
- (setf (tn-sc tn) sc)
- (setf (tn-offset tn) loc)
- (return))))))
-
+ (not (and (minusp (tn-cost tn)) (sc-save-p sc))))
+ (let ((loc (or (find-ok-target-offset original sc)
+ (select-location original sc :optimize optimize)
+ (and restricted
+ (select-location original sc :use-reserved-locs t
+ :optimize optimize))
+ (when (unbounded-sc-p sc)
+ (grow-sc sc)
+ (or (select-location original sc)
+ (error "failed to pack after growing SC?"))))))
+ (when loc
+ (add-location-conflicts original sc loc optimize)
+ (setf (tn-sc tn) sc)
+ (setf (tn-offset tn) loc)
+ (return t))))))
(values))
;;; Pack a wired TN, checking that the offset is in bounds for the SB,
;;; locations for local call arguments (such as OLD-FP) work, since
;;; the caller and callee OLD-FP save locations may conflict when the
;;; save locations don't really (due to being in different frames.)
-(defun pack-wired-tn (tn)
+(defun pack-wired-tn (tn optimize)
(declare (type tn tn))
(let* ((sc (tn-sc tn))
- (sb (sc-sb sc))
- (offset (tn-offset tn))
- (end (+ offset (sc-element-size sc)))
- (original (original-tn tn)))
+ (sb (sc-sb sc))
+ (offset (tn-offset tn))
+ (end (+ offset (sc-element-size sc)))
+ (original (original-tn tn)))
(when (> end (finite-sb-current-size sb))
(unless (eq (sb-kind sb) :unbounded)
- (error "~S is wired to a location that is out of bounds." tn))
+ (error "~S is wired to a location that is out of bounds." tn))
(grow-sc sc end))
;; For non-x86 ports the presence of a save-tn associated with a
;; tn is used to identify the old-fp and return-pc tns. It depends
;; on the old-fp and return-pc being passed in registers.
- #!-x86
+ #!-(or x86 x86-64)
(when (and (not (eq (tn-kind tn) :specified-save))
- (conflicts-in-sc original sc offset))
+ (conflicts-in-sc original sc offset))
(error "~S is wired to a location that it conflicts with." tn))
;; Use the above check, but only print a verbose warning. This can
;; be helpful for debugging the x86 port.
#+nil
(when (and (not (eq (tn-kind tn) :specified-save))
- (conflicts-in-sc original sc offset))
- (format t "~&* Pack-wired-tn possible conflict:~% ~
- tn: ~S; tn-kind: ~S~% ~
- sc: ~S~% ~
- sb: ~S; sb-name: ~S; sb-kind: ~S~% ~
- offset: ~S; end: ~S~% ~
- original ~S~% ~
- tn-save-tn: ~S; tn-kind of tn-save-tn: ~S~%"
- tn (tn-kind tn) sc
- sb (sb-name sb) (sb-kind sb)
- offset end
- original
- (tn-save-tn tn) (tn-kind (tn-save-tn tn))))
+ (conflicts-in-sc original sc offset))
+ (format t "~&* Pack-wired-tn possible conflict:~% ~
+ tn: ~S; tn-kind: ~S~% ~
+ sc: ~S~% ~
+ sb: ~S; sb-name: ~S; sb-kind: ~S~% ~
+ offset: ~S; end: ~S~% ~
+ original ~S~% ~
+ tn-save-tn: ~S; tn-kind of tn-save-tn: ~S~%"
+ tn (tn-kind tn) sc
+ sb (sb-name sb) (sb-kind sb)
+ offset end
+ original
+ (tn-save-tn tn) (tn-kind (tn-save-tn tn))))
;; On the x86 ports the old-fp and return-pc are often passed on
;; the stack so the above hack for the other ports does not always
;; work. Here the old-fp and return-pc tns are identified by being
;; on the stack in their standard save locations.
- #!+x86
+ #!+(or x86 x86-64)
(when (and (not (eq (tn-kind tn) :specified-save))
- (not (and (string= (sb-name sb) "STACK")
- (or (= offset 0)
- (= offset 1))))
- (conflicts-in-sc original sc offset))
- (error "~S is wired to a location that it conflicts with." tn))
+ (not (and (string= (sb-name sb) "STACK")
+ (or (= offset 0)
+ (= offset 1))))
+ (conflicts-in-sc original sc offset))
+ (error "~S is wired to location ~D in SC ~A of kind ~S that it conflicts with."
+ tn offset sc (tn-kind tn)))
- (add-location-conflicts original sc offset)))
+ (add-location-conflicts original sc offset optimize)))
(defevent repack-block "Repacked a block due to TN unpacking.")
+;;; KLUDGE: Prior to SBCL version 0.8.9.xx, this function was known as
+;;; PACK-BEFORE-GC-HOOK, but was non-functional since approximately
+;;; version 0.8.3.xx since the removal of GC hooks from the system.
+;;; This currently (as of 2004-04-12) runs now after every call to
+;;; PACK, rather than -- as was originally intended -- once per GC
+;;; cycle; this is probably non-optimal, and might require tuning,
+;;; maybe to be called when the data structures exceed a certain size,
+;;; or maybe once every N times. The KLUDGE is that this rewrite has
+;;; done nothing to improve the reentrance or threadsafety of the
+;;; compiler; it still fails to be callable from several threads at
+;;; the same time.
+;;;
+;;; Brief experiments indicate that during a compilation cycle this
+;;; causes about 10% more consing, and takes about 1%-2% more time.
+;;;
+;;; -- CSR, 2004-04-12
+(defun clean-up-pack-structures ()
+ (dolist (sb *backend-sb-list*)
+ (unless (eq (sb-kind sb) :non-packed)
+ (let ((size (sb-size sb)))
+ (fill (finite-sb-always-live sb) nil)
+ (setf (finite-sb-always-live sb)
+ (make-array size
+ :initial-element
+ #-sb-xc #*
+ ;; The cross-compiler isn't very good at
+ ;; dumping specialized arrays, so we delay
+ ;; construction of this SIMPLE-BIT-VECTOR
+ ;; until runtime.
+ #+sb-xc (make-array 0 :element-type 'bit)))
+ (setf (finite-sb-always-live-count sb)
+ (make-array size
+ :initial-element
+ #-sb-xc #*
+ ;; Ibid
+ #+sb-xc (make-array 0 :element-type 'fixnum)))
+
+ (fill (finite-sb-conflicts sb) nil)
+ (setf (finite-sb-conflicts sb)
+ (make-array size :initial-element '#()))
+
+ (fill (finite-sb-live-tns sb) nil)
+ (setf (finite-sb-live-tns sb)
+ (make-array size :initial-element nil))))))
+
+(defun tn-lexical-depth (tn)
+ (let ((path t)) ; dummy initial value
+ (labels ((path (lambda)
+ (do ((acc '())
+ (lambda lambda (lambda-parent lambda)))
+ ((null lambda) acc)
+ (push lambda acc)))
+ (register-scope (lambda)
+ (let ((new-path (path lambda)))
+ (setf path (if (eql path t)
+ new-path
+ (subseq path
+ 0 (mismatch path new-path))))))
+ (walk-tn-refs (ref)
+ (do ((ref ref (tn-ref-next ref)))
+ ((or (null ref)
+ (null path)))
+ (awhen (vop-node (tn-ref-vop ref))
+ (register-scope (lexenv-lambda (node-lexenv it)))))))
+ (walk-tn-refs (tn-reads tn))
+ (walk-tn-refs (tn-writes tn))
+ (if (eql path t)
+ most-positive-fixnum
+ (length path)))))
+
(defun pack (component)
- (aver (not *in-pack*))
- (let ((*in-pack* t)
- (optimize (policy *lexenv*
- (or (>= speed compilation-speed)
- (>= space compilation-speed))))
- (2comp (component-info component)))
- (init-sb-vectors component)
-
- ;; Call the target functions.
- (do-ir2-blocks (block component)
- (do ((vop (ir2-block-start-vop block) (vop-next vop)))
- ((null vop))
- (let ((target-fun (vop-info-target-fun (vop-info vop))))
- (when target-fun
- (funcall target-fun vop)))))
-
-
- ;; Pack wired TNs first.
- (do ((tn (ir2-component-wired-tns 2comp) (tn-next tn)))
- ((null tn))
- (pack-wired-tn tn))
-
- ;; Pack restricted component TNs.
- (do ((tn (ir2-component-restricted-tns 2comp) (tn-next tn)))
- ((null tn))
- (when (eq (tn-kind tn) :component)
- (pack-tn tn t)))
+ (unwind-protect
+ (let ((optimize nil)
+ (2comp (component-info component)))
+ (init-sb-vectors component)
+
+ ;; Determine whether we want to do more expensive packing by
+ ;; checking whether any blocks in the component have (> SPEED
+ ;; COMPILE-SPEED).
+ ;;
+ ;; FIXME: This means that a declaration can have a minor
+ ;; effect even outside its scope, and as the packing is done
+ ;; component-globally it'd be tricky to use strict scoping. I
+ ;; think this is still acceptable since it's just a tradeoff
+ ;; between compilation speed and allocation quality and
+ ;; doesn't affect the semantics of the generated code in any
+ ;; way. -- JES 2004-10-06
+ (do-ir2-blocks (block component)
+ (when (policy (block-last (ir2-block-block block))
+ (> speed compilation-speed))
+ (setf optimize t)
+ (return)))
+
+ ;; Call the target functions.
+ (do-ir2-blocks (block component)
+ (do ((vop (ir2-block-start-vop block) (vop-next vop)))
+ ((null vop))
+ (let ((target-fun (vop-info-target-fun (vop-info vop))))
+ (when target-fun
+ (funcall target-fun vop)))))
+
+ ;; Assign costs to normal TNs so we know which ones should always
+ ;; be packed on the stack, and which are important not to spill.
+ (when *pack-assign-costs*
+ (assign-tn-costs component))
+
+ ;; Actually allocate registers for most TNs. After this, only
+ ;; :normal tns may be left unallocated (or TNs :restricted to
+ ;; an unbounded SC).
+ (pack-greedy component 2comp optimize)
+
+ ;; Pack any leftover normal/restricted TN that is not already
+ ;; allocated to a finite SC, or TNs that do not appear in any
+ ;; local TN map (e.g. :MORE TNs). Since we'll likely be
+ ;; allocating on the stack, first allocate TNs that are
+ ;; associated with code at shallow lexical depths: this will
+ ;; allocate long live ranges (i.e. TNs with more conflicts)
+ ;; first, and hopefully minimise stack fragmentation.
+ ;; Component TNs are a degenerate case: they are always live.
+ (let ((component-tns '())
+ (contiguous-tns '())
+ (tns '()))
+ (flet ((register-tn (tn)
+ (unless (tn-offset tn)
+ (case (tn-kind tn)
+ (:component
+ (push tn component-tns))
+ ((:environment :debug-environment)
+ (push tn contiguous-tns))
+ (t
+ (push tn tns))))))
+ (do ((tn (ir2-component-restricted-tns 2comp) (tn-next tn)))
+ ((null tn))
+ ;; by this time, restricted TNs must either be
+ ;; allocated in the right SC or unbounded
+ (aver (or (tn-offset tn) (unbounded-tn-p tn)))
+ (register-tn tn))
+ (do ((tn (ir2-component-normal-tns 2comp) (tn-next tn)))
+ ((null tn))
+ (register-tn tn)))
+ (flet ((pack-tns (tns &optional in-order)
+ (dolist (tn (if in-order
+ tns
+ (schwartzian-stable-sort-list
+ tns #'< :key #'tn-lexical-depth)))
+ (unless (tn-offset tn)
+ (pack-tn tn nil optimize)))))
+ ;; first pack TNs that are known to have simple live
+ ;; ranges (contiguous lexical scopes)
+ (pack-tns component-tns t)
+ (pack-tns contiguous-tns)
+ (pack-tns tns)))
+
+ ;; Do load TN packing and emit saves.
+ (let ((*repack-blocks* nil))
+ (cond ((and optimize *pack-optimize-saves*)
+ (optimized-emit-saves component)
+ (do-ir2-blocks (block component)
+ (pack-load-tns block)))
+ (t
+ (do-ir2-blocks (block component)
+ (emit-saves block)
+ (pack-load-tns block))))
+ (loop
+ (unless *repack-blocks* (return))
+ (let ((orpb *repack-blocks*))
+ (setq *repack-blocks* nil)
+ (dolist (block orpb)
+ (event repack-block)
+ (pack-load-tns block)))))
+
+ (values))
+ (clean-up-pack-structures)))
+
+(defun pack-greedy (component 2comp optimize)
+ (declare (type component component)
+ (type ir2-component 2comp))
+ ;; Pack wired TNs first.
+ (do ((tn (ir2-component-wired-tns 2comp) (tn-next tn)))
+ ((null tn))
+ (pack-wired-tn tn optimize))
- ;; Pack other restricted TNs.
+ ;; Then, pack restricted TNs, ones that are live over the whole
+ ;; component first (they cause no fragmentation). Sort by TN cost
+ ;; to help important TNs get good targeting.
+ (collect ((component)
+ (normal))
(do ((tn (ir2-component-restricted-tns 2comp) (tn-next tn)))
- ((null tn))
- (unless (tn-offset tn)
- (pack-tn tn t)))
-
- ;; Assign costs to normal TNs so we know which ones should always
- ;; be packed on the stack.
- (when (and optimize *pack-assign-costs*)
- (assign-tn-costs component))
-
- ;; Pack normal TNs in the order that they appear in the code. This
- ;; should have some tendency to pack important TNs first, since
- ;; control analysis favors the drop-through. This should also help
- ;; targeting, since we will pack the target TN soon after we
- ;; determine the location of the targeting TN.
- (do-ir2-blocks (block component)
- (let ((ltns (ir2-block-local-tns block)))
- (do ((i (1- (ir2-block-local-tn-count block)) (1- i)))
- ((minusp i))
- (declare (fixnum i))
- (let ((tn (svref ltns i)))
- (unless (or (null tn) (eq tn :more) (tn-offset tn))
- (pack-tn tn nil))))))
-
- ;; Pack any leftover normal TNs. This is to deal with :MORE TNs,
- ;; which could possibly not appear in any local TN map.
- (do ((tn (ir2-component-normal-tns 2comp) (tn-next tn)))
- ((null tn))
- (unless (tn-offset tn)
- (pack-tn tn nil)))
-
- ;; Do load TN packing and emit saves.
- (let ((*repack-blocks* nil))
- (cond ((and optimize *pack-optimize-saves*)
- (optimized-emit-saves component)
- (do-ir2-blocks (block component)
- (pack-load-tns block)))
- (t
- (do-ir2-blocks (block component)
- (emit-saves block)
- (pack-load-tns block))))
- (when *repack-blocks*
- (loop
- (when (zerop (hash-table-count *repack-blocks*)) (return))
- (maphash (lambda (block v)
- (declare (ignore v))
- (remhash block *repack-blocks*)
- (event repack-block)
- (pack-load-tns block))
- *repack-blocks*)))))
-
- (values))
+ ((null tn))
+ (unless (or (tn-offset tn) (unbounded-tn-p tn))
+ (if (eq :component (tn-kind tn))
+ (component tn)
+ (normal tn))))
+ (flet ((pack-tns (tns)
+ (dolist (tn (stable-sort tns #'> :key #'tn-cost))
+ (pack-tn tn t optimize))))
+ (pack-tns (component))
+ (pack-tns (normal))))
+
+ (cond ((and *loop-analyze* *pack-assign-costs*)
+ ;; Allocate normal TNs, starting with the TNs that are
+ ;; heavily used in deep loops (which is taken into account in
+ ;; TN spill costs). Only allocate in finite SCs (i.e. not on
+ ;; the stack).
+ (collect ((tns))
+ (do ((tn (ir2-component-normal-tns 2comp) (tn-next tn)))
+ ((null tn))
+ (unless (or (tn-offset tn)
+ (eq (tn-kind tn) :more)
+ (unbounded-tn-p tn)
+ (and (sc-save-p (tn-sc tn)) ; SC caller-save, but TN
+ (minusp (tn-cost tn)))) ; lives over many calls
+ (tns tn)))
+ (dolist (tn (stable-sort (tns) #'> :key #'tn-cost))
+ (unless (tn-offset tn)
+ ;; if it can't fit in a bounded SC, the final pass will
+ ;; take care of stack packing.
+ (pack-tn tn nil optimize :allow-unbounded-sc nil)))))
+ (t
+ ;; If loop analysis has been disabled we might as well revert
+ ;; to the old behaviour of just packing TNs linearly as they
+ ;; appear.
+ (do-ir2-blocks (block component)
+ (let ((ltns (ir2-block-local-tns block)))
+ (do ((i (1- (ir2-block-local-tn-count block)) (1- i)))
+ ((minusp i))
+ (declare (fixnum i))
+ (let ((tn (svref ltns i)))
+ (unless (or (null tn)
+ (eq tn :more)
+ (tn-offset tn)
+ (unbounded-tn-p tn))
+ (pack-tn tn nil optimize :allow-unbounded-sc nil)))))))))
projects / sbcl.git / blobdiff