X-Git-Url: http://repo.macrolet.net/gitweb/?a=blobdiff_plain;f=src%2Fcompiler%2Fpack.lisp;h=4fa1d162d5a39665a91e583a1e2a999876b5d7c8;hb=3d544b84f2b7ecd617d220145a775079df6c7919;hp=e67f0f9812606bffbfa7f9a0984bef19deb03fd6;hpb=82653abf5573c22c691e2243b70647ecdaa6aea8;p=sbcl.git diff --git a/src/compiler/pack.lisp b/src/compiler/pack.lisp index e67f0f9..4fa1d16 100644 --- a/src/compiler/pack.lisp +++ b/src/compiler/pack.lisp @@ -11,7 +11,7 @@ ;;;; provided with absolutely no warranty. See the COPYING and CREDITS ;;;; files for more information. -(in-package "SB!C") +(in-package "SB!REGALLOC") ;;; for debugging: some parameters controlling which optimizations we ;;; attempt @@ -24,14 +24,14 @@ ;;;; conflict determination -;;; Return true if the element at the specified offset in SB has a -;;; conflict with TN: -;;; -- If a component-live TN (:component kind), then iterate over -;;; all the blocks. If the element at Offset is used anywhere in +;;; Return true if the element at the specified offset, or in any of +;;; the [size-1] subsequent offsets, in SB has a conflict with TN: +;;; -- If a component-live TN (:COMPONENT kind), then iterate over +;;; all the blocks. If the element at OFFSET is used anywhere in ;;; any of the component's blocks (always-live /= 0), then there ;;; is a conflict. ;;; -- If TN is global (Confs true), then iterate over the blocks TN -;;; is live in (using TN-Global-Conflicts). If the TN is live +;;; is live in (using TN-GLOBAL-CONFLICTS). If the TN is live ;;; everywhere in the block (:LIVE), then there is a conflict ;;; if the element at offset is used anywhere in the block ;;; (Always-Live /= 0). Otherwise, we use the local TN number for @@ -39,44 +39,58 @@ ;;; that block. ;;; -- If TN is local, then we just check for a conflict in the block ;;; it is local to. -(defun offset-conflicts-in-sb (tn sb offset) - (declare (type tn tn) (type finite-sb sb) (type index offset)) +;;; +;;; If there is a conflict, returns the first such conflicting offset. +(defun offset-conflicts-in-sb (tn sb offset &key (size 1)) + (declare (type tn tn) (type finite-sb sb) (type index offset size)) (let ((confs (tn-global-conflicts tn)) - (kind (tn-kind tn))) - (cond - ((eq kind :component) - (let ((loc-live (svref (finite-sb-always-live sb) offset))) - (dotimes (i (ir2-block-count *component-being-compiled*) nil) - (when (/= (sbit loc-live i) 0) - (return t))))) - (confs - (let ((loc-confs (svref (finite-sb-conflicts sb) offset)) - (loc-live (svref (finite-sb-always-live sb) offset))) - (do ((conf confs (global-conflicts-tn-next conf))) - ((null conf) - nil) - (let* ((block (global-conflicts-block conf)) - (num (ir2-block-number block))) - (if (eq (global-conflicts-kind conf) :live) - (when (/= (sbit loc-live num) 0) - (return t)) - (when (/= (sbit (svref loc-confs num) - (global-conflicts-number conf)) - 0) - (return t))))))) - (t - (/= (sbit (svref (svref (finite-sb-conflicts sb) offset) - (ir2-block-number (tn-local tn))) - (tn-local-number tn)) - 0))))) + (kind (tn-kind tn)) + (sb-conflicts (finite-sb-conflicts sb)) + (sb-always-live (finite-sb-always-live sb))) + (macrolet ((do-offsets (&body body) + `(loop repeat size + for offset upfrom offset + thereis (progn ,@body)))) + (cond + ((eq kind :component) + (do-offsets + (let ((loc-live (svref sb-always-live offset))) + (dotimes (i (ir2-block-count *component-being-compiled*)) + (when (/= (sbit loc-live i) 0) + (return offset)))))) + (confs + ;; TN is global, iterate over the blocks TN is live in. + (do ((conf confs (global-conflicts-next-tnwise conf))) + ((null conf) + nil) + (let* ((block (global-conflicts-block conf)) + (num (ir2-block-number block))) + (if (eq (global-conflicts-kind conf) :live) + (do-offsets + (let ((loc-live (svref sb-always-live offset))) + (when (/= (sbit loc-live num) 0) + (return-from offset-conflicts-in-sb offset)))) + (do-offsets + (let ((loc-confs (svref sb-conflicts offset))) + (when (/= (sbit (svref loc-confs num) + (global-conflicts-number conf)) + 0) + (return-from offset-conflicts-in-sb offset)))))))) + (t + (do-offsets + (and (/= (sbit (svref (svref sb-conflicts offset) + (ir2-block-number (tn-local tn))) + (tn-local-number tn)) + 0) + offset))))))) ;;; Return true if TN has a conflict in SC at the specified offset. +(declaim (ftype (function (tn sc index) (values (or null index) &optional)) + conflicts-in-sc)) (defun conflicts-in-sc (tn sc offset) (declare (type tn tn) (type sc sc) (type index offset)) - (let ((sb (sc-sb sc))) - (dotimes (i (sc-element-size sc) nil) - (when (offset-conflicts-in-sb tn sb (+ offset i)) - (return t))))) + (offset-conflicts-in-sb tn (sc-sb sc) offset + :size (sc-element-size sc))) ;;; Add TN's conflicts into the conflicts for the location at OFFSET ;;; in SC. We iterate over each location in TN, adding to the @@ -92,189 +106,183 @@ ;;; the bits, otherwise we OR in the local conflict bits. ;;; -- If the TN is local, then we just do the block it is local to, ;;; setting always-live and OR'ing in the local conflicts. -(defun add-location-conflicts (tn sc offset) +(defun add-location-conflicts (tn sc offset optimize) (declare (type tn tn) (type sc sc) (type index offset)) (let ((confs (tn-global-conflicts tn)) - (sb (sc-sb sc)) - (kind (tn-kind tn))) + (sb (sc-sb sc)) + (kind (tn-kind tn))) (dotimes (i (sc-element-size sc)) (declare (type index i)) (let* ((this-offset (+ offset i)) - (loc-confs (svref (finite-sb-conflicts sb) this-offset)) - (loc-live (svref (finite-sb-always-live sb) this-offset))) - (cond - ((eq kind :component) - (dotimes (num (ir2-block-count *component-being-compiled*) nil) - (declare (type index num)) - (setf (sbit loc-live num) 1) - (set-bit-vector (svref loc-confs num)))) - (confs - (do ((conf confs (global-conflicts-tn-next conf))) - ((null conf)) - (let* ((block (global-conflicts-block conf)) - (num (ir2-block-number block)) - (local-confs (svref loc-confs num))) - (declare (type local-tn-bit-vector local-confs)) - (setf (sbit loc-live num) 1) - (if (eq (global-conflicts-kind conf) :live) - (set-bit-vector local-confs) - (bit-ior local-confs (global-conflicts-conflicts conf) t))))) - (t - (let ((num (ir2-block-number (tn-local tn)))) - (setf (sbit loc-live num) 1) - (bit-ior (the local-tn-bit-vector (svref loc-confs num)) - (tn-local-conflicts tn) t)))))))) - -;;; Return the total number of IR2 blocks in Component. + (loc-confs (svref (finite-sb-conflicts sb) this-offset)) + (loc-live (svref (finite-sb-always-live sb) this-offset))) + (cond + ((eq kind :component) + (dotimes (num (ir2-block-count *component-being-compiled*)) + (declare (type index num)) + (setf (sbit loc-live num) 1) + (set-bit-vector (svref loc-confs num)))) + (confs + (do ((conf confs (global-conflicts-next-tnwise conf))) + ((null conf)) + (let* ((block (global-conflicts-block conf)) + (num (ir2-block-number block)) + (local-confs (svref loc-confs num))) + (declare (type local-tn-bit-vector local-confs)) + (setf (sbit loc-live num) 1) + (if (eq (global-conflicts-kind conf) :live) + (set-bit-vector local-confs) + (bit-ior local-confs (global-conflicts-conflicts conf) t))))) + (t + (let ((num (ir2-block-number (tn-local tn)))) + (setf (sbit loc-live num) 1) + (bit-ior (the local-tn-bit-vector (svref loc-confs num)) + (tn-local-conflicts tn) t)))) + ;; Calculating ALWAYS-LIVE-COUNT is moderately expensive, and + ;; currently the information isn't used unless (> SPEED + ;; COMPILE-SPEED). + (when optimize + (setf (svref (finite-sb-always-live-count sb) this-offset) + (find-location-usage sb this-offset)))))) + (values)) + +;; A rought measure of how much a given OFFSET in SB is currently +;; used. Current implementation counts the amount of blocks where the +;; offset has been marked as ALWAYS-LIVE. +(defun find-location-usage (sb offset) + (declare (optimize speed)) + (declare (type sb sb) (type index offset)) + (let* ((always-live (svref (finite-sb-always-live sb) offset))) + (declare (simple-bit-vector always-live)) + (count 1 always-live))) + +;;; Return the total number of IR2-BLOCKs in COMPONENT. (defun ir2-block-count (component) (declare (type component component)) (do ((2block (block-info (block-next (component-head component))) - (ir2-block-next 2block))) + (ir2-block-next 2block))) ((null 2block) (error "What? No ir2 blocks have a non-nil number?")) (when (ir2-block-number 2block) (return (1+ (ir2-block-number 2block)))))) -;;; Ensure that the conflicts vectors for each :Finite SB are large +;;; Ensure that the conflicts vectors for each :FINITE SB are large ;;; enough for the number of blocks allocated. Also clear any old ;;; conflicts and reset the current size to the initial size. (defun init-sb-vectors (component) (let ((nblocks (ir2-block-count component))) (dolist (sb *backend-sb-list*) (unless (eq (sb-kind sb) :non-packed) - (let* ((conflicts (finite-sb-conflicts sb)) - (always-live (finite-sb-always-live sb)) - (max-locs (length conflicts)) - (last-count (finite-sb-last-block-count sb))) - (unless (zerop max-locs) - (let ((current-size (length (the simple-vector - (svref conflicts 0))))) - (cond - ((> nblocks current-size) - (let ((new-size (max nblocks (* current-size 2)))) - (declare (type index new-size)) - (dotimes (i max-locs) - (declare (type index i)) - (let ((new-vec (make-array new-size))) - (let ((old (svref conflicts i))) - (declare (simple-vector old)) - (dotimes (j current-size) - (declare (type index j)) - (setf (svref new-vec j) - (clear-bit-vector (svref old j))))) - - (do ((j current-size (1+ j))) - ((= j new-size)) - (declare (type index j)) - (setf (svref new-vec j) - (make-array local-tn-limit :element-type 'bit - :initial-element 0))) - (setf (svref conflicts i) new-vec)) - (setf (svref always-live i) - (make-array new-size :element-type 'bit - :initial-element 0))))) - (t - (dotimes (i (finite-sb-current-size sb)) - (declare (type index i)) - (let ((conf (svref conflicts i))) - (declare (simple-vector conf)) - (dotimes (j last-count) - (declare (type index j)) - (clear-bit-vector (svref conf j)))) - (clear-bit-vector (svref always-live i))))))) - - (setf (finite-sb-last-block-count sb) nblocks) - (setf (finite-sb-current-size sb) (sb-size sb)) - (setf (finite-sb-last-offset sb) 0)))))) - -;;; Expand the :Unbounded SB backing SC by either the initial size or + (let* ((conflicts (finite-sb-conflicts sb)) + (always-live (finite-sb-always-live sb)) + (always-live-count (finite-sb-always-live-count sb)) + (max-locs (length conflicts)) + (last-count (finite-sb-last-block-count sb))) + (unless (zerop max-locs) + (let ((current-size (length (the simple-vector + (svref conflicts 0))))) + (cond + ((> nblocks current-size) + (let ((new-size (max nblocks (* current-size 2)))) + (declare (type index new-size)) + (dotimes (i max-locs) + (declare (type index i)) + (let ((new-vec (make-array new-size))) + (let ((old (svref conflicts i))) + (declare (simple-vector old)) + (dotimes (j current-size) + (declare (type index j)) + (setf (svref new-vec j) + (clear-bit-vector (svref old j))))) + + (do ((j current-size (1+ j))) + ((= j new-size)) + (declare (type index j)) + (setf (svref new-vec j) + (make-array local-tn-limit :element-type 'bit + :initial-element 0))) + (setf (svref conflicts i) new-vec)) + (setf (svref always-live i) + (make-array new-size :element-type 'bit + :initial-element 0)) + (setf (svref always-live-count i) 0)))) + (t + (dotimes (i (finite-sb-current-size sb)) + (declare (type index i)) + (let ((conf (svref conflicts i))) + (declare (simple-vector conf)) + (dotimes (j last-count) + (declare (type index j)) + (clear-bit-vector (svref conf j)))) + (clear-bit-vector (svref always-live i)) + (setf (svref always-live-count i) 0)))))) + + (setf (finite-sb-last-block-count sb) nblocks) + (setf (finite-sb-current-size sb) (sb-size sb)) + (setf (finite-sb-last-offset sb) 0)))))) + +;;; Expand the :UNBOUNDED SB backing SC by either the initial size or ;;; the SC element size, whichever is larger. If NEEDED-SIZE is ;;; larger, then use that size. (defun grow-sc (sc &optional (needed-size 0)) (declare (type sc sc) (type index needed-size)) (let* ((sb (sc-sb sc)) - (size (finite-sb-current-size sb)) - (align-mask (1- (sc-alignment sc))) - (inc (max (sb-size sb) - (+ (sc-element-size sc) - (- (logandc2 (+ size align-mask) align-mask) - size)) - (- needed-size size))) - (new-size (+ size inc)) - (conflicts (finite-sb-conflicts sb)) - (block-size (if (zerop (length conflicts)) - (ir2-block-count *component-being-compiled*) - (length (the simple-vector (svref conflicts 0)))))) - (declare (type index inc new-size)) + (size (finite-sb-current-size sb)) + (align-mask (1- (sc-alignment sc))) + (inc (max (finite-sb-size-increment sb) + (+ (sc-element-size sc) + (- (logandc2 (+ size align-mask) align-mask) + size)) + (- needed-size size))) + (new-size (let ((align-mask (1- (finite-sb-size-alignment sb)))) + (logandc2 (+ size inc align-mask) align-mask))) + (conflicts (finite-sb-conflicts sb)) + (block-size (if (zerop (length conflicts)) + (ir2-block-count *component-being-compiled*) + (length (the simple-vector (svref conflicts 0))))) + (padded-size (ash 1 (integer-length (1- new-size))))) + (declare (type index inc new-size padded-size)) (aver (eq (sb-kind sb) :unbounded)) - (when (> new-size (length conflicts)) - (let ((new-conf (make-array new-size))) - (replace new-conf conflicts) - (do ((i size (1+ i))) - ((= i new-size)) - (declare (type index i)) - (let ((loc-confs (make-array block-size))) - (dotimes (j block-size) - (setf (svref loc-confs j) - (make-array local-tn-limit - :initial-element 0 - :element-type 'bit))) - (setf (svref new-conf i) loc-confs))) - (setf (finite-sb-conflicts sb) new-conf)) - - (let ((new-live (make-array new-size))) - (replace new-live (finite-sb-always-live sb)) - (do ((i size (1+ i))) - ((= i new-size)) - (setf (svref new-live i) - (make-array block-size - :initial-element 0 - :element-type 'bit))) - (setf (finite-sb-always-live sb) new-live)) - - (let ((new-tns (make-array new-size :initial-element nil))) - (replace new-tns (finite-sb-live-tns sb)) - (fill (finite-sb-live-tns sb) nil) - (setf (finite-sb-live-tns sb) new-tns))) + (when (> padded-size (length conflicts)) + (let ((new-conf (make-array padded-size))) + (replace new-conf conflicts) + (do ((i size (1+ i))) + ((= i padded-size)) + (declare (type index i)) + (let ((loc-confs (make-array block-size))) + (dotimes (j block-size) + (setf (svref loc-confs j) + (make-array local-tn-limit + :initial-element 0 + :element-type 'bit))) + (setf (svref new-conf i) loc-confs))) + (setf (finite-sb-conflicts sb) new-conf)) + + (let ((new-live (make-array padded-size))) + (replace new-live (finite-sb-always-live sb)) + (do ((i size (1+ i))) + ((= i padded-size)) + (setf (svref new-live i) + (make-array block-size + :initial-element 0 + :element-type 'bit))) + (setf (finite-sb-always-live sb) new-live)) + + (let ((new-live-count (make-array padded-size))) + (declare (optimize speed)) ;; FILL deftransform + (replace new-live-count (finite-sb-always-live-count sb)) + (fill new-live-count 0 :start size) + (setf (finite-sb-always-live-count sb) new-live-count)) + + (let ((new-tns (make-array padded-size :initial-element nil))) + (replace new-tns (finite-sb-live-tns sb)) + (fill (finite-sb-live-tns sb) nil) + (setf (finite-sb-live-tns sb) new-tns))) (setf (finite-sb-current-size sb) new-size)) (values)) -;;; This variable is true whenever we are in pack (and thus the per-SB -;;; conflicts information is in use.) -(defvar *in-pack* nil) - -;;; In order to prevent the conflict data structures from growing -;;; arbitrarily large, we clear them whenever a GC happens and we -;;; aren't currently in pack. We revert to the initial number of -;;; locations and 0 blocks. -(defun pack-before-gc-hook () - (unless *in-pack* - (dolist (sb *backend-sb-list*) - (unless (eq (sb-kind sb) :non-packed) - (let ((size (sb-size sb))) - (fill nil (finite-sb-always-live sb)) - (setf (finite-sb-always-live sb) - (make-array size - :initial-element - #-sb-xc #* - ;; The cross-compiler isn't very good at - ;; dumping specialized arrays, so we delay - ;; construction of this SIMPLE-BIT-VECTOR - ;; until runtime. - #+sb-xc (make-array 0 :element-type 'bit))) - - (fill nil (finite-sb-conflicts sb)) - (setf (finite-sb-conflicts sb) - (make-array size :initial-element '#())) - - (fill nil (finite-sb-live-tns sb)) - (setf (finite-sb-live-tns sb) - (make-array size :initial-element nil)))))) - (values)) - -(pushnew 'pack-before-gc-hook sb!ext:*before-gc-hooks*) ;;;; internal errors @@ -282,64 +290,64 @@ ;;; defined to move from SRC to DEST. (defun no-load-fun-error (src dest) (let* ((src-sc (tn-sc src)) - (src-name (sc-name src-sc)) - (dest-sc (tn-sc dest)) - (dest-name (sc-name dest-sc))) + (src-name (sc-name src-sc)) + (dest-sc (tn-sc dest)) + (dest-name (sc-name dest-sc))) (cond ((eq (sb-kind (sc-sb src-sc)) :non-packed) - (unless (member src-sc (sc-constant-scs dest-sc)) - (error "loading from an invalid constant SC?~@ - VM definition inconsistent, try recompiling.")) - (error "no load function defined to load SC ~S ~ - from its constant SC ~S" - dest-name src-name)) - ((member src-sc (sc-alternate-scs dest-sc)) - (error "no load function defined to load SC ~S from its ~ - alternate SC ~S" - dest-name src-name)) - ((member dest-sc (sc-alternate-scs src-sc)) - (error "no load function defined to save SC ~S in its ~ - alternate SC ~S" - src-name dest-name)) - (t - ;; FIXME: "VM definition is inconsistent" shouldn't be a - ;; possibility in SBCL. - (error "loading to/from SCs that aren't alternates?~@ - VM definition is inconsistent, try recompiling."))))) + (unless (member src-sc (sc-constant-scs dest-sc)) + (error "loading from an invalid constant SC?~@ + VM definition inconsistent, try recompiling.")) + (error "no load function defined to load SC ~S ~ + from its constant SC ~S" + dest-name src-name)) + ((member src-sc (sc-alternate-scs dest-sc)) + (error "no load function defined to load SC ~S from its ~ + alternate SC ~S" + dest-name src-name)) + ((member dest-sc (sc-alternate-scs src-sc)) + (error "no load function defined to save SC ~S in its ~ + alternate SC ~S" + src-name dest-name)) + (t + ;; FIXME: "VM definition is inconsistent" shouldn't be a + ;; possibility in SBCL. + (error "loading to/from SCs that aren't alternates?~@ + VM definition is inconsistent, try recompiling."))))) ;;; Called when we failed to pack TN. If RESTRICTED is true, then we ;;; are restricted to pack TN in its SC. (defun failed-to-pack-error (tn restricted) (declare (type tn tn)) (let* ((sc (tn-sc tn)) - (scs (cons sc (sc-alternate-scs sc)))) + (scs (cons sc (sc-alternate-scs sc)))) (cond (restricted (error "failed to pack restricted TN ~S in its SC ~S" - tn (sc-name sc))) + tn (sc-name sc))) (t (aver (not (find :unbounded scs - :key (lambda (x) (sb-kind (sc-sb x)))))) + :key (lambda (x) (sb-kind (sc-sb x)))))) (let ((ptype (tn-primitive-type tn))) - (cond - (ptype - (aver (member (sc-number sc) (primitive-type-scs ptype))) - (error "SC ~S doesn't have any :Unbounded alternate SCs, but is~@ - a SC for primitive-type ~S." - (sc-name sc) (primitive-type-name ptype))) - (t - (error "SC ~S doesn't have any :Unbounded alternate SCs." - (sc-name sc))))))))) + (cond + (ptype + (aver (member (sc-number sc) (primitive-type-scs ptype))) + (error "SC ~S doesn't have any :UNBOUNDED alternate SCs, but is~@ + a SC for primitive-type ~S." + (sc-name sc) (primitive-type-name ptype))) + (t + (error "SC ~S doesn't have any :UNBOUNDED alternate SCs." + (sc-name sc))))))))) ;;; Return a list of format arguments describing how TN is used in ;;; OP's VOP. (defun describe-tn-use (loc tn op) (let* ((vop (tn-ref-vop op)) - (args (vop-args vop)) - (results (vop-results vop)) - (name (with-output-to-string (stream) - (print-tn-guts tn stream))) - (2comp (component-info *component-being-compiled*)) - temp) + (args (vop-args vop)) + (results (vop-results vop)) + (name (with-output-to-string (stream) + (print-tn-guts tn stream))) + (2comp (component-info *component-being-compiled*)) + temp) (cond ((setq temp (position-in #'tn-ref-across tn args :key #'tn-ref-tn)) `("~2D: ~A (~:R argument)" ,loc ,name ,(1+ temp))) @@ -348,15 +356,15 @@ ((setq temp (position-in #'tn-ref-across tn args :key #'tn-ref-load-tn)) `("~2D: ~A (~:R argument load TN)" ,loc ,name ,(1+ temp))) ((setq temp (position-in #'tn-ref-across tn results :key - #'tn-ref-load-tn)) + #'tn-ref-load-tn)) `("~2D: ~A (~:R result load TN)" ,loc ,name ,(1+ temp))) ((setq temp (position-in #'tn-ref-across tn (vop-temps vop) - :key #'tn-ref-tn)) + :key #'tn-ref-tn)) `("~2D: ~A (temporary ~A)" ,loc ,name - ,(operand-parse-name (elt (vop-parse-temps - (vop-parse-or-lose - (vop-info-name (vop-info vop)))) - temp)))) + ,(operand-parse-name (elt (vop-parse-temps + (vop-parse-or-lose + (vop-info-name (vop-info vop)))) + temp)))) ((eq (tn-kind tn) :component) `("~2D: ~A (component live)" ,loc ,name)) ((position-in #'tn-next tn (ir2-component-wired-tns 2comp)) @@ -371,66 +379,66 @@ (defun failed-to-pack-load-tn-error (scs op) (declare (list scs) (type tn-ref op)) (collect ((used) - (unused)) + (unused)) (dolist (sc scs) (let* ((sb (sc-sb sc)) - (confs (finite-sb-live-tns sb))) - (aver (eq (sb-kind sb) :finite)) - (dolist (el (sc-locations sc)) - (declare (type index el)) - (let ((conf (load-tn-conflicts-in-sc op sc el t))) - (if conf - (used (describe-tn-use el conf op)) - (do ((i el (1+ i)) - (end (+ el (sc-element-size sc)))) - ((= i end) - (unused el)) - (declare (type index i end)) - (let ((victim (svref confs i))) - (when victim - (used (describe-tn-use el victim op)) - (return t))))))))) + (confs (finite-sb-live-tns sb))) + (aver (eq (sb-kind sb) :finite)) + (dolist (el (sc-locations sc)) + (declare (type index el)) + (let ((conf (load-tn-conflicts-in-sc op sc el t))) + (if conf + (used (describe-tn-use el conf op)) + (do ((i el (1+ i)) + (end (+ el (sc-element-size sc)))) + ((= i end) + (unused el)) + (declare (type index i end)) + (let ((victim (svref confs i))) + (when victim + (used (describe-tn-use el victim op)) + (return t))))))))) (multiple-value-bind (arg-p n more-p costs load-scs incon) - (get-operand-info op) + (get-operand-info op) (declare (ignore costs load-scs)) - (aver (not more-p)) - (error "unable to pack a Load-TN in SC ~{~A~#[~^~;, or ~:;,~]~} ~ - for the ~:R ~:[result~;argument~] to~@ - the ~S VOP,~@ - ~:[since all SC elements are in use:~:{~%~@?~}~%~;~ - ~:*but these SC elements are not in use:~% ~S~%Bug?~*~]~ - ~:[~;~@ - Current cost info inconsistent with that in effect at compile ~ - time. Recompile.~%Compilation order may be incorrect.~]" - (mapcar #'sc-name scs) - n arg-p - (vop-info-name (vop-info (tn-ref-vop op))) - (unused) (used) - incon)))) + (aver (not more-p)) + (error "unable to pack a Load-TN in SC ~{~A~#[~^~;, or ~:;,~]~} ~ + for the ~:R ~:[result~;argument~] to~@ + the ~S VOP,~@ + ~:[since all SC elements are in use:~:{~%~@?~}~%~;~ + ~:*but these SC elements are not in use:~% ~S~%Bug?~*~]~ + ~:[~;~@ + Current cost info inconsistent with that in effect at compile ~ + time. Recompile.~%Compilation order may be incorrect.~]" + (mapcar #'sc-name scs) + n arg-p + (vop-info-name (vop-info (tn-ref-vop op))) + (unused) (used) + incon)))) ;;; This is called when none of the SCs that we can load OP into are ;;; allowed by OP's primitive-type. (defun no-load-scs-allowed-by-primitive-type-error (ref) (declare (type tn-ref ref)) (let* ((tn (tn-ref-tn ref)) - (ptype (tn-primitive-type tn))) + (ptype (tn-primitive-type tn))) (multiple-value-bind (arg-p pos more-p costs load-scs incon) - (get-operand-info ref) + (get-operand-info ref) (declare (ignore costs)) (aver (not more-p)) (error "~S is not valid as the ~:R ~:[result~;argument~] to VOP:~ - ~% ~S,~@ - since the TN's primitive type ~S doesn't allow any of the SCs~@ - allowed by the operand restriction:~% ~S~ - ~:[~;~@ - Current cost info inconsistent with that in effect at compile ~ - time. Recompile.~%Compilation order may be incorrect.~]" - tn pos arg-p - (template-name (vop-info (tn-ref-vop ref))) - (primitive-type-name ptype) - (mapcar #'sc-name (listify-restrictions load-scs)) - incon)))) + ~% ~S,~@ + since the TN's primitive type ~S doesn't allow any of the SCs~@ + allowed by the operand restriction:~% ~S~ + ~:[~;~@ + Current cost info inconsistent with that in effect at compile ~ + time. Recompile.~%Compilation order may be incorrect.~]" + tn pos arg-p + (template-name (vop-info (tn-ref-vop ref))) + (primitive-type-name ptype) + (mapcar #'sc-name (listify-restrictions load-scs)) + incon)))) ;;;; register saving @@ -449,40 +457,40 @@ (declare (type tn tn)) (let ((res (make-tn 0 :save nil nil))) (dolist (alt (sc-alternate-scs (tn-sc tn)) - (error "no unbounded alternate for SC ~S" - (sc-name (tn-sc tn)))) + (error "no unbounded alternate for SC ~S" + (sc-name (tn-sc tn)))) (when (eq (sb-kind (sc-sb alt)) :unbounded) - (setf (tn-save-tn tn) res) - (setf (tn-save-tn res) tn) - (setf (tn-sc res) alt) - (pack-tn res t) - (return res))))) + (setf (tn-save-tn tn) res) + (setf (tn-save-tn res) tn) + (setf (tn-sc res) alt) + (pack-tn res t nil) + (return res))))) ;;; Find the load function for moving from SRC to DEST and emit a ;;; MOVE-OPERAND VOP with that function as its info arg. (defun emit-operand-load (node block src dest before) (declare (type node node) (type ir2-block block) - (type tn src dest) (type (or vop null) before)) + (type tn src dest) (type (or vop null) before)) (emit-load-template node block - (template-or-lose 'move-operand) - src dest - (list (or (svref (sc-move-funs (tn-sc dest)) - (sc-number (tn-sc src))) - (no-load-fun-error src dest))) - before) + (template-or-lose 'move-operand) + src dest + (list (or (svref (sc-move-funs (tn-sc dest)) + (sc-number (tn-sc src))) + (no-load-fun-error src dest))) + before) (values)) ;;; Find the preceding use of the VOP NAME in the emit order, starting ;;; with VOP. We must find the VOP in the same IR1 block. (defun reverse-find-vop (name vop) (do* ((block (vop-block vop) (ir2-block-prev block)) - (last vop (ir2-block-last-vop block))) + (last vop (ir2-block-last-vop block))) (nil) (aver (eq (ir2-block-block block) (ir2-block-block (vop-block vop)))) (do ((current last (vop-prev current))) - ((null current)) + ((null current)) (when (eq (vop-info-name (vop-info current)) name) - (return-from reverse-find-vop current))))) + (return-from reverse-find-vop current))))) ;;; For TNs that have other than one writer, we save the TN before ;;; each call. If a local call (MOVE-ARGS is :LOCAL-CALL), then we @@ -493,18 +501,18 @@ ;;; which the values are known to be good. (defun save-complex-writer-tn (tn vop) (let ((save (or (tn-save-tn tn) - (pack-save-tn tn))) - (node (vop-node vop)) - (block (vop-block vop)) - (next (vop-next vop))) + (pack-save-tn tn))) + (node (vop-node vop)) + (block (vop-block vop)) + (next (vop-next vop))) (when (eq (tn-kind save) :specified-save) (setf (tn-kind save) :save)) (aver (eq (tn-kind save) :save)) (emit-operand-load node block tn save - (if (eq (vop-info-move-args (vop-info vop)) - :local-call) - (reverse-find-vop 'allocate-frame vop) - vop)) + (if (eq (vop-info-move-args (vop-info vop)) + :local-call) + (reverse-find-vop 'allocate-frame vop) + vop)) (emit-operand-load node block save tn next))) ;;; Return a VOP after which is an OK place to save the value of TN. @@ -523,17 +531,17 @@ (res nil)) ((null write) (when (and res - (do ((read (tn-reads tn) (tn-ref-next read))) - ((not read) t) - (when (eq (vop-info-move-args - (vop-info - (tn-ref-vop read))) - :local-call) - (return nil)))) - (tn-ref-vop res))) + (do ((read (tn-reads tn) (tn-ref-next read))) + ((not read) t) + (when (eq (vop-info-move-args + (vop-info + (tn-ref-vop read))) + :local-call) + (return nil)))) + (tn-ref-vop res))) (unless (eq (vop-info-name (vop-info (tn-ref-vop write))) - 'move-operand) + 'move-operand) (when res (return nil)) (setq res write)))) @@ -542,13 +550,13 @@ (defun save-single-writer-tn (tn) (declare (type tn tn)) (let* ((old-save (tn-save-tn tn)) - (save (or old-save (pack-save-tn tn))) - (writer (find-single-writer tn))) + (save (or old-save (pack-save-tn tn))) + (writer (find-single-writer tn))) (when (and writer - (or (not old-save) - (eq (tn-kind old-save) :specified-save))) + (or (not old-save) + (eq (tn-kind old-save) :specified-save))) (emit-operand-load (vop-node writer) (vop-block writer) - tn save (vop-next writer)) + tn save (vop-next writer)) (setf (tn-kind save) :save-once) t))) @@ -566,11 +574,11 @@ (declare (type tn tn) (type vop vop)) (let ((save (tn-save-tn tn))) (cond ((and save (eq (tn-kind save) :save-once)) - (restore-single-writer-tn tn vop)) - ((save-single-writer-tn tn) - (restore-single-writer-tn tn vop)) - (t - (save-complex-writer-tn tn vop)))) + (restore-single-writer-tn tn vop)) + ((save-single-writer-tn tn) + (restore-single-writer-tn tn vop)) + (t + (save-complex-writer-tn tn vop)))) (values)) ;;; Scan over the VOPs in BLOCK, emiting saving code for TNs noted in @@ -581,17 +589,17 @@ ((null vop)) (when (eq (vop-info-save-p (vop-info vop)) t) (do-live-tns (tn (vop-save-set vop) block) - (when (and (sc-save-p (tn-sc tn)) - (not (eq (tn-kind tn) :component))) - (basic-save-tn tn vop))))) + (when (and (sc-save-p (tn-sc tn)) + (not (eq (tn-kind tn) :component))) + (basic-save-tn tn vop))))) (values)) ;;;; optimized saving ;;; Save TN if it isn't a single-writer TN that has already been -;;; saved. If multi-write, we insert the save Before the specified -;;; VOP. Context is a VOP used to tell which node/block to use for the +;;; saved. If multi-write, we insert the save BEFORE the specified +;;; VOP. CONTEXT is a VOP used to tell which node/block to use for the ;;; new VOP. (defun save-if-necessary (tn before context) (declare (type tn tn) (type (or vop null) before) (type vop context)) @@ -601,34 +609,20 @@ (aver (member (tn-kind save) '(:save :save-once))) (unless (eq (tn-kind save) :save-once) (or (save-single-writer-tn tn) - (emit-operand-load (vop-node context) (vop-block context) - tn save before)))) + (emit-operand-load (vop-node context) (vop-block context) + tn save before)))) (values)) ;;; Load the TN from its save location, allocating one if necessary. -;;; The load is inserted Before the specifier VOP. Context is a VOP +;;; The load is inserted BEFORE the specified VOP. CONTEXT is a VOP ;;; used to tell which node/block to use for the new VOP. (defun restore-tn (tn before context) (declare (type tn tn) (type (or vop null) before) (type vop context)) (let ((save (or (tn-save-tn tn) (pack-save-tn tn)))) (emit-operand-load (vop-node context) (vop-block context) - save tn before)) + save tn before)) (values)) -(eval-when (:compile-toplevel :execute) - -;;; Do stuff to note a read of TN, for OPTIMIZED-EMIT-SAVES-BLOCK. -(defmacro save-note-read (tn) - `(let* ((tn ,tn) - (num (tn-number tn))) - (when (and (sc-save-p (tn-sc tn)) - (zerop (sbit restores num)) - (not (eq (tn-kind tn) :component))) - (setf (sbit restores num) 1) - (push tn restores-list)))) - -) ; EVAL-WHEN - ;;; Start scanning backward at the end of BLOCK, looking which TNs are ;;; live and looking for places where we have to save. We manipulate ;;; two sets: SAVES and RESTORES. @@ -659,90 +653,88 @@ (defun optimized-emit-saves-block (block saves restores) (declare (type ir2-block block) (type simple-bit-vector saves restores)) (let ((1block (ir2-block-block block)) - (saves-list ()) - (restores-list ()) - (skipping nil)) + (saves-list ()) + (restores-list ()) + (skipping nil)) (declare (list saves-list restores-list)) (clear-bit-vector saves) (clear-bit-vector restores) (do-live-tns (tn (ir2-block-live-in block) block) (when (and (sc-save-p (tn-sc tn)) - (not (eq (tn-kind tn) :component))) - (let ((num (tn-number tn))) - (setf (sbit restores num) 1) - (push tn restores-list)))) + (not (eq (tn-kind tn) :component))) + (let ((num (tn-number tn))) + (setf (sbit restores num) 1) + (push tn restores-list)))) (do ((block block (ir2-block-prev block)) - (prev nil block)) - ((not (eq (ir2-block-block block) 1block)) - (aver (not skipping)) - (dolist (save saves-list) - (let ((start (ir2-block-start-vop prev))) - (save-if-necessary save start start))) - prev) + (prev nil block)) + ((not (eq (ir2-block-block block) 1block)) + (aver (not skipping)) + (dolist (save saves-list) + (let ((start (ir2-block-start-vop prev))) + (save-if-necessary save start start))) + prev) (do ((vop (ir2-block-last-vop block) (vop-prev vop))) - ((null vop)) - (let ((info (vop-info vop))) - (case (vop-info-name info) - (allocate-frame - (aver skipping) - (setq skipping nil)) - (note-environment-start - (aver (not skipping)) - (dolist (save saves-list) - (save-if-necessary save (vop-next vop) vop)) - (return-from optimized-emit-saves-block block))) - - (unless skipping - (do ((write (vop-results vop) (tn-ref-across write))) - ((null write)) - (let* ((tn (tn-ref-tn write)) - (num (tn-number tn))) - (unless (zerop (sbit restores num)) - (setf (sbit restores num) 0) - (setq restores-list - (delete tn restores-list :test #'eq))) - (unless (zerop (sbit saves num)) - (setf (sbit saves num) 0) - (save-if-necessary tn (vop-next vop) vop) - (setq saves-list - (delete tn saves-list :test #'eq)))))) - - (macrolet (;; Do stuff to note a read of TN, for - ;; OPTIMIZED-EMIT-SAVES-BLOCK. - (save-note-read (tn) - `(let* ((tn ,tn) - (num (tn-number tn))) - (when (and (sc-save-p (tn-sc tn)) - (zerop (sbit restores num)) - (not (eq (tn-kind tn) :component))) - (setf (sbit restores num) 1) - (push tn restores-list))))) - - (case (vop-info-save-p info) - ((t) - (dolist (tn restores-list) - (restore-tn tn (vop-next vop) vop) - (let ((num (tn-number tn))) - (when (zerop (sbit saves num)) - (push tn saves-list) - (setf (sbit saves num) 1)))) - (setq restores-list nil) - (clear-bit-vector restores)) - (:compute-only - (cond ((policy (vop-node vop) (= speed 3)) - (do-live-tns (tn (vop-save-set vop) block) - (when (zerop (sbit restores (tn-number tn))) - (note-spilled-tn tn vop)))) - (t - (do-live-tns (tn (vop-save-set vop) block) - (save-note-read tn)))))) - - (if (eq (vop-info-move-args info) :local-call) - (setq skipping t) - (do ((read (vop-args vop) (tn-ref-across read))) - ((null read)) - (save-note-read (tn-ref-tn read)))))))))) + ((null vop)) + (let ((info (vop-info vop))) + (case (vop-info-name info) + (allocate-frame + (aver skipping) + (setq skipping nil)) + (note-environment-start + (aver (not skipping)) + (dolist (save saves-list) + (save-if-necessary save (vop-next vop) vop)) + (return-from optimized-emit-saves-block block))) + + (unless skipping + (do ((write (vop-results vop) (tn-ref-across write))) + ((null write)) + (let* ((tn (tn-ref-tn write)) + (num (tn-number tn))) + (unless (zerop (sbit restores num)) + (setf (sbit restores num) 0) + (setq restores-list + (delete tn restores-list :test #'eq))) + (unless (zerop (sbit saves num)) + (setf (sbit saves num) 0) + (save-if-necessary tn (vop-next vop) vop) + (setq saves-list + (delete tn saves-list :test #'eq)))))) + + (macrolet ((save-note-read (tn) + `(let* ((tn ,tn) + (num (tn-number tn))) + (when (and (sc-save-p (tn-sc tn)) + (zerop (sbit restores num)) + (not (eq (tn-kind tn) :component))) + (setf (sbit restores num) 1) + (push tn restores-list))))) + + (case (vop-info-save-p info) + ((t) + (dolist (tn restores-list) + (restore-tn tn (vop-next vop) vop) + (let ((num (tn-number tn))) + (when (zerop (sbit saves num)) + (push tn saves-list) + (setf (sbit saves num) 1)))) + (setq restores-list nil) + (clear-bit-vector restores)) + (:compute-only + (cond ((policy (vop-node vop) (= speed 3)) + (do-live-tns (tn (vop-save-set vop) block) + (when (zerop (sbit restores (tn-number tn))) + (note-spilled-tn tn vop)))) + (t + (do-live-tns (tn (vop-save-set vop) block) + (save-note-read tn)))))) + + (if (eq (vop-info-move-args info) :local-call) + (setq skipping t) + (do ((read (vop-args vop) (tn-ref-across read))) + ((null read)) + (save-note-read (tn-ref-tn read)))))))))) ;;; This is like EMIT-SAVES, only different. We avoid redundant saving ;;; within the block, and don't restore values that aren't used before @@ -752,44 +744,98 @@ (defun optimized-emit-saves (component) (declare (type component component)) (let* ((gtn-count (1+ (ir2-component-global-tn-counter - (component-info component)))) - (saves (make-array gtn-count :element-type 'bit)) - (restores (make-array gtn-count :element-type 'bit)) - (block (ir2-block-prev (block-info (component-tail component)))) - (head (block-info (component-head component)))) + (component-info component)))) + (saves (make-array gtn-count :element-type 'bit)) + (restores (make-array gtn-count :element-type 'bit)) + (block (ir2-block-prev (block-info (component-tail component)))) + (head (block-info (component-head component)))) (loop (when (eq block head) (return)) (when (do ((vop (ir2-block-start-vop block) (vop-next vop))) - ((null vop) nil) - (when (eq (vop-info-save-p (vop-info vop)) t) - (return t))) - (setq block (optimized-emit-saves-block block saves restores))) + ((null vop) nil) + (when (eq (vop-info-save-p (vop-info vop)) t) + (return t))) + (setq block (optimized-emit-saves-block block saves restores))) (setq block (ir2-block-prev block))))) - + ;;; Iterate over the normal TNs, finding the cost of packing on the -;;; stack in units of the number of references. We count all -;;; references as +1, and subtract out REGISTER-SAVE-PENALTY for each -;;; place where we would have to save a register. +;;; stack in units of the number of references. We count all read +;;; references as +1, write references as + *tn-write-cost*, and +;;; subtract out REGISTER-SAVE-PENALTY for each place where we would +;;; have to save a register. +;;; The subtraction reflects the fact that having a value in a +;;; register around a call means that code to spill and unspill must +;;; be inserted. +;;; +;;; The costs also take into account the loop depth at which each +;;; reference occurs: the penalty or cost is incremented by the depth +;;; scaled by *tn-loop-depth-multiplier*. The default (NIL) is to let +;;; this be one more than the max of the cost for reads (1), for write +;;; references and for being live across a call. +(defvar *tn-write-cost* 2) +(defvar *tn-loop-depth-multiplier* nil) + (defun assign-tn-costs (component) - (do-ir2-blocks (block component) - (do ((vop (ir2-block-start-vop block) (vop-next vop))) - ((null vop)) - (when (eq (vop-info-save-p (vop-info vop)) t) - (do-live-tns (tn (vop-save-set vop) block) - (decf (tn-cost tn) *backend-register-save-penalty*))))) - - (do ((tn (ir2-component-normal-tns (component-info component)) - (tn-next tn))) - ((null tn)) - (let ((cost (tn-cost tn))) - (declare (fixnum cost)) - (do ((ref (tn-reads tn) (tn-ref-next ref))) - ((null ref)) - (incf cost)) - (do ((ref (tn-writes tn) (tn-ref-next ref))) - ((null ref)) - (incf cost)) - (setf (tn-cost tn) cost)))) + (let* ((save-penalty *backend-register-save-penalty*) + (write-cost *tn-write-cost*) + (depth-scale (or *tn-loop-depth-multiplier* + (1+ (max 1 write-cost save-penalty))))) + (flet ((vop-depth-cost (vop) + (let ((loop (block-loop + (ir2-block-block + (vop-block vop))))) + (if loop + (* depth-scale (loop-depth loop)) + 0)))) + (do-ir2-blocks (block component) + (do ((vop (ir2-block-start-vop block) (vop-next vop))) + ((null vop)) + (when (eq (vop-info-save-p (vop-info vop)) t) + (let ((penalty (+ save-penalty (vop-depth-cost vop)))) + (do-live-tns (tn (vop-save-set vop) block) + (decf (tn-cost tn) penalty)))))) + + (do ((tn (ir2-component-normal-tns (component-info component)) + (tn-next tn))) + ((null tn)) + (let ((cost (tn-cost tn))) + (declare (fixnum cost)) + (do ((ref (tn-reads tn) (tn-ref-next ref))) + ((null ref)) + (incf cost (1+ (vop-depth-cost (tn-ref-vop ref))))) + (do ((ref (tn-writes tn) (tn-ref-next ref))) + ((null ref)) + (incf cost (+ write-cost (vop-depth-cost (tn-ref-vop ref))))) + (setf (tn-cost tn) cost)))))) + +;;; Iterate over the normal TNs, folding over the depth of the looops +;;; that the TN is used in and storing the result in TN-LOOP-DEPTH. +;;: reducer is the function used to join depth values together. #'max +;;; gives the maximum depth, #'+ the sum. +(defun assign-tn-depths (component &key (reducer #'max)) + (declare (type function reducer)) + (when *loop-analyze* + ;; We only use tn depth for normal TNs + (do ((tn (ir2-component-normal-tns (component-info component)) + (tn-next tn))) + ((null tn)) + (let ((depth 0)) + (declare (type fixnum depth)) + (flet ((frob (ref) + (declare (type (or null tn-ref) ref)) + (do ((ref ref (tn-ref-next ref))) + ((null ref)) + (let* ((vop (tn-ref-vop ref)) + (block (ir2-block-block (vop-block vop))) + (loop (block-loop block))) + (setf depth (funcall reducer + depth + (if loop + (loop-depth loop) + 0))))))) + (frob (tn-reads tn)) + (frob (tn-writes tn)) + (setf (tn-loop-depth tn) depth)))))) ;;;; load TN packing @@ -803,9 +849,9 @@ ;;; sticking them in this hash-table. This is initially null. We ;;; create the hashtable if we do any unpacking. (defvar *repack-blocks*) -(declaim (type (or hash-table null) *repack-blocks*)) +(declaim (type list *repack-blocks*)) -;;; Set the Live-TNs vectors in all :FINITE SBs to represent the TNs +;;; Set the LIVE-TNS vectors in all :FINITE SBs to represent the TNs ;;; live at the end of BLOCK. (defun init-live-tns (block) (dolist (sb *backend-sb-list*) @@ -814,13 +860,23 @@ (do-live-tns (tn (ir2-block-live-in block) block) (let* ((sc (tn-sc tn)) - (sb (sc-sb sc))) + (sb (sc-sb sc))) (when (eq (sb-kind sb) :finite) - (do ((offset (tn-offset tn) (1+ offset)) - (end (+ (tn-offset tn) (sc-element-size sc)))) - ((= offset end)) - (declare (type index offset end)) - (setf (svref (finite-sb-live-tns sb) offset) tn))))) + ;; KLUDGE: we can have "live" TNs that are neither read + ;; to nor written from, due to more aggressive (type- + ;; directed) constant propagation. Such TNs will never + ;; be assigned an offset nor be in conflict with anything. + ;; + ;; Ideally, it seems to me we could make sure these TNs + ;; are never allocated in the first place in + ;; ASSIGN-LAMBDA-VAR-TNS. + (if (tn-offset tn) + (do ((offset (tn-offset tn) (1+ offset)) + (end (+ (tn-offset tn) (sc-element-size sc)))) + ((= offset end)) + (declare (type index offset end)) + (setf (svref (finite-sb-live-tns sb) offset) tn)) + (aver (and (null (tn-reads tn)) (null (tn-writes tn)))))))) (setq *live-block* block) (setq *live-vop* (ir2-block-last-vop block)) @@ -840,44 +896,44 @@ (do ((current *live-vop* (vop-prev current))) ((eq current vop) (do ((res (vop-results vop) (tn-ref-across res))) - ((null res)) - (let* ((tn (tn-ref-tn res)) - (sc (tn-sc tn)) - (sb (sc-sb sc))) - (when (eq (sb-kind sb) :finite) - (do ((offset (tn-offset tn) (1+ offset)) - (end (+ (tn-offset tn) (sc-element-size sc)))) - ((= offset end)) - (declare (type index offset end)) - (setf (svref (finite-sb-live-tns sb) offset) nil)))))) + ((null res)) + (let* ((tn (tn-ref-tn res)) + (sc (tn-sc tn)) + (sb (sc-sb sc))) + (when (eq (sb-kind sb) :finite) + (do ((offset (tn-offset tn) (1+ offset)) + (end (+ (tn-offset tn) (sc-element-size sc)))) + ((= offset end)) + (declare (type index offset end)) + (setf (svref (finite-sb-live-tns sb) offset) nil)))))) (do ((ref (vop-refs current) (tn-ref-next-ref ref))) - ((null ref)) + ((null ref)) (let ((ltn (tn-ref-load-tn ref))) - (when ltn - (let* ((sc (tn-sc ltn)) - (sb (sc-sb sc))) - (when (eq (sb-kind sb) :finite) - (let ((tns (finite-sb-live-tns sb))) - (do ((offset (tn-offset ltn) (1+ offset)) - (end (+ (tn-offset ltn) (sc-element-size sc)))) - ((= offset end)) - (declare (type index offset end)) - (aver (null (svref tns offset))))))))) + (when ltn + (let* ((sc (tn-sc ltn)) + (sb (sc-sb sc))) + (when (eq (sb-kind sb) :finite) + (let ((tns (finite-sb-live-tns sb))) + (do ((offset (tn-offset ltn) (1+ offset)) + (end (+ (tn-offset ltn) (sc-element-size sc)))) + ((= offset end)) + (declare (type index offset end)) + (aver (null (svref tns offset))))))))) (let* ((tn (tn-ref-tn ref)) - (sc (tn-sc tn)) - (sb (sc-sb sc))) - (when (eq (sb-kind sb) :finite) - (let ((tns (finite-sb-live-tns sb))) - (do ((offset (tn-offset tn) (1+ offset)) - (end (+ (tn-offset tn) (sc-element-size sc)))) - ((= offset end)) - (declare (type index offset end)) - (if (tn-ref-write-p ref) - (setf (svref tns offset) nil) - (let ((old (svref tns offset))) - (aver (or (null old) (eq old tn))) - (setf (svref tns offset) tn))))))))) + (sc (tn-sc tn)) + (sb (sc-sb sc))) + (when (eq (sb-kind sb) :finite) + (let ((tns (finite-sb-live-tns sb))) + (do ((offset (tn-offset tn) (1+ offset)) + (end (+ (tn-offset tn) (sc-element-size sc)))) + ((= offset end)) + (declare (type index offset end)) + (if (tn-ref-write-p ref) + (setf (svref tns offset) nil) + (let ((old (svref tns offset))) + (aver (or (null old) (eq old tn))) + (setf (svref tns offset) tn))))))))) (setq *live-vop* vop) (values)) @@ -898,8 +954,8 @@ ;;; aren't any TN-REFs to represent the implicit reading of results or ;;; writing of arguments. ;;; -;;; The second bullet corresponds conflicts with temporaries or between -;;; arguments and results. +;;; The second bullet corresponds to conflicts with temporaries or +;;; between arguments and results. ;;; ;;; We consider both the TN-REF-TN and the TN-REF-LOAD-TN (if any) to ;;; be referenced simultaneously and in the same way. This causes @@ -912,36 +968,36 @@ (aver (eq (sb-kind sb) :finite)) (let ((vop (tn-ref-vop op))) (labels ((tn-overlaps (tn) - (let ((sc (tn-sc tn)) - (tn-offset (tn-offset tn))) - (when (and (eq (sc-sb sc) sb) - (<= tn-offset offset) - (< offset - (the index - (+ tn-offset (sc-element-size sc))))) - tn))) - (same (ref) - (let ((tn (tn-ref-tn ref)) - (ltn (tn-ref-load-tn ref))) - (or (tn-overlaps tn) - (and ltn (tn-overlaps ltn))))) - (is-op (ops) - (do ((ops ops (tn-ref-across ops))) - ((null ops) nil) - (let ((found (same ops))) - (when (and found (not (eq ops op))) - (return found))))) - (is-ref (refs end) - (do ((refs refs (tn-ref-next-ref refs))) - ((eq refs end) nil) - (let ((found (same refs))) - (when found (return found)))))) + (let ((sc (tn-sc tn)) + (tn-offset (tn-offset tn))) + (when (and (eq (sc-sb sc) sb) + (<= tn-offset offset) + (< offset + (the index + (+ tn-offset (sc-element-size sc))))) + tn))) + (same (ref) + (let ((tn (tn-ref-tn ref)) + (ltn (tn-ref-load-tn ref))) + (or (tn-overlaps tn) + (and ltn (tn-overlaps ltn))))) + (is-op (ops) + (do ((ops ops (tn-ref-across ops))) + ((null ops) nil) + (let ((found (same ops))) + (when (and found (not (eq ops op))) + (return found))))) + (is-ref (refs end) + (do ((refs refs (tn-ref-next-ref refs))) + ((eq refs end) nil) + (let ((found (same refs))) + (when found (return found)))))) (declare (inline is-op is-ref tn-overlaps)) (if (tn-ref-write-p op) - (or (is-op (vop-results vop)) - (is-ref (vop-refs vop) op)) - (or (is-op (vop-args vop)) - (is-ref (tn-ref-next-ref op) nil)))))) + (or (is-op (vop-results vop)) + (is-ref (vop-refs vop) op)) + (or (is-op (vop-args vop)) + (is-ref (tn-ref-next-ref op) nil)))))) ;;; Iterate over all the elements in the SB that would be allocated by ;;; allocating a TN in SC at Offset, checking for conflict with @@ -953,16 +1009,16 @@ ;;; We return a conflicting TN, or :OVERFLOW if the TN won't fit. (defun load-tn-conflicts-in-sc (op sc offset ignore-live) (let* ((sb (sc-sb sc)) - (size (finite-sb-current-size sb))) + (size (finite-sb-current-size sb))) (do ((i offset (1+ i)) - (end (+ offset (sc-element-size sc)))) - ((= i end) nil) + (end (+ offset (sc-element-size sc)))) + ((= i end) nil) (declare (type index i end)) (let ((res (or (when (>= i size) :overflow) - (and (not ignore-live) - (svref (finite-sb-live-tns sb) i)) - (load-tn-offset-conflicts-in-sb op sb i)))) - (when res (return res)))))) + (and (not ignore-live) + (svref (finite-sb-live-tns sb) i)) + (load-tn-offset-conflicts-in-sb op sb i)))) + (when res (return res)))))) ;;; If a load-TN for OP is targeted to a legal location in SC, then ;;; return the offset, otherwise return NIL. We see whether the target @@ -981,12 +1037,12 @@ (let ((target (tn-ref-target op))) (when target (let* ((tn (tn-ref-tn target)) - (loc (tn-offset tn))) - (if (and (eq (tn-sc tn) sc) - (member (the index loc) (sc-locations sc)) - (not (load-tn-conflicts-in-sc op sc loc nil))) - loc - nil))))) + (loc (tn-offset tn))) + (if (and (eq (tn-sc tn) sc) + (member (the index loc) (sc-locations sc)) + (not (load-tn-conflicts-in-sc op sc loc nil))) + loc + nil))))) ;;; Select a legal location for a load TN for Op in SC. We just ;;; iterate over the SC's locations. If we can't find a legal @@ -998,11 +1054,11 @@ (let ((target (tn-ref-target op))) (when target (let* ((tn (tn-ref-tn target)) - (loc (tn-offset tn))) - (when (and (eq (sc-sb sc) (sc-sb (tn-sc tn))) - (member (the index loc) (sc-locations sc)) - (not (load-tn-conflicts-in-sc op sc loc nil))) - (return-from select-load-tn-location loc))))) + (loc (tn-offset tn))) + (when (and (eq (sc-sb sc) (sc-sb (tn-sc tn))) + (member (the index loc) (sc-locations sc)) + (not (load-tn-conflicts-in-sc op sc loc nil))) + (return-from select-load-tn-location loc))))) (dolist (loc (sc-locations sc) nil) (unless (load-tn-conflicts-in-sc op sc loc nil) @@ -1017,16 +1073,16 @@ (defun unpack-tn (tn) (event unpack-tn) (let ((stn (or (tn-save-tn tn) - (pack-save-tn tn)))) + (pack-save-tn tn)))) (setf (tn-sc tn) (tn-sc stn)) (setf (tn-offset tn) (tn-offset stn)) (flet ((zot (refs) - (do ((ref refs (tn-ref-next ref))) - ((null ref)) - (let ((vop (tn-ref-vop ref))) - (if (eq (vop-info-name (vop-info vop)) 'move-operand) - (delete-vop vop) - (setf (gethash (vop-block vop) *repack-blocks*) t)))))) + (do ((ref refs (tn-ref-next ref))) + ((null ref)) + (let ((vop (tn-ref-vop ref))) + (if (eq (vop-info-name (vop-info vop)) 'move-operand) + (delete-vop vop) + (pushnew (vop-block vop) *repack-blocks*)))))) (zot (tn-reads tn)) (zot (tn-writes tn)))) @@ -1036,9 +1092,9 @@ ;;; This is called by PACK-LOAD-TN where there isn't any location free ;;; that we can pack into. What we do is move some live TN in one of -;;; the specified SCs to memory, then mark this block all blocks that -;;; reference the TN as needing repacking. If we succeed, we throw to -;;; UNPACKED-TN. If we fail, we return NIL. +;;; the specified SCs to memory, then mark all blocks that reference +;;; the TN as needing repacking. If we succeed, we throw to UNPACKED-TN. +;;; If we fail, we return NIL. ;;; ;;; We can unpack any live TN that appears in the NORMAL-TNs list ;;; (isn't wired or restricted.) We prefer to unpack TNs that are not @@ -1048,45 +1104,43 @@ (defun unpack-for-load-tn (sc op) (declare (type sc sc) (type tn-ref op)) (let ((sb (sc-sb sc)) - (normal-tns (ir2-component-normal-tns - (component-info *component-being-compiled*))) - (node (vop-node (tn-ref-vop op))) - (fallback nil)) + (normal-tns (ir2-component-normal-tns + (component-info *component-being-compiled*))) + (node (vop-node (tn-ref-vop op))) + (fallback nil)) (flet ((unpack-em (victims) - (unless *repack-blocks* - (setq *repack-blocks* (make-hash-table :test 'eq))) - (setf (gethash (vop-block (tn-ref-vop op)) *repack-blocks*) t) - (dolist (victim victims) - (event unpack-tn node) - (unpack-tn victim)) - (throw 'unpacked-tn nil))) + (pushnew (vop-block (tn-ref-vop op)) *repack-blocks*) + (dolist (victim victims) + (event unpack-tn node) + (unpack-tn victim)) + (throw 'unpacked-tn nil))) (dolist (loc (sc-locations sc)) - (declare (type index loc)) - (block SKIP - (collect ((victims nil adjoin)) - (do ((i loc (1+ i)) - (end (+ loc (sc-element-size sc)))) - ((= i end)) - (declare (type index i end)) - (let ((victim (svref (finite-sb-live-tns sb) i))) - (when victim - (unless (find-in #'tn-next victim normal-tns) - (return-from SKIP)) - (victims victim)))) - - (let ((conf (load-tn-conflicts-in-sc op sc loc t))) - (cond ((not conf) - (unpack-em (victims))) - ((eq conf :overflow)) - ((not fallback) - (cond ((find conf (victims)) - (setq fallback (victims))) - ((find-in #'tn-next conf normal-tns) - (setq fallback (list conf)))))))))) + (declare (type index loc)) + (block SKIP + (collect ((victims nil adjoin)) + (do ((i loc (1+ i)) + (end (+ loc (sc-element-size sc)))) + ((= i end)) + (declare (type index i end)) + (let ((victim (svref (finite-sb-live-tns sb) i))) + (when victim + (unless (find-in #'tn-next victim normal-tns) + (return-from SKIP)) + (victims victim)))) + + (let ((conf (load-tn-conflicts-in-sc op sc loc t))) + (cond ((not conf) + (unpack-em (victims))) + ((eq conf :overflow)) + ((not fallback) + (cond ((find conf (victims)) + (setq fallback (victims))) + ((find-in #'tn-next conf normal-tns) + (setq fallback (list conf)))))))))) (when fallback - (event unpack-fallback node) - (unpack-em fallback)))) + (event unpack-fallback node) + (unpack-em fallback)))) nil) @@ -1098,7 +1152,7 @@ ;;; if that location is in a SC not allowed by the primitive type. ;;; (The SC must still be allowed by the operand restriction.) This ;;; makes move VOPs more efficient, since we won't do a move from the -;;; stack into a non-descriptor any-reg though a descriptor argument +;;; stack into a non-descriptor any-reg through a descriptor argument ;;; load-TN. This does give targeting some real semantics, making it ;;; not a pure advisory to pack. It allows pack to do some packing it ;;; wouldn't have done before. @@ -1108,29 +1162,29 @@ (compute-live-tns (vop-block vop) vop)) (let* ((tn (tn-ref-tn op)) - (ptype (tn-primitive-type tn)) - (scs (svref load-scs (sc-number (tn-sc tn))))) + (ptype (tn-primitive-type tn)) + (scs (svref load-scs (sc-number (tn-sc tn))))) (let ((current-scs scs) - (allowed ())) + (allowed ())) (loop - (cond - ((null current-scs) - (unless allowed - (no-load-scs-allowed-by-primitive-type-error op)) - (dolist (sc allowed) - (unpack-for-load-tn sc op)) - (failed-to-pack-load-tn-error allowed op)) - (t - (let* ((sc (svref *backend-sc-numbers* (pop current-scs))) - (target (find-load-tn-target op sc))) - (when (or target (sc-allowed-by-primitive-type sc ptype)) - (let ((loc (or target - (select-load-tn-location op sc)))) - (when loc - (let ((res (make-tn 0 :load nil sc))) - (setf (tn-offset res) loc) - (return res)))) - (push sc allowed))))))))) + (cond + ((null current-scs) + (unless allowed + (no-load-scs-allowed-by-primitive-type-error op)) + (dolist (sc allowed) + (unpack-for-load-tn sc op)) + (failed-to-pack-load-tn-error allowed op)) + (t + (let* ((sc (svref *backend-sc-numbers* (pop current-scs))) + (target (find-load-tn-target op sc))) + (when (or target (sc-allowed-by-primitive-type sc ptype)) + (let ((loc (or target + (select-load-tn-location op sc)))) + (when loc + (let ((res (make-tn 0 :load nil sc))) + (setf (tn-offset res) loc) + (return res)))) + (push sc allowed))))))))) ;;; Scan a list of load-SCs vectors and a list of TN-REFS threaded by ;;; TN-REF-ACROSS. When we find a reference whose TN doesn't satisfy @@ -1146,31 +1200,31 @@ (op ops (tn-ref-across op))) ((null scs)) (let ((target (tn-ref-target op))) - (when target - (let* ((load-tn (tn-ref-load-tn op)) - (load-scs (svref (car scs) - (sc-number - (tn-sc (or load-tn (tn-ref-tn op))))))) - (if load-tn - (aver (eq load-scs t)) - (unless (eq load-scs t) - (setf (tn-ref-load-tn op) - (pack-load-tn (car scs) op)))))))) + (when target + (let* ((load-tn (tn-ref-load-tn op)) + (load-scs (svref (car scs) + (sc-number + (tn-sc (or load-tn (tn-ref-tn op))))))) + (if load-tn + (aver (eq load-scs t)) + (unless (eq load-scs t) + (setf (tn-ref-load-tn op) + (pack-load-tn (car scs) op)))))))) (do ((scs scs (cdr scs)) (op ops (tn-ref-across op))) ((null scs)) (let ((target (tn-ref-target op))) - (unless target - (let* ((load-tn (tn-ref-load-tn op)) - (load-scs (svref (car scs) - (sc-number - (tn-sc (or load-tn (tn-ref-tn op))))))) - (if load-tn - (aver (eq load-scs t)) - (unless (eq load-scs t) - (setf (tn-ref-load-tn op) - (pack-load-tn (car scs) op)))))))) + (unless target + (let* ((load-tn (tn-ref-load-tn op)) + (load-scs (svref (car scs) + (sc-number + (tn-sc (or load-tn (tn-ref-tn op))))))) + (if load-tn + (aver (eq load-scs t)) + (unless (eq load-scs t) + (setf (tn-ref-load-tn op) + (pack-load-tn (car scs) op)))))))) (values)) @@ -1180,14 +1234,14 @@ (defun pack-load-tns (block) (catch 'unpacked-tn (let ((*live-block* nil) - (*live-vop* nil)) + (*live-vop* nil)) (do ((vop (ir2-block-last-vop block) (vop-prev vop))) - ((null vop)) - (let ((info (vop-info vop))) - (check-operand-restrictions (vop-info-result-load-scs info) - (vop-results vop)) - (check-operand-restrictions (vop-info-arg-load-scs info) - (vop-args vop)))))) + ((null vop)) + (let ((info (vop-info vop))) + (check-operand-restrictions (vop-info-result-load-scs info) + (vop-results vop)) + (check-operand-restrictions (vop-info-arg-load-scs info) + (vop-args vop)))))) (values)) ;;;; targeting @@ -1211,120 +1265,123 @@ (defun check-ok-target (target tn sc) (declare (type tn target tn) (type sc sc) (inline member)) (let* ((loc (tn-offset target)) - (target-sc (tn-sc target)) - (target-sb (sc-sb target-sc))) + (target-sc (tn-sc target)) + (target-sb (sc-sb target-sc))) (declare (type index loc)) ;; We can honor a preference if: ;; -- TARGET's location is in SC's locations. ;; -- The element sizes of the two SCs are the same. ;; -- TN doesn't conflict with target's location. (if (and (eq target-sb (sc-sb sc)) - (or (eq (sb-kind target-sb) :unbounded) - (member loc (sc-locations sc))) - (= (sc-element-size target-sc) (sc-element-size sc)) - (not (conflicts-in-sc tn sc loc)) - (zerop (mod loc (sc-alignment sc)))) - loc - nil))) + (or (eq (sb-kind target-sb) :unbounded) + (member loc (sc-locations sc))) + (= (sc-element-size target-sc) (sc-element-size sc)) + (not (conflicts-in-sc tn sc loc)) + (zerop (mod loc (sc-alignment sc)))) + loc + nil))) ;;; Scan along the target path from TN, looking at readers or writers. -;;; When we find a packed TN, return CHECK-OK-TARGET of that TN. If -;;; there is no target, or if the TN has multiple readers (writers), -;;; then we return NIL. We also always return NIL after 10 iterations -;;; to get around potential circularity problems. +;;; When we find a TN, call CALLEE with that TN, and then resume +;;; walking down that TN's target. As soon as there is no target, or +;;; if the TN has multiple readers (writers), we stop walking the +;;; targetting chain. We also always stop after 10 iterations to get +;;; around potential circularity problems. ;;; -;;; FIXME: (30 minutes of reverse engineering?) It'd be nice to -;;; rewrite the header comment here to explain the interface and its -;;; motivation, and move remarks about implementation details (like -;;; 10!) inside. +;;; Why the single-reader/writer constraint? As far as I can tell, +;;; this is concerned with straight pipeline of data, e.g. CASTs. In +;;; that case, limiting to chains of length 10 seems to be more than +;;; enough. +(declaim (inline %call-with-target-tns)) +(defun %call-with-target-tns (tn callee + &key (limit 10) (reads t) (writes t)) + (declare (type tn tn) (type function callee) (type index limit)) + (flet ((frob-slot (slot-function) + (declare (type function slot-function)) + (let ((count limit) + (current tn)) + (declare (type index count)) + (loop + (let ((refs (funcall slot-function current))) + (unless (and (plusp count) + refs + (not (tn-ref-next refs))) + (return nil)) + (let ((target (tn-ref-target refs))) + (unless target (return nil)) + (setq current (tn-ref-tn target)) + (funcall callee current) + (decf count))))))) + (when reads + (frob-slot #'tn-reads)) + (when writes + (frob-slot #'tn-writes)) + nil)) + +(defmacro do-target-tns ((target-variable source-tn + &rest keys &key limit reads writes) + &body body) + (declare (ignore limit reads writes)) + (let ((callback (gensym "CALLBACK"))) + `(flet ((,callback (,target-variable) + ,@body)) + (declare (dynamic-extent #',callback)) + (%call-with-target-tns ,source-tn #',callback ,@keys)))) + (defun find-ok-target-offset (tn sc) (declare (type tn tn) (type sc sc)) - (flet ((frob-slot (slot-fun) - (declare (type function slot-fun)) - (let ((count 10) - (current tn)) - (declare (type index count)) - (loop - (let ((refs (funcall slot-fun current))) - (unless (and (plusp count) - refs - (not (tn-ref-next refs))) - (return nil)) - (let ((target (tn-ref-target refs))) - (unless target (return nil)) - (setq current (tn-ref-tn target)) - (when (tn-offset current) - (return (check-ok-target current tn sc))) - (decf count))))))) - (declare (inline frob-slot)) ; until DYNAMIC-EXTENT works - (or (frob-slot #'tn-reads) - (frob-slot #'tn-writes)))) + (do-target-tns (target tn) + (awhen (and (tn-offset target) + (check-ok-target target tn sc)) + (return-from find-ok-target-offset it)))) ;;;; location selection ;;; Select some location for TN in SC, returning the offset if we -;;; succeed, and NIL if we fail. We start scanning at the Last-Offset -;;; in an attempt to distribute the TNs across all storage. -;;; -;;; We call OFFSET-CONFLICTS-IN-SB directly, rather than using -;;; CONFLICTS-IN-SC. This allows us to more efficient in packing -;;; multi-location TNs: we don't have to multiply the number of tests -;;; by the TN size. This falls out naturally, since we have to be -;;; aware of TN size anyway so that we don't call CONFLICTS-IN-SC on a -;;; bogus offset. -;;; -;;; We give up on finding a location after our current pointer has -;;; wrapped twice. This will result in testing some locations twice in -;;; the case that we fail, but is simpler than trying to figure out -;;; the soonest failure point. +;;; succeed, and NIL if we fail. ;;; -;;; We also give up without bothering to wrap if the current size -;;; isn't large enough to hold a single element of element-size -;;; without bothering to wrap. If it doesn't fit this iteration, it -;;; won't fit next. +;;; For :UNBOUNDED SCs just find the smallest correctly aligned offset +;;; where the TN doesn't conflict with the TNs that have already been +;;; packed. For :FINITE SCs try to pack the TN into the most heavily +;;; used locations first (as estimated in FIND-LOCATION-USAGE). ;;; -;;; ### Note that we actually try to pack as many consecutive TNs as -;;; possible in the same location, since we start scanning at the same -;;; offset that the last TN was successfully packed in. This is a -;;; weakening of the scattering hueristic that was put in to prevent -;;; restricted VOP temps from hogging all of the registers. This way, -;;; all of these temps probably end up in one register. -(defun select-location (tn sc &optional use-reserved-locs) +;;; Historically SELECT-LOCATION tried did the opposite and tried to +;;; distribute the TNs evenly across the available locations. At least +;;; on register-starved architectures (x86) this seems to be a bad +;;; strategy. -- JES 2004-09-11 +(defun select-location (tn sc &key use-reserved-locs optimize) (declare (type tn tn) (type sc sc) (inline member)) (let* ((sb (sc-sb sc)) - (element-size (sc-element-size sc)) - (alignment (sc-alignment sc)) - (align-mask (1- alignment)) - (size (finite-sb-current-size sb)) - (start-offset (finite-sb-last-offset sb))) - (let ((current-start - (logandc2 (the index (+ start-offset align-mask)) align-mask)) - (wrap-p nil)) - (declare (type index current-start)) - (loop - (when (> (+ current-start element-size) size) - (cond ((or wrap-p (> element-size size)) - (return nil)) - (t - (setq current-start 0) - (setq wrap-p t)))) - - (if (or (eq (sb-kind sb) :unbounded) - (and (member current-start (sc-locations sc)) - (or use-reserved-locs - (not (member current-start - (sc-reserve-locations sc)))))) - (dotimes (i element-size - (return-from select-location current-start)) - (declare (type index i)) - (let ((offset (+ current-start i))) - (when (offset-conflicts-in-sb tn sb offset) - (setq current-start - (logandc2 (the index (+ (the index (1+ offset)) - align-mask)) - align-mask)) - (return)))) - (incf current-start alignment)))))) + (element-size (sc-element-size sc)) + (alignment (sc-alignment sc)) + (align-mask (1- alignment)) + (size (finite-sb-current-size sb))) + (flet ((attempt-location (start-offset) + (let ((conflict (conflicts-in-sc tn sc start-offset))) + (if conflict + (logandc2 (+ conflict align-mask 1) + align-mask) + (return-from select-location start-offset))))) + (if (eq (sb-kind sb) :unbounded) + (loop with offset = 0 + until (> (+ offset element-size) size) do + (setf offset (attempt-location offset))) + (let ((locations (sc-locations sc))) + (when optimize + (setf locations + (schwartzian-stable-sort-list + locations '> + :key (lambda (location-offset) + (loop for offset from location-offset + repeat element-size + maximize (svref + (finite-sb-always-live-count sb) + offset)))))) + (dolist (offset locations) + (when (or use-reserved-locs + (not (member offset + (sc-reserve-locations sc)))) + (attempt-location offset)))))))) ;;; If a save TN, return the saved TN, otherwise return TN. This is ;;; useful for getting the conflicts of a TN that might be a save TN. @@ -1336,6 +1393,15 @@ ;;;; pack interface +;; Misc. utilities +(declaim (inline unbounded-sc-p)) +(defun unbounded-sc-p (sc) + (eq (sb-kind (sc-sb sc)) :unbounded)) + +(defun unbounded-tn-p (tn) + (unbounded-sc-p (tn-sc tn))) +(declaim (notinline unbounded-sc-p)) + ;;; Attempt to pack TN in all possible SCs, first in the SC chosen by ;;; representation selection, then in the alternate SCs in the order ;;; they were specified in the SC definition. If the TN-COST is @@ -1346,187 +1412,342 @@ ;;; If we are attempting to pack in the SC of the save TN for a TN ;;; with a :SPECIFIED-SAVE TN, then we pack in that location, instead ;;; of allocating a new stack location. -(defun pack-tn (tn restricted) +(defun pack-tn (tn restricted optimize &key (allow-unbounded-sc t)) (declare (type tn tn)) + (aver (not (tn-offset tn))) (let* ((original (original-tn tn)) - (fsc (tn-sc tn)) - (alternates (unless restricted (sc-alternate-scs fsc))) - (save (tn-save-tn tn)) - (specified-save-sc - (when (and save - (eq (tn-kind save) :specified-save)) - (tn-sc save)))) - + (fsc (tn-sc tn)) + (alternates (unless restricted (sc-alternate-scs fsc))) + (save (tn-save-tn tn)) + (specified-save-sc + (when (and save + (eq (tn-kind save) :specified-save)) + (tn-sc save)))) (do ((sc fsc (pop alternates))) - ((null sc) - (failed-to-pack-error tn restricted)) + ((null sc) + (failed-to-pack-error tn restricted)) + (unless (or allow-unbounded-sc + (not (unbounded-sc-p sc))) + (return nil)) (when (eq sc specified-save-sc) - (unless (tn-offset save) - (pack-tn save nil)) - (setf (tn-offset tn) (tn-offset save)) - (setf (tn-sc tn) (tn-sc save)) - (return)) + (unless (tn-offset save) + (pack-tn save nil optimize)) + (setf (tn-offset tn) (tn-offset save)) + (setf (tn-sc tn) (tn-sc save)) + (return t)) (when (or restricted - (not (and (minusp (tn-cost tn)) (sc-save-p sc)))) - (let ((loc (or (find-ok-target-offset original sc) - (select-location original sc) - (and restricted - (select-location original sc t)) - (when (eq (sb-kind (sc-sb sc)) :unbounded) - (grow-sc sc) - (or (select-location original sc) - (error "failed to pack after growing SC?")))))) - (when loc - (add-location-conflicts original sc loc) - (setf (tn-sc tn) sc) - (setf (tn-offset tn) loc) - (return)))))) - + (not (and (minusp (tn-cost tn)) (sc-save-p sc)))) + (let ((loc (or (find-ok-target-offset original sc) + (select-location original sc :optimize optimize) + (and restricted + (select-location original sc :use-reserved-locs t + :optimize optimize)) + (when (unbounded-sc-p sc) + (grow-sc sc) + (or (select-location original sc) + (error "failed to pack after growing SC?")))))) + (when loc + (add-location-conflicts original sc loc optimize) + (setf (tn-sc tn) sc) + (setf (tn-offset tn) loc) + (return t)))))) (values)) ;;; Pack a wired TN, checking that the offset is in bounds for the SB, ;;; and that the TN doesn't conflict with some other TN already packed ;;; in that location. If the TN is wired to a location beyond the end -;;; of a :Unbounded SB, then grow the SB enough to hold the TN. +;;; of a :UNBOUNDED SB, then grow the SB enough to hold the TN. ;;; ;;; ### Checking for conflicts is disabled for :SPECIFIED-SAVE TNs. ;;; This is kind of a hack to make specifying wired stack save ;;; locations for local call arguments (such as OLD-FP) work, since ;;; the caller and callee OLD-FP save locations may conflict when the ;;; save locations don't really (due to being in different frames.) -(defun pack-wired-tn (tn) +(defun pack-wired-tn (tn optimize) (declare (type tn tn)) (let* ((sc (tn-sc tn)) - (sb (sc-sb sc)) - (offset (tn-offset tn)) - (end (+ offset (sc-element-size sc))) - (original (original-tn tn))) + (sb (sc-sb sc)) + (offset (tn-offset tn)) + (end (+ offset (sc-element-size sc))) + (original (original-tn tn))) (when (> end (finite-sb-current-size sb)) (unless (eq (sb-kind sb) :unbounded) - (error "~S is wired to a location that is out of bounds." tn)) + (error "~S is wired to a location that is out of bounds." tn)) (grow-sc sc end)) ;; For non-x86 ports the presence of a save-tn associated with a ;; tn is used to identify the old-fp and return-pc tns. It depends ;; on the old-fp and return-pc being passed in registers. - #!-x86 + #!-(or x86 x86-64) (when (and (not (eq (tn-kind tn) :specified-save)) - (conflicts-in-sc original sc offset)) + (conflicts-in-sc original sc offset)) (error "~S is wired to a location that it conflicts with." tn)) ;; Use the above check, but only print a verbose warning. This can ;; be helpful for debugging the x86 port. #+nil (when (and (not (eq (tn-kind tn) :specified-save)) - (conflicts-in-sc original sc offset)) - (format t "~&* Pack-wired-tn possible conflict:~% ~ - tn: ~S; tn-kind: ~S~% ~ - sc: ~S~% ~ - sb: ~S; sb-name: ~S; sb-kind: ~S~% ~ - offset: ~S; end: ~S~% ~ - original ~S~% ~ - tn-save-tn: ~S; tn-kind of tn-save-tn: ~S~%" - tn (tn-kind tn) sc - sb (sb-name sb) (sb-kind sb) - offset end - original - (tn-save-tn tn) (tn-kind (tn-save-tn tn)))) + (conflicts-in-sc original sc offset)) + (format t "~&* Pack-wired-tn possible conflict:~% ~ + tn: ~S; tn-kind: ~S~% ~ + sc: ~S~% ~ + sb: ~S; sb-name: ~S; sb-kind: ~S~% ~ + offset: ~S; end: ~S~% ~ + original ~S~% ~ + tn-save-tn: ~S; tn-kind of tn-save-tn: ~S~%" + tn (tn-kind tn) sc + sb (sb-name sb) (sb-kind sb) + offset end + original + (tn-save-tn tn) (tn-kind (tn-save-tn tn)))) ;; On the x86 ports the old-fp and return-pc are often passed on ;; the stack so the above hack for the other ports does not always ;; work. Here the old-fp and return-pc tns are identified by being ;; on the stack in their standard save locations. - #!+x86 + #!+(or x86 x86-64) (when (and (not (eq (tn-kind tn) :specified-save)) - (not (and (string= (sb-name sb) "STACK") - (or (= offset 0) - (= offset 1)))) - (conflicts-in-sc original sc offset)) - (error "~S is wired to a location that it conflicts with." tn)) + (not (and (string= (sb-name sb) "STACK") + (or (= offset 0) + (= offset 1)))) + (conflicts-in-sc original sc offset)) + (error "~S is wired to location ~D in SC ~A of kind ~S that it conflicts with." + tn offset sc (tn-kind tn))) - (add-location-conflicts original sc offset))) + (add-location-conflicts original sc offset optimize))) (defevent repack-block "Repacked a block due to TN unpacking.") +;;; KLUDGE: Prior to SBCL version 0.8.9.xx, this function was known as +;;; PACK-BEFORE-GC-HOOK, but was non-functional since approximately +;;; version 0.8.3.xx since the removal of GC hooks from the system. +;;; This currently (as of 2004-04-12) runs now after every call to +;;; PACK, rather than -- as was originally intended -- once per GC +;;; cycle; this is probably non-optimal, and might require tuning, +;;; maybe to be called when the data structures exceed a certain size, +;;; or maybe once every N times. The KLUDGE is that this rewrite has +;;; done nothing to improve the reentrance or threadsafety of the +;;; compiler; it still fails to be callable from several threads at +;;; the same time. +;;; +;;; Brief experiments indicate that during a compilation cycle this +;;; causes about 10% more consing, and takes about 1%-2% more time. +;;; +;;; -- CSR, 2004-04-12 +(defun clean-up-pack-structures () + (dolist (sb *backend-sb-list*) + (unless (eq (sb-kind sb) :non-packed) + (let ((size (sb-size sb))) + (fill (finite-sb-always-live sb) nil) + (setf (finite-sb-always-live sb) + (make-array size + :initial-element + #-sb-xc #* + ;; The cross-compiler isn't very good at + ;; dumping specialized arrays, so we delay + ;; construction of this SIMPLE-BIT-VECTOR + ;; until runtime. + #+sb-xc (make-array 0 :element-type 'bit))) + (setf (finite-sb-always-live-count sb) + (make-array size + :initial-element + #-sb-xc #* + ;; Ibid + #+sb-xc (make-array 0 :element-type 'fixnum))) + + (fill (finite-sb-conflicts sb) nil) + (setf (finite-sb-conflicts sb) + (make-array size :initial-element '#())) + + (fill (finite-sb-live-tns sb) nil) + (setf (finite-sb-live-tns sb) + (make-array size :initial-element nil)))))) + +(defun tn-lexical-depth (tn) + (let ((path t)) ; dummy initial value + (labels ((path (lambda) + (do ((acc '()) + (lambda lambda (lambda-parent lambda))) + ((null lambda) acc) + (push lambda acc))) + (register-scope (lambda) + (let ((new-path (path lambda))) + (setf path (if (eql path t) + new-path + (subseq path + 0 (mismatch path new-path)))))) + (walk-tn-refs (ref) + (do ((ref ref (tn-ref-next ref))) + ((or (null ref) + (null path))) + (awhen (vop-node (tn-ref-vop ref)) + (register-scope (lexenv-lambda (node-lexenv it))))))) + (walk-tn-refs (tn-reads tn)) + (walk-tn-refs (tn-writes tn)) + (if (eql path t) + most-positive-fixnum + (length path))))) + (defun pack (component) - (aver (not *in-pack*)) - (let ((*in-pack* t) - (optimize (policy *lexenv* - (or (>= speed compilation-speed) - (>= space compilation-speed)))) - (2comp (component-info component))) - (init-sb-vectors component) - - ;; Call the target functions. - (do-ir2-blocks (block component) - (do ((vop (ir2-block-start-vop block) (vop-next vop))) - ((null vop)) - (let ((target-fun (vop-info-target-fun (vop-info vop)))) - (when target-fun - (funcall target-fun vop))))) - - - ;; Pack wired TNs first. - (do ((tn (ir2-component-wired-tns 2comp) (tn-next tn))) - ((null tn)) - (pack-wired-tn tn)) - - ;; Pack restricted component TNs. - (do ((tn (ir2-component-restricted-tns 2comp) (tn-next tn))) - ((null tn)) - (when (eq (tn-kind tn) :component) - (pack-tn tn t))) + (unwind-protect + (let ((optimize nil) + (2comp (component-info component))) + (init-sb-vectors component) + + ;; Determine whether we want to do more expensive packing by + ;; checking whether any blocks in the component have (> SPEED + ;; COMPILE-SPEED). + ;; + ;; FIXME: This means that a declaration can have a minor + ;; effect even outside its scope, and as the packing is done + ;; component-globally it'd be tricky to use strict scoping. I + ;; think this is still acceptable since it's just a tradeoff + ;; between compilation speed and allocation quality and + ;; doesn't affect the semantics of the generated code in any + ;; way. -- JES 2004-10-06 + (do-ir2-blocks (block component) + (when (policy (block-last (ir2-block-block block)) + (> speed compilation-speed)) + (setf optimize t) + (return))) + + ;; Call the target functions. + (do-ir2-blocks (block component) + (do ((vop (ir2-block-start-vop block) (vop-next vop))) + ((null vop)) + (let ((target-fun (vop-info-target-fun (vop-info vop)))) + (when target-fun + (funcall target-fun vop))))) + + ;; Assign costs to normal TNs so we know which ones should always + ;; be packed on the stack, and which are important not to spill. + (when *pack-assign-costs* + (assign-tn-costs component)) + + ;; Actually allocate registers for most TNs. After this, only + ;; :normal tns may be left unallocated (or TNs :restricted to + ;; an unbounded SC). + (pack-greedy component 2comp optimize) + + ;; Pack any leftover normal/restricted TN that is not already + ;; allocated to a finite SC, or TNs that do not appear in any + ;; local TN map (e.g. :MORE TNs). Since we'll likely be + ;; allocating on the stack, first allocate TNs that are + ;; associated with code at shallow lexical depths: this will + ;; allocate long live ranges (i.e. TNs with more conflicts) + ;; first, and hopefully minimise stack fragmentation. + ;; Component TNs are a degenerate case: they are always live. + (let ((component-tns '()) + (contiguous-tns '()) + (tns '())) + (flet ((register-tn (tn) + (unless (tn-offset tn) + (case (tn-kind tn) + (:component + (push tn component-tns)) + ((:environment :debug-environment) + (push tn contiguous-tns)) + (t + (push tn tns)))))) + (do ((tn (ir2-component-restricted-tns 2comp) (tn-next tn))) + ((null tn)) + ;; by this time, restricted TNs must either be + ;; allocated in the right SC or unbounded + (aver (or (tn-offset tn) (unbounded-tn-p tn))) + (register-tn tn)) + (do ((tn (ir2-component-normal-tns 2comp) (tn-next tn))) + ((null tn)) + (register-tn tn))) + (flet ((pack-tns (tns &optional in-order) + (dolist (tn (if in-order + tns + (schwartzian-stable-sort-list + tns #'< :key #'tn-lexical-depth))) + (unless (tn-offset tn) + (pack-tn tn nil optimize))))) + ;; first pack TNs that are known to have simple live + ;; ranges (contiguous lexical scopes) + (pack-tns component-tns t) + (pack-tns contiguous-tns) + (pack-tns tns))) + + ;; Do load TN packing and emit saves. + (let ((*repack-blocks* nil)) + (cond ((and optimize *pack-optimize-saves*) + (optimized-emit-saves component) + (do-ir2-blocks (block component) + (pack-load-tns block))) + (t + (do-ir2-blocks (block component) + (emit-saves block) + (pack-load-tns block)))) + (loop + (unless *repack-blocks* (return)) + (let ((orpb *repack-blocks*)) + (setq *repack-blocks* nil) + (dolist (block orpb) + (event repack-block) + (pack-load-tns block))))) + + (values)) + (clean-up-pack-structures))) + +(defun pack-greedy (component 2comp optimize) + (declare (type component component) + (type ir2-component 2comp)) + ;; Pack wired TNs first. + (do ((tn (ir2-component-wired-tns 2comp) (tn-next tn))) + ((null tn)) + (pack-wired-tn tn optimize)) - ;; Pack other restricted TNs. + ;; Then, pack restricted TNs, ones that are live over the whole + ;; component first (they cause no fragmentation). Sort by TN cost + ;; to help important TNs get good targeting. + (collect ((component) + (normal)) (do ((tn (ir2-component-restricted-tns 2comp) (tn-next tn))) - ((null tn)) - (unless (tn-offset tn) - (pack-tn tn t))) - - ;; Assign costs to normal TNs so we know which ones should always - ;; be packed on the stack. - (when (and optimize *pack-assign-costs*) - (assign-tn-costs component)) - - ;; Pack normal TNs in the order that they appear in the code. This - ;; should have some tendency to pack important TNs first, since - ;; control analysis favors the drop-through. This should also help - ;; targeting, since we will pack the target TN soon after we - ;; determine the location of the targeting TN. - (do-ir2-blocks (block component) - (let ((ltns (ir2-block-local-tns block))) - (do ((i (1- (ir2-block-local-tn-count block)) (1- i))) - ((minusp i)) - (declare (fixnum i)) - (let ((tn (svref ltns i))) - (unless (or (null tn) (eq tn :more) (tn-offset tn)) - (pack-tn tn nil)))))) - - ;; Pack any leftover normal TNs. This is to deal with :MORE TNs, - ;; which could possibly not appear in any local TN map. - (do ((tn (ir2-component-normal-tns 2comp) (tn-next tn))) - ((null tn)) - (unless (tn-offset tn) - (pack-tn tn nil))) - - ;; Do load TN packing and emit saves. - (let ((*repack-blocks* nil)) - (cond ((and optimize *pack-optimize-saves*) - (optimized-emit-saves component) - (do-ir2-blocks (block component) - (pack-load-tns block))) - (t - (do-ir2-blocks (block component) - (emit-saves block) - (pack-load-tns block)))) - (when *repack-blocks* - (loop - (when (zerop (hash-table-count *repack-blocks*)) (return)) - (maphash (lambda (block v) - (declare (ignore v)) - (remhash block *repack-blocks*) - (event repack-block) - (pack-load-tns block)) - *repack-blocks*))))) - - (values)) + ((null tn)) + (unless (or (tn-offset tn) (unbounded-tn-p tn)) + (if (eq :component (tn-kind tn)) + (component tn) + (normal tn)))) + (flet ((pack-tns (tns) + (dolist (tn (stable-sort tns #'> :key #'tn-cost)) + (pack-tn tn t optimize)))) + (pack-tns (component)) + (pack-tns (normal)))) + + (cond ((and *loop-analyze* *pack-assign-costs*) + ;; Allocate normal TNs, starting with the TNs that are + ;; heavily used in deep loops (which is taken into account in + ;; TN spill costs). Only allocate in finite SCs (i.e. not on + ;; the stack). + (collect ((tns)) + (do ((tn (ir2-component-normal-tns 2comp) (tn-next tn))) + ((null tn)) + (unless (or (tn-offset tn) + (eq (tn-kind tn) :more) + (unbounded-tn-p tn) + (and (sc-save-p (tn-sc tn)) ; SC caller-save, but TN + (minusp (tn-cost tn)))) ; lives over many calls + (tns tn))) + (dolist (tn (stable-sort (tns) #'> :key #'tn-cost)) + (unless (tn-offset tn) + ;; if it can't fit in a bounded SC, the final pass will + ;; take care of stack packing. + (pack-tn tn nil optimize :allow-unbounded-sc nil))))) + (t + ;; If loop analysis has been disabled we might as well revert + ;; to the old behaviour of just packing TNs linearly as they + ;; appear. + (do-ir2-blocks (block component) + (let ((ltns (ir2-block-local-tns block))) + (do ((i (1- (ir2-block-local-tn-count block)) (1- i))) + ((minusp i)) + (declare (fixnum i)) + (let ((tn (svref ltns i))) + (unless (or (null tn) + (eq tn :more) + (tn-offset tn) + (unbounded-tn-p tn)) + (pack-tn tn nil optimize :allow-unbounded-sc nil)))))))))