X-Git-Url: http://repo.macrolet.net/gitweb/?a=blobdiff_plain;f=src%2Fcompiler%2Fpack.lisp;h=114e61ea9240f5db28d81d45a598170bbaf6535e;hb=77d94d36bcfd3d5eea73ad51e6ee621a8938f995;hp=b0a94674d3cf39a556dac91d16cb0fa47d1e4d59;hpb=5ec8d0c1c8b7939818b75118b472fac1af554f9a;p=sbcl.git diff --git a/src/compiler/pack.lisp b/src/compiler/pack.lisp index b0a9467..114e61e 100644 --- a/src/compiler/pack.lisp +++ b/src/compiler/pack.lisp @@ -24,19 +24,21 @@ ;;;; conflict determination -;;; Return true if the element at the specified offset in SB has a conflict -;;; with TN: -;;; -- If an component-live TN (:component kind), then iterate over all the -;;; blocks. If the element at Offset is used anywhere in any of the -;;; component's blocks (always-live /= 0), then there is a conflict. -;;; -- If TN is global (Confs true), then iterate over the blocks TN is live in -;;; (using TN-Global-Conflicts). If the TN is live everywhere in the block -;;; (:Live), then there is a conflict if the element at offset is used -;;; anywhere in the block (Always-Live /= 0). Otherwise, we use the local -;;; TN number for TN in block to find whether TN has a conflict at Offset in +;;; Return true if the element at the specified offset in SB has a +;;; conflict with TN: +;;; -- If a component-live TN (:COMPONENT kind), then iterate over +;;; all the blocks. If the element at OFFSET is used anywhere in +;;; any of the component's blocks (always-live /= 0), then there +;;; is a conflict. +;;; -- If TN is global (Confs true), then iterate over the blocks TN +;;; is live in (using TN-GLOBAL-CONFLICTS). If the TN is live +;;; everywhere in the block (:LIVE), then there is a conflict +;;; if the element at offset is used anywhere in the block +;;; (Always-Live /= 0). Otherwise, we use the local TN number for +;;; TN in block to find whether TN has a conflict at Offset in ;;; that block. -;;; -- If TN is local, then we just check for a conflict in the block it is -;;; local to. +;;; -- If TN is local, then we just check for a conflict in the block +;;; it is local to. (defun offset-conflicts-in-sb (tn sb offset) (declare (type tn tn) (type finite-sb sb) (type index offset)) (let ((confs (tn-global-conflicts tn)) @@ -50,7 +52,7 @@ (confs (let ((loc-confs (svref (finite-sb-conflicts sb) offset)) (loc-live (svref (finite-sb-always-live sb) offset))) - (do ((conf confs (global-conflicts-tn-next conf))) + (do ((conf confs (global-conflicts-next-tnwise conf))) ((null conf) nil) (let* ((block (global-conflicts-block conf)) @@ -76,21 +78,21 @@ (when (offset-conflicts-in-sb tn sb (+ offset i)) (return t))))) -;;; Add TN's conflicts into the conflicts for the location at Offset in SC. -;;; We iterate over each location in TN, adding to the conflicts for that -;;; location: -;;; -- If TN is a :Component TN, then iterate over all the blocks, setting -;;; all of the local conflict bits and the always-live bit. This records a -;;; conflict with any TN that has a LTN number in the block, as well as with -;;; :Always-Live and :Environment TNs. +;;; Add TN's conflicts into the conflicts for the location at OFFSET +;;; in SC. We iterate over each location in TN, adding to the +;;; conflicts for that location: +;;; -- If TN is a :COMPONENT TN, then iterate over all the blocks, +;;; setting all of the local conflict bits and the always-live bit. +;;; This records a conflict with any TN that has a LTN number in +;;; the block, as well as with :ALWAYS-LIVE and :ENVIRONMENT TNs. ;;; -- If TN is global, then iterate over the blocks TN is live in. In -;;; addition to setting the always-live bit to represent the conflict with -;;; TNs live throughout the block, we also set bits in the local conflicts. -;;; If TN is :Always-Live in the block, we set all the bits, otherwise we or -;;; in the local conflict bits. -;;; -- If the TN is local, then we just do the block it is local to, setting -;;; always-live and OR'ing in the local conflicts. -(defun add-location-conflicts (tn sc offset) +;;; addition to setting the always-live bit to represent the conflict +;;; with TNs live throughout the block, we also set bits in the +;;; local conflicts. If TN is :ALWAYS-LIVE in the block, we set all +;;; the bits, otherwise we OR in the local conflict bits. +;;; -- If the TN is local, then we just do the block it is local to, +;;; setting always-live and OR'ing in the local conflicts. +(defun add-location-conflicts (tn sc offset optimize) (declare (type tn tn) (type sc sc) (type index offset)) (let ((confs (tn-global-conflicts tn)) (sb (sc-sb sc)) @@ -102,12 +104,12 @@ (loc-live (svref (finite-sb-always-live sb) this-offset))) (cond ((eq kind :component) - (dotimes (num (ir2-block-count *component-being-compiled*) nil) + (dotimes (num (ir2-block-count *component-being-compiled*)) (declare (type index num)) (setf (sbit loc-live num) 1) (set-bit-vector (svref loc-confs num)))) (confs - (do ((conf confs (global-conflicts-tn-next conf))) + (do ((conf confs (global-conflicts-next-tnwise conf))) ((null conf)) (let* ((block (global-conflicts-block conf)) (num (ir2-block-number block)) @@ -121,9 +123,26 @@ (let ((num (ir2-block-number (tn-local tn)))) (setf (sbit loc-live num) 1) (bit-ior (the local-tn-bit-vector (svref loc-confs num)) - (tn-local-conflicts tn) t)))))))) + (tn-local-conflicts tn) t)))) + ;; Calculating ALWAYS-LIVE-COUNT is moderately expensive, and + ;; currently the information isn't used unless (> SPEED + ;; COMPILE-SPEED). + (when optimize + (setf (svref (finite-sb-always-live-count sb) this-offset) + (find-location-usage sb this-offset)))))) + (values)) -;;; Return the total number of IR2 blocks in Component. +;; A rought measure of how much a given OFFSET in SB is currently +;; used. Current implementation counts the amount of blocks where the +;; offset has been marked as ALWAYS-LIVE. +(defun find-location-usage (sb offset) + (declare (optimize speed)) + (declare (type sb sb) (type index offset)) + (let* ((always-live (svref (finite-sb-always-live sb) offset))) + (declare (simple-bit-vector always-live)) + (count 1 always-live))) + +;;; Return the total number of IR2-BLOCKs in COMPONENT. (defun ir2-block-count (component) (declare (type component component)) (do ((2block (block-info (block-next (component-head component))) @@ -133,7 +152,7 @@ (when (ir2-block-number 2block) (return (1+ (ir2-block-number 2block)))))) -;;; Ensure that the conflicts vectors for each :Finite SB are large +;;; Ensure that the conflicts vectors for each :FINITE SB are large ;;; enough for the number of blocks allocated. Also clear any old ;;; conflicts and reset the current size to the initial size. (defun init-sb-vectors (component) @@ -142,6 +161,7 @@ (unless (eq (sb-kind sb) :non-packed) (let* ((conflicts (finite-sb-conflicts sb)) (always-live (finite-sb-always-live sb)) + (always-live-count (finite-sb-always-live-count sb)) (max-locs (length conflicts)) (last-count (finite-sb-last-block-count sb))) (unless (zerop max-locs) @@ -170,7 +190,8 @@ (setf (svref conflicts i) new-vec)) (setf (svref always-live i) (make-array new-size :element-type 'bit - :initial-element 0))))) + :initial-element 0)) + (setf (svref always-live-count i) 0)))) (t (dotimes (i (finite-sb-current-size sb)) (declare (type index i)) @@ -179,14 +200,15 @@ (dotimes (j last-count) (declare (type index j)) (clear-bit-vector (svref conf j)))) - (clear-bit-vector (svref always-live i))))))) + (clear-bit-vector (svref always-live i)) + (setf (svref always-live-count i) 0)))))) (setf (finite-sb-last-block-count sb) nblocks) (setf (finite-sb-current-size sb) (sb-size sb)) (setf (finite-sb-last-offset sb) 0)))))) -;;; Expand the :Unbounded SB backing SC by either the initial size or -;;; the SC element size, whichever is larger. If Needed-Size is +;;; Expand the :UNBOUNDED SB backing SC by either the initial size or +;;; the SC element size, whichever is larger. If NEEDED-SIZE is ;;; larger, then use that size. (defun grow-sc (sc &optional (needed-size 0)) (declare (type sc sc) (type index needed-size)) @@ -231,6 +253,12 @@ :element-type 'bit))) (setf (finite-sb-always-live sb) new-live)) + (let ((new-live-count (make-array new-size))) + (declare (optimize speed)) ;; FILL deftransform + (replace new-live-count (finite-sb-always-live-count sb)) + (fill new-live-count 0 :start size) + (setf (finite-sb-always-live-count sb) new-live-count)) + (let ((new-tns (make-array new-size :initial-element nil))) (replace new-tns (finite-sb-live-tns sb)) (fill (finite-sb-live-tns sb) nil) @@ -239,45 +267,12 @@ (setf (finite-sb-current-size sb) new-size)) (values)) -;;; This variable is true whenever we are in pack (and thus the per-SB -;;; conflicts information is in use.) -(defvar *in-pack* nil) - -;;; In order to prevent the conflict data structures from growing -;;; arbitrarily large, we clear them whenever a GC happens and we -;;; aren't currently in pack. We revert to the initial number of -;;; locations and 0 blocks. -(defun pack-before-gc-hook () - (unless *in-pack* - (dolist (sb *backend-sb-list*) - (unless (eq (sb-kind sb) :non-packed) - (let ((size (sb-size sb))) - (fill nil (finite-sb-always-live sb)) - (setf (finite-sb-always-live sb) - (make-array size - :initial-element - #-sb-xc #* - ;; The cross-compiler isn't very good at dumping - ;; specialized arrays, so we delay construction of - ;; this SIMPLE-BIT-VECTOR until runtime. - #+sb-xc (make-array 0 :element-type 'bit))) - - (fill nil (finite-sb-conflicts sb)) - (setf (finite-sb-conflicts sb) - (make-array size :initial-element '#())) - - (fill nil (finite-sb-live-tns sb)) - (setf (finite-sb-live-tns sb) - (make-array size :initial-element nil)))))) - (values)) - -(pushnew 'pack-before-gc-hook sb!ext:*before-gc-hooks*) ;;;; internal errors -;;; Give someone a hard time because there isn't any load function defined -;;; to move from Src to Dest. -(defun no-load-function-error (src dest) +;;; Give someone a hard time because there isn't any load function +;;; defined to move from SRC to DEST. +(defun no-load-fun-error (src dest) (let* ((src-sc (tn-sc src)) (src-name (sc-name src-sc)) (dest-sc (tn-sc dest)) @@ -285,23 +280,23 @@ (cond ((eq (sb-kind (sc-sb src-sc)) :non-packed) (unless (member src-sc (sc-constant-scs dest-sc)) (error "loading from an invalid constant SC?~@ - VM definition inconsistent, try recompiling.")) + VM definition inconsistent, try recompiling.")) (error "no load function defined to load SC ~S ~ - from its constant SC ~S" + from its constant SC ~S" dest-name src-name)) ((member src-sc (sc-alternate-scs dest-sc)) (error "no load function defined to load SC ~S from its ~ - alternate SC ~S" + alternate SC ~S" dest-name src-name)) ((member dest-sc (sc-alternate-scs src-sc)) (error "no load function defined to save SC ~S in its ~ - alternate SC ~S" + alternate SC ~S" src-name dest-name)) (t ;; FIXME: "VM definition is inconsistent" shouldn't be a ;; possibility in SBCL. (error "loading to/from SCs that aren't alternates?~@ - VM definition is inconsistent, try recompiling."))))) + VM definition is inconsistent, try recompiling."))))) ;;; Called when we failed to pack TN. If RESTRICTED is true, then we ;;; are restricted to pack TN in its SC. @@ -315,16 +310,16 @@ tn (sc-name sc))) (t (aver (not (find :unbounded scs - :key #'(lambda (x) (sb-kind (sc-sb x)))))) + :key (lambda (x) (sb-kind (sc-sb x)))))) (let ((ptype (tn-primitive-type tn))) (cond (ptype (aver (member (sc-number sc) (primitive-type-scs ptype))) - (error "SC ~S doesn't have any :Unbounded alternate SCs, but is~@ - a SC for primitive-type ~S." + (error "SC ~S doesn't have any :UNBOUNDED alternate SCs, but is~@ + a SC for primitive-type ~S." (sc-name sc) (primitive-type-name ptype))) (t - (error "SC ~S doesn't have any :Unbounded alternate SCs." + (error "SC ~S doesn't have any :UNBOUNDED alternate SCs." (sc-name sc))))))))) ;;; Return a list of format arguments describing how TN is used in @@ -334,7 +329,7 @@ (args (vop-args vop)) (results (vop-results vop)) (name (with-output-to-string (stream) - (print-tn tn stream))) + (print-tn-guts tn stream))) (2comp (component-info *component-being-compiled*)) temp) (cond @@ -393,13 +388,13 @@ (declare (ignore costs load-scs)) (aver (not more-p)) (error "unable to pack a Load-TN in SC ~{~A~#[~^~;, or ~:;,~]~} ~ - for the ~:R ~:[result~;argument~] to~@ - the ~S VOP,~@ - ~:[since all SC elements are in use:~:{~%~@?~}~%~;~ - ~:*but these SC elements are not in use:~% ~S~%Bug?~*~]~ - ~:[~;~@ - Current cost info inconsistent with that in effect at compile ~ - time. Recompile.~%Compilation order may be incorrect.~]" + for the ~:R ~:[result~;argument~] to~@ + the ~S VOP,~@ + ~:[since all SC elements are in use:~:{~%~@?~}~%~;~ + ~:*but these SC elements are not in use:~% ~S~%Bug?~*~]~ + ~:[~;~@ + Current cost info inconsistent with that in effect at compile ~ + time. Recompile.~%Compilation order may be incorrect.~]" (mapcar #'sc-name scs) n arg-p (vop-info-name (vop-info (tn-ref-vop op))) @@ -417,12 +412,12 @@ (declare (ignore costs)) (aver (not more-p)) (error "~S is not valid as the ~:R ~:[result~;argument~] to VOP:~ - ~% ~S,~@ - since the TN's primitive type ~S doesn't allow any of the SCs~@ - allowed by the operand restriction:~% ~S~ - ~:[~;~@ - Current cost info inconsistent with that in effect at compile ~ - time. Recompile.~%Compilation order may be incorrect.~]" + ~% ~S,~@ + since the TN's primitive type ~S doesn't allow any of the SCs~@ + allowed by the operand restriction:~% ~S~ + ~:[~;~@ + Current cost info inconsistent with that in effect at compile ~ + time. Recompile.~%Compilation order may be incorrect.~]" tn pos arg-p (template-name (vop-info (tn-ref-vop ref))) (primitive-type-name ptype) @@ -439,8 +434,9 @@ (pushnew tn (gethash vop (ir2-component-spilled-vops 2comp))))) (values)) -;;; Make a save TN for TN, pack it, and return it. We copy various conflict -;;; information from the TN so that pack does the right thing. +;;; Make a save TN for TN, pack it, and return it. We copy various +;;; conflict information from the TN so that pack does the right +;;; thing. (defun pack-save-tn (tn) (declare (type tn tn)) (let ((res (make-tn 0 :save nil nil))) @@ -451,7 +447,7 @@ (setf (tn-save-tn tn) res) (setf (tn-save-tn res) tn) (setf (tn-sc res) alt) - (pack-tn res t) + (pack-tn res t nil) (return res))))) ;;; Find the load function for moving from SRC to DEST and emit a @@ -462,9 +458,9 @@ (emit-load-template node block (template-or-lose 'move-operand) src dest - (list (or (svref (sc-move-functions (tn-sc dest)) + (list (or (svref (sc-move-funs (tn-sc dest)) (sc-number (tn-sc src))) - (no-load-function-error src dest))) + (no-load-fun-error src dest))) before) (values)) @@ -503,7 +499,7 @@ vop)) (emit-operand-load node block save tn next))) -;;; Return a VOP after which is an o.k. place to save the value of TN. +;;; Return a VOP after which is an OK place to save the value of TN. ;;; For correctness, it is only required that this location be after ;;; any possible write and before any possible restore location. ;;; @@ -569,8 +565,8 @@ (save-complex-writer-tn tn vop)))) (values)) -;;; Scan over the VOPs in Block, emiting saving code for TNs noted in the -;;; codegen info that are packed into saved SCs. +;;; Scan over the VOPs in BLOCK, emiting saving code for TNs noted in +;;; the codegen info that are packed into saved SCs. (defun emit-saves (block) (declare (type ir2-block block)) (do ((vop (ir2-block-start-vop block) (vop-next vop))) @@ -586,8 +582,8 @@ ;;;; optimized saving ;;; Save TN if it isn't a single-writer TN that has already been -;;; saved. If multi-write, we insert the save Before the specified -;;; VOP. Context is a VOP used to tell which node/block to use for the +;;; saved. If multi-write, we insert the save BEFORE the specified +;;; VOP. CONTEXT is a VOP used to tell which node/block to use for the ;;; new VOP. (defun save-if-necessary (tn before context) (declare (type tn tn) (type (or vop null) before) (type vop context)) @@ -602,7 +598,7 @@ (values)) ;;; Load the TN from its save location, allocating one if necessary. -;;; The load is inserted Before the specifier VOP. Context is a VOP +;;; The load is inserted BEFORE the specifier VOP. CONTEXT is a VOP ;;; used to tell which node/block to use for the new VOP. (defun restore-tn (tn before context) (declare (type tn tn) (type (or vop null) before) (type vop context)) @@ -645,10 +641,10 @@ ;;; ;;; SAVES and RESTORES are represented using both a list and a ;;; bit-vector so that we can quickly iterate and test for membership. -;;; The incoming Saves and Restores args are used for computing these +;;; The incoming SAVES and RESTORES args are used for computing these ;;; sets (the initial contents are ignored.) ;;; -;;; When we hit a VOP with :COMPUTE-ONLY Save-P (an internal error +;;; When we hit a VOP with :COMPUTE-ONLY SAVE-P (an internal error ;;; location), we pretend that all live TNs were read, unless (= speed ;;; 3), in which case we mark all the TNs that are live but not ;;; restored as spilled. @@ -739,10 +735,10 @@ (do ((read (vop-args vop) (tn-ref-across read))) ((null read)) (save-note-read (tn-ref-tn read)))))))))) - -;;; Like EMIT-SAVES, only different. We avoid redundant saving within -;;; the block, and don't restore values that aren't used before the -;;; next call. This function is just the top level loop over the + +;;; This is like EMIT-SAVES, only different. We avoid redundant saving +;;; within the block, and don't restore values that aren't used before +;;; the next call. This function is just the top level loop over the ;;; blocks in the component, which locates blocks that need saving ;;; done. (defun optimized-emit-saves (component) @@ -786,12 +782,46 @@ ((null ref)) (incf cost)) (setf (tn-cost tn) cost)))) + +;;; Iterate over the normal TNs, storing the depth of the deepest loop +;;; that the TN is used in TN-LOOP-DEPTH. +(defun assign-tn-depths (component) + (when *loop-analyze* + (do-ir2-blocks (block component) + (do ((vop (ir2-block-start-vop block) + (vop-next vop))) + ((null vop)) + (flet ((find-all-tns (head-fun) + (collect ((tns)) + (do ((ref (funcall head-fun vop) (tn-ref-across ref))) + ((null ref)) + (tns (tn-ref-tn ref))) + (tns)))) + (dolist (tn (nconc (find-all-tns #'vop-args) + (find-all-tns #'vop-results) + (find-all-tns #'vop-temps) + ;; What does "references in this VOP + ;; mean"? Probably something that isn't + ;; useful in this context, since these + ;; TN-REFs are linked with TN-REF-NEXT + ;; instead of TN-REF-ACROSS. --JES + ;; 2004-09-11 + ;; (find-all-tns #'vop-refs) + )) + (setf (tn-loop-depth tn) + (max (tn-loop-depth tn) + (let* ((ir1-block (ir2-block-block (vop-block vop))) + (loop (block-loop ir1-block))) + (if loop + (loop-depth loop) + 0)))))))))) + ;;;; load TN packing ;;; These variables indicate the last location at which we computed -;;; the Live-TNs. They hold the Block and VOP values that were passed -;;; to Compute-Live-TNs. +;;; the Live-TNs. They hold the BLOCK and VOP values that were passed +;;; to COMPUTE-LIVE-TNS. (defvar *live-block*) (defvar *live-vop*) @@ -801,8 +831,8 @@ (defvar *repack-blocks*) (declaim (type (or hash-table null) *repack-blocks*)) -;;; Set the Live-TNs vectors in all :Finite SBs to represent the TNs -;;; live at the end of Block. +;;; Set the LIVE-TNS vectors in all :FINITE SBs to represent the TNs +;;; live at the end of BLOCK. (defun init-live-tns (block) (dolist (sb *backend-sb-list*) (when (eq (sb-kind sb) :finite) @@ -823,8 +853,8 @@ (values)) -;;; Set the Live-TNs in :Finite SBs to represent the TNs live -;;; immediately after the evaluation of VOP in Block, excluding +;;; Set the LIVE-TNs in :FINITE SBs to represent the TNs live +;;; immediately after the evaluation of VOP in BLOCK, excluding ;;; results of the VOP. If VOP is null, then compute the live TNs at ;;; the beginning of the block. Sequential calls on the same block ;;; must be in reverse VOP order. @@ -878,7 +908,7 @@ (setq *live-vop* vop) (values)) -;;; This is kind of like Offset-Conflicts-In-SB, except that it uses +;;; This is kind of like OFFSET-CONFLICTS-IN-SB, except that it uses ;;; the VOP refs to determine whether a Load-TN for OP could be packed ;;; in the specified location, disregarding conflicts with TNs not ;;; referenced by this VOP. There is a conflict if either: @@ -960,7 +990,7 @@ (load-tn-offset-conflicts-in-sb op sb i)))) (when res (return res)))))) -;;; If a load-TN for Op is targeted to a legal location in SC, then +;;; If a load-TN for OP is targeted to a legal location in SC, then ;;; return the offset, otherwise return NIL. We see whether the target ;;; of the operand is packed, and try that location. There isn't any ;;; need to chain down the target path, since everything is packed @@ -1128,8 +1158,8 @@ (return res)))) (push sc allowed))))))))) -;;; Scan a list of load-SCs vectors and a list of TN-Refs threaded by -;;; TN-Ref-Across. When we find a reference whose TN doesn't satisfy +;;; Scan a list of load-SCs vectors and a list of TN-REFS threaded by +;;; TN-REF-ACROSS. When we find a reference whose TN doesn't satisfy ;;; the restriction, we pack a Load-TN and load the operand into it. ;;; If a load-tn has already been allocated, we can assume that the ;;; restriction is satisfied. @@ -1170,7 +1200,7 @@ (values)) -;;; Scan the VOPs in Block, looking for operands whose SC restrictions +;;; Scan the VOPs in BLOCK, looking for operands whose SC restrictions ;;; aren't satisfied. We do the results first, since they are ;;; evaluated later, and our conflict analysis is a backward scan. (defun pack-load-tns (block) @@ -1186,30 +1216,34 @@ (vop-args vop)))))) (values)) -;;;; location-selection, targeting & pack interface - ;;;; targeting -;;; Link the TN-Refs Read and Write together using the TN-Ref-Target when -;;; this seems like a good idea. Currently we always do, as this increases the -;;; success of load-TN targeting. +;;; Link the TN-REFS READ and WRITE together using the TN-REF-TARGET +;;; when this seems like a good idea. Currently we always do, as this +;;; increases the success of load-TN targeting. (defun target-if-desirable (read write) (declare (type tn-ref read write)) + ;; As per the comments at the definition of TN-REF-TARGET, read and + ;; write refs are always paired, with TARGET in the read pointing to + ;; the write and vice versa. + (aver (eq (tn-ref-write-p read) + (not (tn-ref-write-p write)))) (setf (tn-ref-target read) write) (setf (tn-ref-target write) read)) ;;; If TN can be packed into SC so as to honor a preference to TARGET, ;;; then return the offset to pack at, otherwise return NIL. TARGET -;;; must be already packed. We can honor a preference if: -;;; -- TARGET's location is in SC's locations. -;;; -- The element sizes of the two SCs are the same. -;;; -- TN doesn't conflict with target's location. +;;; must be already packed. (defun check-ok-target (target tn sc) (declare (type tn target tn) (type sc sc) (inline member)) (let* ((loc (tn-offset target)) (target-sc (tn-sc target)) (target-sb (sc-sb target-sc))) (declare (type index loc)) + ;; We can honor a preference if: + ;; -- TARGET's location is in SC's locations. + ;; -- The element sizes of the two SCs are the same. + ;; -- TN doesn't conflict with target's location. (if (and (eq target-sb (sc-sb sc)) (or (eq (sb-kind target-sb) :unbounded) (member loc (sc-locations sc))) @@ -1220,94 +1254,87 @@ nil))) ;;; Scan along the target path from TN, looking at readers or writers. -;;; When we find a packed TN, return Check-OK-Target of that TN. If +;;; When we find a packed TN, return CHECK-OK-TARGET of that TN. If ;;; there is no target, or if the TN has multiple readers (writers), ;;; then we return NIL. We also always return NIL after 10 iterations ;;; to get around potential circularity problems. -(macrolet ((frob (slot) - `(let ((count 10) - (current tn)) - (declare (type index count)) - (loop - (let ((refs (,slot current))) - (unless (and (plusp count) refs (not (tn-ref-next refs))) - (return nil)) - (let ((target (tn-ref-target refs))) - (unless target (return nil)) - (setq current (tn-ref-tn target)) - (when (tn-offset current) - (return (check-ok-target current tn sc))) - (decf count))))))) - (defun find-ok-target-offset (tn sc) - (declare (type tn tn) (type sc sc)) - (or (frob tn-reads) - (frob tn-writes)))) - +;;; +;;; FIXME: (30 minutes of reverse engineering?) It'd be nice to +;;; rewrite the header comment here to explain the interface and its +;;; motivation, and move remarks about implementation details (like +;;; 10!) inside. +(defun find-ok-target-offset (tn sc) + (declare (type tn tn) (type sc sc)) + (flet ((frob-slot (slot-fun) + (declare (type function slot-fun)) + (let ((count 10) + (current tn)) + (declare (type index count)) + (loop + (let ((refs (funcall slot-fun current))) + (unless (and (plusp count) + refs + (not (tn-ref-next refs))) + (return nil)) + (let ((target (tn-ref-target refs))) + (unless target (return nil)) + (setq current (tn-ref-tn target)) + (when (tn-offset current) + (return (check-ok-target current tn sc))) + (decf count))))))) + (declare (inline frob-slot)) ; until DYNAMIC-EXTENT works + (or (frob-slot #'tn-reads) + (frob-slot #'tn-writes)))) + ;;;; location selection ;;; Select some location for TN in SC, returning the offset if we -;;; succeed, and NIL if we fail. We start scanning at the Last-Offset -;;; in an attempt to distribute the TNs across all storage. -;;; -;;; We call Offset-Conflicts-In-SB directly, rather than using -;;; Conflicts-In-SC. This allows us to more efficient in packing -;;; multi-location TNs: we don't have to multiply the number of tests -;;; by the TN size. This falls out natually, since we have to be aware -;;; of TN size anyway so that we don't call Conflicts-In-SC on a bogus -;;; offset. -;;; -;;; We give up on finding a location after our current pointer has -;;; wrapped twice. This will result in testing some locations twice in -;;; the case that we fail, but is simpler than trying to figure out -;;; the soonest failure point. +;;; succeed, and NIL if we fail. ;;; -;;; We also give up without bothering to wrap if the current size -;;; isn't large enough to hold a single element of element-size -;;; without bothering to wrap. If it doesn't fit this iteration, it -;;; won't fit next. +;;; For :UNBOUNDED SCs just find the smallest correctly aligned offset +;;; where the TN doesn't conflict with the TNs that have already been +;;; packed. For :FINITE SCs try to pack the TN into the most heavily +;;; used locations first (as estimated in FIND-LOCATION-USAGE). ;;; -;;; ### Note that we actually try to pack as many consecutive TNs as -;;; possible in the same location, since we start scanning at the same -;;; offset that the last TN was successfully packed in. This is a -;;; weakening of the scattering hueristic that was put in to prevent -;;; restricted VOP temps from hogging all of the registers. This way, -;;; all of these temps probably end up in one register. -(defun select-location (tn sc &optional use-reserved-locs) +;;; Historically SELECT-LOCATION tried did the opposite and tried to +;;; distribute the TNs evenly across the available locations. At least +;;; on register-starved architectures (x86) this seems to be a bad +;;; strategy. -- JES 2004-09-11 +(defun select-location (tn sc &key use-reserved-locs optimize) (declare (type tn tn) (type sc sc) (inline member)) (let* ((sb (sc-sb sc)) (element-size (sc-element-size sc)) (alignment (sc-alignment sc)) (align-mask (1- alignment)) - (size (finite-sb-current-size sb)) - (start-offset (finite-sb-last-offset sb))) - (let ((current-start - (logandc2 (the index (+ start-offset align-mask)) align-mask)) - (wrap-p nil)) - (declare (type index current-start)) - (loop - (when (> (+ current-start element-size) size) - (cond ((or wrap-p (> element-size size)) - (return nil)) - (t - (setq current-start 0) - (setq wrap-p t)))) - - (if (or (eq (sb-kind sb) :unbounded) - (and (member current-start (sc-locations sc)) - (or use-reserved-locs - (not (member current-start - (sc-reserve-locations sc)))))) - (dotimes (i element-size - (return-from select-location current-start)) - (declare (type index i)) - (let ((offset (+ current-start i))) - (when (offset-conflicts-in-sb tn sb offset) - (setq current-start - (logandc2 (the index (+ (the index (1+ offset)) - align-mask)) - align-mask)) - (return)))) - (incf current-start alignment)))))) + (size (finite-sb-current-size sb))) + (flet ((attempt-location (start-offset) + (dotimes (i element-size + (return-from select-location start-offset)) + (declare (type index i)) + (let ((offset (+ start-offset i))) + (when (offset-conflicts-in-sb tn sb offset) + (return (logandc2 (the index (+ (the index (1+ offset)) + align-mask)) + align-mask))))))) + (if (eq (sb-kind sb) :unbounded) + (loop with offset = 0 + until (> (+ offset element-size) size) do + (setf offset (attempt-location offset))) + (let ((locations (sc-locations sc))) + (when optimize + (setf locations + (stable-sort (copy-list locations) #'> + :key (lambda (location-offset) + (loop for offset from location-offset + repeat element-size + maximize (svref + (finite-sb-always-live-count sb) + offset)))))) + (dolist (offset locations) + (when (or use-reserved-locs + (not (member offset + (sc-reserve-locations sc)))) + (attempt-location offset)))))))) ;;; If a save TN, return the saved TN, otherwise return TN. This is ;;; useful for getting the conflicts of a TN that might be a save TN. @@ -1316,7 +1343,7 @@ (if (member (tn-kind tn) '(:save :save-once :specified-save)) (tn-save-tn tn) tn)) - + ;;;; pack interface ;;; Attempt to pack TN in all possible SCs, first in the SC chosen by @@ -1329,7 +1356,7 @@ ;;; If we are attempting to pack in the SC of the save TN for a TN ;;; with a :SPECIFIED-SAVE TN, then we pack in that location, instead ;;; of allocating a new stack location. -(defun pack-tn (tn restricted) +(defun pack-tn (tn restricted optimize) (declare (type tn tn)) (let* ((original (original-tn tn)) (fsc (tn-sc tn)) @@ -1339,13 +1366,12 @@ (when (and save (eq (tn-kind save) :specified-save)) (tn-sc save)))) - (do ((sc fsc (pop alternates))) ((null sc) (failed-to-pack-error tn restricted)) (when (eq sc specified-save-sc) (unless (tn-offset save) - (pack-tn save nil)) + (pack-tn save nil optimize)) (setf (tn-offset tn) (tn-offset save)) (setf (tn-sc tn) (tn-sc save)) (return)) @@ -1354,30 +1380,29 @@ (let ((loc (or (find-ok-target-offset original sc) (select-location original sc) (and restricted - (select-location original sc t)) + (select-location original sc :use-reserved-locs t)) (when (eq (sb-kind (sc-sb sc)) :unbounded) (grow-sc sc) (or (select-location original sc) (error "failed to pack after growing SC?")))))) (when loc - (add-location-conflicts original sc loc) + (add-location-conflicts original sc loc optimize) (setf (tn-sc tn) sc) (setf (tn-offset tn) loc) (return)))))) - (values)) ;;; Pack a wired TN, checking that the offset is in bounds for the SB, ;;; and that the TN doesn't conflict with some other TN already packed ;;; in that location. If the TN is wired to a location beyond the end -;;; of a :Unbounded SB, then grow the SB enough to hold the TN. +;;; of a :UNBOUNDED SB, then grow the SB enough to hold the TN. ;;; ;;; ### Checking for conflicts is disabled for :SPECIFIED-SAVE TNs. ;;; This is kind of a hack to make specifying wired stack save ;;; locations for local call arguments (such as OLD-FP) work, since ;;; the caller and callee OLD-FP save locations may conflict when the ;;; save locations don't really (due to being in different frames.) -(defun pack-wired-tn (tn) +(defun pack-wired-tn (tn optimize) (declare (type tn tn)) (let* ((sc (tn-sc tn)) (sb (sc-sb sc)) @@ -1403,12 +1428,12 @@ (when (and (not (eq (tn-kind tn) :specified-save)) (conflicts-in-sc original sc offset)) (format t "~&* Pack-wired-tn possible conflict:~% ~ - tn: ~S; tn-kind: ~S~% ~ - sc: ~S~% ~ - sb: ~S; sb-name: ~S; sb-kind: ~S~% ~ - offset: ~S; end: ~S~% ~ - original ~S~% ~ - tn-save-tn: ~S; tn-kind of tn-save-tn: ~S~%" + tn: ~S; tn-kind: ~S~% ~ + sc: ~S~% ~ + sb: ~S; sb-name: ~S; sb-kind: ~S~% ~ + offset: ~S; end: ~S~% ~ + original ~S~% ~ + tn-save-tn: ~S; tn-kind of tn-save-tn: ~S~%" tn (tn-kind tn) sc sb (sb-name sb) (sb-kind sb) offset end @@ -1427,89 +1452,166 @@ (conflicts-in-sc original sc offset)) (error "~S is wired to a location that it conflicts with." tn)) - (add-location-conflicts original sc offset))) + (add-location-conflicts original sc offset optimize))) (defevent repack-block "Repacked a block due to TN unpacking.") -(defun pack (component) - (aver (not *in-pack*)) - (let ((*in-pack* t) - (optimize (policy *lexenv* - (or (>= speed compilation-speed) - (>= space compilation-speed)))) - (2comp (component-info component))) - (init-sb-vectors component) - - ;; Call the target functions. - (do-ir2-blocks (block component) - (do ((vop (ir2-block-start-vop block) (vop-next vop))) - ((null vop)) - (let ((target-fun (vop-info-target-function (vop-info vop)))) - (when target-fun - (funcall target-fun vop))))) - - - ;; Pack wired TNs first. - (do ((tn (ir2-component-wired-tns 2comp) (tn-next tn))) - ((null tn)) - (pack-wired-tn tn)) - - ;; Pack restricted component TNs. - (do ((tn (ir2-component-restricted-tns 2comp) (tn-next tn))) - ((null tn)) - (when (eq (tn-kind tn) :component) - (pack-tn tn t))) - - ;; Pack other restricted TNs. - (do ((tn (ir2-component-restricted-tns 2comp) (tn-next tn))) - ((null tn)) - (unless (tn-offset tn) - (pack-tn tn t))) - - ;; Assign costs to normal TNs so we know which ones should always - ;; be packed on the stack. - (when (and optimize *pack-assign-costs*) - (assign-tn-costs component)) - - ;; Pack normal TNs in the order that they appear in the code. This - ;; should have some tendency to pack important TNs first, since - ;; control analysis favors the drop-through. This should also help - ;; targeting, since we will pack the target TN soon after we - ;; determine the location of the targeting TN. - (do-ir2-blocks (block component) - (let ((ltns (ir2-block-local-tns block))) - (do ((i (1- (ir2-block-local-tn-count block)) (1- i))) - ((minusp i)) - (declare (fixnum i)) - (let ((tn (svref ltns i))) - (unless (or (null tn) (eq tn :more) (tn-offset tn)) - (pack-tn tn nil)))))) - - ;; Pack any leftover normal TNs. This is to deal with :MORE TNs, - ;; which could possibly not appear in any local TN map. - (do ((tn (ir2-component-normal-tns 2comp) (tn-next tn))) - ((null tn)) - (unless (tn-offset tn) - (pack-tn tn nil))) - - ;; Do load TN packing and emit saves. - (let ((*repack-blocks* nil)) - (cond ((and optimize *pack-optimize-saves*) - (optimized-emit-saves component) - (do-ir2-blocks (block component) - (pack-load-tns block))) - (t - (do-ir2-blocks (block component) - (emit-saves block) - (pack-load-tns block)))) - (when *repack-blocks* - (loop - (when (zerop (hash-table-count *repack-blocks*)) (return)) - (maphash #'(lambda (block v) - (declare (ignore v)) - (remhash block *repack-blocks*) - (event repack-block) - (pack-load-tns block)) - *repack-blocks*))))) +;;; KLUDGE: Prior to SBCL version 0.8.9.xx, this function was known as +;;; PACK-BEFORE-GC-HOOK, but was non-functional since approximately +;;; version 0.8.3.xx since the removal of GC hooks from the system. +;;; This currently (as of 2004-04-12) runs now after every call to +;;; PACK, rather than -- as was originally intended -- once per GC +;;; cycle; this is probably non-optimal, and might require tuning, +;;; maybe to be called when the data structures exceed a certain size, +;;; or maybe once every N times. The KLUDGE is that this rewrite has +;;; done nothing to improve the reentrance or threadsafety of the +;;; compiler; it still fails to be callable from several threads at +;;; the same time. +;;; +;;; Brief experiments indicate that during a compilation cycle this +;;; causes about 10% more consing, and takes about 1%-2% more time. +;;; +;;; -- CSR, 2004-04-12 +(defun clean-up-pack-structures () + (dolist (sb *backend-sb-list*) + (unless (eq (sb-kind sb) :non-packed) + (let ((size (sb-size sb))) + (fill (finite-sb-always-live sb) nil) + (setf (finite-sb-always-live sb) + (make-array size + :initial-element + #-sb-xc #* + ;; The cross-compiler isn't very good at + ;; dumping specialized arrays, so we delay + ;; construction of this SIMPLE-BIT-VECTOR + ;; until runtime. + #+sb-xc (make-array 0 :element-type 'bit))) + (setf (finite-sb-always-live-count sb) + (make-array size + :initial-element + #-sb-xc #* + ;; Ibid + #+sb-xc (make-array 0 :element-type 'fixnum))) + + (fill (finite-sb-conflicts sb) nil) + (setf (finite-sb-conflicts sb) + (make-array size :initial-element '#())) + + (fill (finite-sb-live-tns sb) nil) + (setf (finite-sb-live-tns sb) + (make-array size :initial-element nil)))))) - (values)) +(defun pack (component) + (unwind-protect + (let ((optimize nil) + (2comp (component-info component))) + (init-sb-vectors component) + + ;; Determine whether we want to do more expensive packing by + ;; checking whether any blocks in the component have (> SPEED + ;; COMPILE-SPEED). + ;; + ;; FIXME: This means that a declaration can have a minor + ;; effect even outside its scope, and as the packing is done + ;; component-globally it'd be tricky to use strict scoping. I + ;; think this is still acceptable since it's just a tradeoff + ;; between compilation speed and allocation quality and + ;; doesn't affect the semantics of the generated code in any + ;; way. -- JES 2004-10-06 + (do-ir2-blocks (block component) + (when (policy (block-last (ir2-block-block block)) + (> speed compilation-speed)) + (setf optimize t) + (return))) + + ;; Call the target functions. + (do-ir2-blocks (block component) + (do ((vop (ir2-block-start-vop block) (vop-next vop))) + ((null vop)) + (let ((target-fun (vop-info-target-fun (vop-info vop)))) + (when target-fun + (funcall target-fun vop))))) + + ;; Pack wired TNs first. + (do ((tn (ir2-component-wired-tns 2comp) (tn-next tn))) + ((null tn)) + (pack-wired-tn tn optimize)) + + ;; Pack restricted component TNs. + (do ((tn (ir2-component-restricted-tns 2comp) (tn-next tn))) + ((null tn)) + (when (eq (tn-kind tn) :component) + (pack-tn tn t optimize))) + + ;; Pack other restricted TNs. + (do ((tn (ir2-component-restricted-tns 2comp) (tn-next tn))) + ((null tn)) + (unless (tn-offset tn) + (pack-tn tn t optimize))) + + ;; Assign costs to normal TNs so we know which ones should + ;; always be packed on the stack. + (when *pack-assign-costs* + (assign-tn-costs component) + (assign-tn-depths component)) + + ;; Allocate normal TNs, starting with the TNs that are used + ;; in deep loops. + (collect ((tns)) + (do-ir2-blocks (block component) + (let ((ltns (ir2-block-local-tns block))) + (do ((i (1- (ir2-block-local-tn-count block)) (1- i))) + ((minusp i)) + (declare (fixnum i)) + (let ((tn (svref ltns i))) + (unless (or (null tn) + (eq tn :more) + (tn-offset tn)) + ;; If loop analysis has been disabled we might as + ;; well revert to the old behaviour of just + ;; packing TNs linearly as they appear. + (unless *loop-analyze* + (pack-tn tn nil optimize)) + (tns tn)))))) + (dolist (tn (stable-sort (tns) + (lambda (a b) + (cond + ((> (tn-loop-depth a) + (tn-loop-depth b)) + t) + ((= (tn-loop-depth a) + (tn-loop-depth b)) + (> (tn-cost a) (tn-cost b))) + (t nil))))) + (unless (tn-offset tn) + (pack-tn tn nil optimize)))) + + ;; Pack any leftover normal TNs. This is to deal with :MORE TNs, + ;; which could possibly not appear in any local TN map. + (do ((tn (ir2-component-normal-tns 2comp) (tn-next tn))) + ((null tn)) + (unless (tn-offset tn) + (pack-tn tn nil optimize))) + + ;; Do load TN packing and emit saves. + (let ((*repack-blocks* nil)) + (cond ((and optimize *pack-optimize-saves*) + (optimized-emit-saves component) + (do-ir2-blocks (block component) + (pack-load-tns block))) + (t + (do-ir2-blocks (block component) + (emit-saves block) + (pack-load-tns block)))) + (when *repack-blocks* + (loop + (when (zerop (hash-table-count *repack-blocks*)) (return)) + (maphash (lambda (block v) + (declare (ignore v)) + (remhash block *repack-blocks*) + (event repack-block) + (pack-load-tns block)) + *repack-blocks*)))) + + (values)) + (clean-up-pack-structures)))