1 ;;;; This file contains the implementation-independent code for Pack
2 ;;;; phase in the compiler. Pack is responsible for assigning TNs to
3 ;;;; storage allocations or "register allocation".
5 ;;;; This software is part of the SBCL system. See the README file for
8 ;;;; This software is derived from the CMU CL system, which was
9 ;;;; written at Carnegie Mellon University and released into the
10 ;;;; public domain. The software is in the public domain and is
11 ;;;; provided with absolutely no warranty. See the COPYING and CREDITS
12 ;;;; files for more information.
16 ;;; for debugging: some parameters controlling which optimizations we
18 (defvar *pack-assign-costs* t)
19 (defvar *pack-optimize-saves* t)
20 ;;; FIXME: Perhaps SB-FLUID should be renamed to SB-TWEAK and these
21 ;;; should be made conditional on SB-TWEAK.
23 (declaim (ftype (function (component) index) ir2-block-count))
25 ;;;; conflict determination
27 ;;; Return true if the element at the specified offset, or in any of
28 ;;; the [size-1] subsequent offsets, in SB has a conflict with TN:
29 ;;; -- If a component-live TN (:COMPONENT kind), then iterate over
30 ;;; all the blocks. If the element at OFFSET is used anywhere in
31 ;;; any of the component's blocks (always-live /= 0), then there
33 ;;; -- If TN is global (Confs true), then iterate over the blocks TN
34 ;;; is live in (using TN-GLOBAL-CONFLICTS). If the TN is live
35 ;;; everywhere in the block (:LIVE), then there is a conflict
36 ;;; if the element at offset is used anywhere in the block
37 ;;; (Always-Live /= 0). Otherwise, we use the local TN number for
38 ;;; TN in block to find whether TN has a conflict at Offset in
40 ;;; -- If TN is local, then we just check for a conflict in the block
43 ;;; If there is a conflict, returns the first such conflicting offset.
44 (defun offset-conflicts-in-sb (tn sb offset &key (size 1))
45 (declare (type tn tn) (type finite-sb sb) (type index offset size))
46 (let ((confs (tn-global-conflicts tn))
48 (sb-conflicts (finite-sb-conflicts sb))
49 (sb-always-live (finite-sb-always-live sb)))
50 (macrolet ((do-offsets (&body body)
52 for offset upfrom offset
53 thereis (progn ,@body))))
57 (let ((loc-live (svref sb-always-live offset)))
58 (dotimes (i (ir2-block-count *component-being-compiled*))
59 (when (/= (sbit loc-live i) 0)
62 ;; TN is global, iterate over the blocks TN is live in.
63 (do ((conf confs (global-conflicts-next-tnwise conf)))
66 (let* ((block (global-conflicts-block conf))
67 (num (ir2-block-number block)))
68 (if (eq (global-conflicts-kind conf) :live)
70 (let ((loc-live (svref sb-always-live offset)))
71 (when (/= (sbit loc-live num) 0)
72 (return-from offset-conflicts-in-sb offset))))
74 (let ((loc-confs (svref sb-conflicts offset)))
75 (when (/= (sbit (svref loc-confs num)
76 (global-conflicts-number conf))
78 (return-from offset-conflicts-in-sb offset))))))))
81 (and (/= (sbit (svref (svref sb-conflicts offset)
82 (ir2-block-number (tn-local tn)))
87 ;;; Return true if TN has a conflict in SC at the specified offset.
88 (declaim (ftype (function (tn sc index) (values (or null index) &optional))
90 (defun conflicts-in-sc (tn sc offset)
91 (declare (type tn tn) (type sc sc) (type index offset))
92 (offset-conflicts-in-sb tn (sc-sb sc) offset
93 :size (sc-element-size sc)))
95 ;;; Add TN's conflicts into the conflicts for the location at OFFSET
96 ;;; in SC. We iterate over each location in TN, adding to the
97 ;;; conflicts for that location:
98 ;;; -- If TN is a :COMPONENT TN, then iterate over all the blocks,
99 ;;; setting all of the local conflict bits and the always-live bit.
100 ;;; This records a conflict with any TN that has a LTN number in
101 ;;; the block, as well as with :ALWAYS-LIVE and :ENVIRONMENT TNs.
102 ;;; -- If TN is global, then iterate over the blocks TN is live in. In
103 ;;; addition to setting the always-live bit to represent the conflict
104 ;;; with TNs live throughout the block, we also set bits in the
105 ;;; local conflicts. If TN is :ALWAYS-LIVE in the block, we set all
106 ;;; the bits, otherwise we OR in the local conflict bits.
107 ;;; -- If the TN is local, then we just do the block it is local to,
108 ;;; setting always-live and OR'ing in the local conflicts.
109 (defun add-location-conflicts (tn sc offset optimize)
110 (declare (type tn tn) (type sc sc) (type index offset))
111 (let ((confs (tn-global-conflicts tn))
114 (dotimes (i (sc-element-size sc))
115 (declare (type index i))
116 (let* ((this-offset (+ offset i))
117 (loc-confs (svref (finite-sb-conflicts sb) this-offset))
118 (loc-live (svref (finite-sb-always-live sb) this-offset)))
120 ((eq kind :component)
121 (dotimes (num (ir2-block-count *component-being-compiled*))
122 (declare (type index num))
123 (setf (sbit loc-live num) 1)
124 (set-bit-vector (svref loc-confs num))))
126 (do ((conf confs (global-conflicts-next-tnwise conf)))
128 (let* ((block (global-conflicts-block conf))
129 (num (ir2-block-number block))
130 (local-confs (svref loc-confs num)))
131 (declare (type local-tn-bit-vector local-confs))
132 (setf (sbit loc-live num) 1)
133 (if (eq (global-conflicts-kind conf) :live)
134 (set-bit-vector local-confs)
135 (bit-ior local-confs (global-conflicts-conflicts conf) t)))))
137 (let ((num (ir2-block-number (tn-local tn))))
138 (setf (sbit loc-live num) 1)
139 (bit-ior (the local-tn-bit-vector (svref loc-confs num))
140 (tn-local-conflicts tn) t))))
141 ;; Calculating ALWAYS-LIVE-COUNT is moderately expensive, and
142 ;; currently the information isn't used unless (> SPEED
145 (setf (svref (finite-sb-always-live-count sb) this-offset)
146 (find-location-usage sb this-offset))))))
149 ;; A rought measure of how much a given OFFSET in SB is currently
150 ;; used. Current implementation counts the amount of blocks where the
151 ;; offset has been marked as ALWAYS-LIVE.
152 (defun find-location-usage (sb offset)
153 (declare (optimize speed))
154 (declare (type sb sb) (type index offset))
155 (let* ((always-live (svref (finite-sb-always-live sb) offset)))
156 (declare (simple-bit-vector always-live))
157 (count 1 always-live)))
159 ;;; Return the total number of IR2-BLOCKs in COMPONENT.
160 (defun ir2-block-count (component)
161 (declare (type component component))
162 (do ((2block (block-info (block-next (component-head component)))
163 (ir2-block-next 2block)))
165 (error "What? No ir2 blocks have a non-nil number?"))
166 (when (ir2-block-number 2block)
167 (return (1+ (ir2-block-number 2block))))))
169 ;;; Ensure that the conflicts vectors for each :FINITE SB are large
170 ;;; enough for the number of blocks allocated. Also clear any old
171 ;;; conflicts and reset the current size to the initial size.
172 (defun init-sb-vectors (component)
173 (let ((nblocks (ir2-block-count component)))
174 (dolist (sb *backend-sb-list*)
175 (unless (eq (sb-kind sb) :non-packed)
176 (let* ((conflicts (finite-sb-conflicts sb))
177 (always-live (finite-sb-always-live sb))
178 (always-live-count (finite-sb-always-live-count sb))
179 (max-locs (length conflicts))
180 (last-count (finite-sb-last-block-count sb)))
181 (unless (zerop max-locs)
182 (let ((current-size (length (the simple-vector
183 (svref conflicts 0)))))
185 ((> nblocks current-size)
186 (let ((new-size (max nblocks (* current-size 2))))
187 (declare (type index new-size))
188 (dotimes (i max-locs)
189 (declare (type index i))
190 (let ((new-vec (make-array new-size)))
191 (let ((old (svref conflicts i)))
192 (declare (simple-vector old))
193 (dotimes (j current-size)
194 (declare (type index j))
195 (setf (svref new-vec j)
196 (clear-bit-vector (svref old j)))))
198 (do ((j current-size (1+ j)))
200 (declare (type index j))
201 (setf (svref new-vec j)
202 (make-array local-tn-limit :element-type 'bit
203 :initial-element 0)))
204 (setf (svref conflicts i) new-vec))
205 (setf (svref always-live i)
206 (make-array new-size :element-type 'bit
208 (setf (svref always-live-count i) 0))))
210 (dotimes (i (finite-sb-current-size sb))
211 (declare (type index i))
212 (let ((conf (svref conflicts i)))
213 (declare (simple-vector conf))
214 (dotimes (j last-count)
215 (declare (type index j))
216 (clear-bit-vector (svref conf j))))
217 (clear-bit-vector (svref always-live i))
218 (setf (svref always-live-count i) 0))))))
220 (setf (finite-sb-last-block-count sb) nblocks)
221 (setf (finite-sb-current-size sb) (sb-size sb))
222 (setf (finite-sb-last-offset sb) 0))))))
224 ;;; Expand the :UNBOUNDED SB backing SC by either the initial size or
225 ;;; the SC element size, whichever is larger. If NEEDED-SIZE is
226 ;;; larger, then use that size.
227 (defun grow-sc (sc &optional (needed-size 0))
228 (declare (type sc sc) (type index needed-size))
229 (let* ((sb (sc-sb sc))
230 (size (finite-sb-current-size sb))
231 (align-mask (1- (sc-alignment sc)))
232 (inc (max (finite-sb-size-increment sb)
233 (+ (sc-element-size sc)
234 (- (logandc2 (+ size align-mask) align-mask)
236 (- needed-size size)))
237 (new-size (let ((align-mask (1- (finite-sb-size-alignment sb))))
238 (logandc2 (+ size inc align-mask) align-mask)))
239 (conflicts (finite-sb-conflicts sb))
240 (block-size (if (zerop (length conflicts))
241 (ir2-block-count *component-being-compiled*)
242 (length (the simple-vector (svref conflicts 0)))))
243 (padded-size (ash 1 (integer-length (1- new-size)))))
244 (declare (type index inc new-size padded-size))
245 (aver (eq (sb-kind sb) :unbounded))
247 (when (> padded-size (length conflicts))
248 (let ((new-conf (make-array padded-size)))
249 (replace new-conf conflicts)
250 (do ((i size (1+ i)))
252 (declare (type index i))
253 (let ((loc-confs (make-array block-size)))
254 (dotimes (j block-size)
255 (setf (svref loc-confs j)
256 (make-array local-tn-limit
258 :element-type 'bit)))
259 (setf (svref new-conf i) loc-confs)))
260 (setf (finite-sb-conflicts sb) new-conf))
262 (let ((new-live (make-array padded-size)))
263 (replace new-live (finite-sb-always-live sb))
264 (do ((i size (1+ i)))
266 (setf (svref new-live i)
267 (make-array block-size
269 :element-type 'bit)))
270 (setf (finite-sb-always-live sb) new-live))
272 (let ((new-live-count (make-array padded-size)))
273 (declare (optimize speed)) ;; FILL deftransform
274 (replace new-live-count (finite-sb-always-live-count sb))
275 (fill new-live-count 0 :start size)
276 (setf (finite-sb-always-live-count sb) new-live-count))
278 (let ((new-tns (make-array padded-size :initial-element nil)))
279 (replace new-tns (finite-sb-live-tns sb))
280 (fill (finite-sb-live-tns sb) nil)
281 (setf (finite-sb-live-tns sb) new-tns)))
283 (setf (finite-sb-current-size sb) new-size))
289 ;;; Give someone a hard time because there isn't any load function
290 ;;; defined to move from SRC to DEST.
291 (defun no-load-fun-error (src dest)
292 (let* ((src-sc (tn-sc src))
293 (src-name (sc-name src-sc))
294 (dest-sc (tn-sc dest))
295 (dest-name (sc-name dest-sc)))
296 (cond ((eq (sb-kind (sc-sb src-sc)) :non-packed)
297 (unless (member src-sc (sc-constant-scs dest-sc))
298 (error "loading from an invalid constant SC?~@
299 VM definition inconsistent, try recompiling."))
300 (error "no load function defined to load SC ~S ~
301 from its constant SC ~S"
303 ((member src-sc (sc-alternate-scs dest-sc))
304 (error "no load function defined to load SC ~S from its ~
307 ((member dest-sc (sc-alternate-scs src-sc))
308 (error "no load function defined to save SC ~S in its ~
312 ;; FIXME: "VM definition is inconsistent" shouldn't be a
313 ;; possibility in SBCL.
314 (error "loading to/from SCs that aren't alternates?~@
315 VM definition is inconsistent, try recompiling.")))))
317 ;;; Called when we failed to pack TN. If RESTRICTED is true, then we
318 ;;; are restricted to pack TN in its SC.
319 (defun failed-to-pack-error (tn restricted)
320 (declare (type tn tn))
321 (let* ((sc (tn-sc tn))
322 (scs (cons sc (sc-alternate-scs sc))))
325 (error "failed to pack restricted TN ~S in its SC ~S"
328 (aver (not (find :unbounded scs
329 :key (lambda (x) (sb-kind (sc-sb x))))))
330 (let ((ptype (tn-primitive-type tn)))
333 (aver (member (sc-number sc) (primitive-type-scs ptype)))
334 (error "SC ~S doesn't have any :UNBOUNDED alternate SCs, but is~@
335 a SC for primitive-type ~S."
336 (sc-name sc) (primitive-type-name ptype)))
338 (error "SC ~S doesn't have any :UNBOUNDED alternate SCs."
341 ;;; Return a list of format arguments describing how TN is used in
343 (defun describe-tn-use (loc tn op)
344 (let* ((vop (tn-ref-vop op))
345 (args (vop-args vop))
346 (results (vop-results vop))
347 (name (with-output-to-string (stream)
348 (print-tn-guts tn stream)))
349 (2comp (component-info *component-being-compiled*))
352 ((setq temp (position-in #'tn-ref-across tn args :key #'tn-ref-tn))
353 `("~2D: ~A (~:R argument)" ,loc ,name ,(1+ temp)))
354 ((setq temp (position-in #'tn-ref-across tn results :key #'tn-ref-tn))
355 `("~2D: ~A (~:R result)" ,loc ,name ,(1+ temp)))
356 ((setq temp (position-in #'tn-ref-across tn args :key #'tn-ref-load-tn))
357 `("~2D: ~A (~:R argument load TN)" ,loc ,name ,(1+ temp)))
358 ((setq temp (position-in #'tn-ref-across tn results :key
360 `("~2D: ~A (~:R result load TN)" ,loc ,name ,(1+ temp)))
361 ((setq temp (position-in #'tn-ref-across tn (vop-temps vop)
363 `("~2D: ~A (temporary ~A)" ,loc ,name
364 ,(operand-parse-name (elt (vop-parse-temps
366 (vop-info-name (vop-info vop))))
368 ((eq (tn-kind tn) :component)
369 `("~2D: ~A (component live)" ,loc ,name))
370 ((position-in #'tn-next tn (ir2-component-wired-tns 2comp))
371 `("~2D: ~A (wired)" ,loc ,name))
372 ((position-in #'tn-next tn (ir2-component-restricted-tns 2comp))
373 `("~2D: ~A (restricted)" ,loc ,name))
375 `("~2D: not referenced?" ,loc)))))
377 ;;; If load TN packing fails, try to give a helpful error message. We
378 ;;; find a TN in each location that conflicts, and print it.
379 (defun failed-to-pack-load-tn-error (scs op)
380 (declare (list scs) (type tn-ref op))
384 (let* ((sb (sc-sb sc))
385 (confs (finite-sb-live-tns sb)))
386 (aver (eq (sb-kind sb) :finite))
387 (dolist (el (sc-locations sc))
388 (declare (type index el))
389 (let ((conf (load-tn-conflicts-in-sc op sc el t)))
391 (used (describe-tn-use el conf op))
393 (end (+ el (sc-element-size sc))))
396 (declare (type index i end))
397 (let ((victim (svref confs i)))
399 (used (describe-tn-use el victim op))
402 (multiple-value-bind (arg-p n more-p costs load-scs incon)
403 (get-operand-info op)
404 (declare (ignore costs load-scs))
406 (error "unable to pack a Load-TN in SC ~{~A~#[~^~;, or ~:;,~]~} ~
407 for the ~:R ~:[result~;argument~] to~@
409 ~:[since all SC elements are in use:~:{~%~@?~}~%~;~
410 ~:*but these SC elements are not in use:~% ~S~%Bug?~*~]~
412 Current cost info inconsistent with that in effect at compile ~
413 time. Recompile.~%Compilation order may be incorrect.~]"
414 (mapcar #'sc-name scs)
416 (vop-info-name (vop-info (tn-ref-vop op)))
420 ;;; This is called when none of the SCs that we can load OP into are
421 ;;; allowed by OP's primitive-type.
422 (defun no-load-scs-allowed-by-primitive-type-error (ref)
423 (declare (type tn-ref ref))
424 (let* ((tn (tn-ref-tn ref))
425 (ptype (tn-primitive-type tn)))
426 (multiple-value-bind (arg-p pos more-p costs load-scs incon)
427 (get-operand-info ref)
428 (declare (ignore costs))
430 (error "~S is not valid as the ~:R ~:[result~;argument~] to VOP:~
432 since the TN's primitive type ~S doesn't allow any of the SCs~@
433 allowed by the operand restriction:~% ~S~
435 Current cost info inconsistent with that in effect at compile ~
436 time. Recompile.~%Compilation order may be incorrect.~]"
438 (template-name (vop-info (tn-ref-vop ref)))
439 (primitive-type-name ptype)
440 (mapcar #'sc-name (listify-restrictions load-scs))
445 ;;; Do stuff to note that TN is spilled at VOP for the debugger's benefit.
446 (defun note-spilled-tn (tn vop)
447 (when (and (tn-leaf tn) (vop-save-set vop))
448 (let ((2comp (component-info *component-being-compiled*)))
449 (setf (gethash tn (ir2-component-spilled-tns 2comp)) t)
450 (pushnew tn (gethash vop (ir2-component-spilled-vops 2comp)))))
453 ;;; Make a save TN for TN, pack it, and return it. We copy various
454 ;;; conflict information from the TN so that pack does the right
456 (defun pack-save-tn (tn)
457 (declare (type tn tn))
458 (let ((res (make-tn 0 :save nil nil)))
459 (dolist (alt (sc-alternate-scs (tn-sc tn))
460 (error "no unbounded alternate for SC ~S"
461 (sc-name (tn-sc tn))))
462 (when (eq (sb-kind (sc-sb alt)) :unbounded)
463 (setf (tn-save-tn tn) res)
464 (setf (tn-save-tn res) tn)
465 (setf (tn-sc res) alt)
469 ;;; Find the load function for moving from SRC to DEST and emit a
470 ;;; MOVE-OPERAND VOP with that function as its info arg.
471 (defun emit-operand-load (node block src dest before)
472 (declare (type node node) (type ir2-block block)
473 (type tn src dest) (type (or vop null) before))
474 (emit-load-template node block
475 (template-or-lose 'move-operand)
477 (list (or (svref (sc-move-funs (tn-sc dest))
478 (sc-number (tn-sc src)))
479 (no-load-fun-error src dest)))
483 ;;; Find the preceding use of the VOP NAME in the emit order, starting
484 ;;; with VOP. We must find the VOP in the same IR1 block.
485 (defun reverse-find-vop (name vop)
486 (do* ((block (vop-block vop) (ir2-block-prev block))
487 (last vop (ir2-block-last-vop block)))
489 (aver (eq (ir2-block-block block) (ir2-block-block (vop-block vop))))
490 (do ((current last (vop-prev current)))
492 (when (eq (vop-info-name (vop-info current)) name)
493 (return-from reverse-find-vop current)))))
495 ;;; For TNs that have other than one writer, we save the TN before
496 ;;; each call. If a local call (MOVE-ARGS is :LOCAL-CALL), then we
497 ;;; scan back for the ALLOCATE-FRAME VOP, and emit the save there.
498 ;;; This is necessary because in a self-recursive local call, the
499 ;;; registers holding the current arguments may get trashed by setting
500 ;;; up the call arguments. The ALLOCATE-FRAME VOP marks a place at
501 ;;; which the values are known to be good.
502 (defun save-complex-writer-tn (tn vop)
503 (let ((save (or (tn-save-tn tn)
505 (node (vop-node vop))
506 (block (vop-block vop))
507 (next (vop-next vop)))
508 (when (eq (tn-kind save) :specified-save)
509 (setf (tn-kind save) :save))
510 (aver (eq (tn-kind save) :save))
511 (emit-operand-load node block tn save
512 (if (eq (vop-info-move-args (vop-info vop))
514 (reverse-find-vop 'allocate-frame vop)
516 (emit-operand-load node block save tn next)))
518 ;;; Return a VOP after which is an OK place to save the value of TN.
519 ;;; For correctness, it is only required that this location be after
520 ;;; any possible write and before any possible restore location.
522 ;;; In practice, we return the unique writer VOP, but give up if the
523 ;;; TN is ever read by a VOP with MOVE-ARGS :LOCAL-CALL. This prevents
524 ;;; us from being confused by non-tail local calls.
526 ;;; When looking for writes, we have to ignore uses of MOVE-OPERAND,
527 ;;; since they will correspond to restores that we have already done.
528 (defun find-single-writer (tn)
529 (declare (type tn tn))
530 (do ((write (tn-writes tn) (tn-ref-next write))
534 (do ((read (tn-reads tn) (tn-ref-next read)))
536 (when (eq (vop-info-move-args
543 (unless (eq (vop-info-name (vop-info (tn-ref-vop write)))
545 (when res (return nil))
548 ;;; Try to save TN at a single location. If we succeed, return T,
550 (defun save-single-writer-tn (tn)
551 (declare (type tn tn))
552 (let* ((old-save (tn-save-tn tn))
553 (save (or old-save (pack-save-tn tn)))
554 (writer (find-single-writer tn)))
557 (eq (tn-kind old-save) :specified-save)))
558 (emit-operand-load (vop-node writer) (vop-block writer)
559 tn save (vop-next writer))
560 (setf (tn-kind save) :save-once)
563 ;;; Restore a TN with a :SAVE-ONCE save TN.
564 (defun restore-single-writer-tn (tn vop)
565 (declare (type tn) (type vop vop))
566 (let ((save (tn-save-tn tn)))
567 (aver (eq (tn-kind save) :save-once))
568 (emit-operand-load (vop-node vop) (vop-block vop) save tn (vop-next vop)))
571 ;;; Save a single TN that needs to be saved, choosing save-once if
572 ;;; appropriate. This is also called by SPILL-AND-PACK-LOAD-TN.
573 (defun basic-save-tn (tn vop)
574 (declare (type tn tn) (type vop vop))
575 (let ((save (tn-save-tn tn)))
576 (cond ((and save (eq (tn-kind save) :save-once))
577 (restore-single-writer-tn tn vop))
578 ((save-single-writer-tn tn)
579 (restore-single-writer-tn tn vop))
581 (save-complex-writer-tn tn vop))))
584 ;;; Scan over the VOPs in BLOCK, emiting saving code for TNs noted in
585 ;;; the codegen info that are packed into saved SCs.
586 (defun emit-saves (block)
587 (declare (type ir2-block block))
588 (do ((vop (ir2-block-start-vop block) (vop-next vop)))
590 (when (eq (vop-info-save-p (vop-info vop)) t)
591 (do-live-tns (tn (vop-save-set vop) block)
592 (when (and (sc-save-p (tn-sc tn))
593 (not (eq (tn-kind tn) :component)))
594 (basic-save-tn tn vop)))))
598 ;;;; optimized saving
600 ;;; Save TN if it isn't a single-writer TN that has already been
601 ;;; saved. If multi-write, we insert the save BEFORE the specified
602 ;;; VOP. CONTEXT is a VOP used to tell which node/block to use for the
604 (defun save-if-necessary (tn before context)
605 (declare (type tn tn) (type (or vop null) before) (type vop context))
606 (let ((save (tn-save-tn tn)))
607 (when (eq (tn-kind save) :specified-save)
608 (setf (tn-kind save) :save))
609 (aver (member (tn-kind save) '(:save :save-once)))
610 (unless (eq (tn-kind save) :save-once)
611 (or (save-single-writer-tn tn)
612 (emit-operand-load (vop-node context) (vop-block context)
616 ;;; Load the TN from its save location, allocating one if necessary.
617 ;;; The load is inserted BEFORE the specified VOP. CONTEXT is a VOP
618 ;;; used to tell which node/block to use for the new VOP.
619 (defun restore-tn (tn before context)
620 (declare (type tn tn) (type (or vop null) before) (type vop context))
621 (let ((save (or (tn-save-tn tn) (pack-save-tn tn))))
622 (emit-operand-load (vop-node context) (vop-block context)
626 ;;; Start scanning backward at the end of BLOCK, looking which TNs are
627 ;;; live and looking for places where we have to save. We manipulate
628 ;;; two sets: SAVES and RESTORES.
630 ;;; SAVES is a set of all the TNs that have to be saved because they
631 ;;; are restored after some call. We normally delay saving until the
632 ;;; beginning of the block, but we must save immediately if we see a
633 ;;; write of the saved TN. We also immediately save all TNs and exit
634 ;;; when we see a NOTE-ENVIRONMENT-START VOP, since saves can't be
635 ;;; done before the environment is properly initialized.
637 ;;; RESTORES is a set of all the TNs read (and not written) between
638 ;;; here and the next call, i.e. the set of TNs that must be restored
639 ;;; when we reach the next (earlier) call VOP. Unlike SAVES, this set
640 ;;; is cleared when we do the restoring after a call. Any TNs that
641 ;;; were in RESTORES are moved into SAVES to ensure that they are
642 ;;; saved at some point.
644 ;;; SAVES and RESTORES are represented using both a list and a
645 ;;; bit-vector so that we can quickly iterate and test for membership.
646 ;;; The incoming SAVES and RESTORES args are used for computing these
647 ;;; sets (the initial contents are ignored.)
649 ;;; When we hit a VOP with :COMPUTE-ONLY SAVE-P (an internal error
650 ;;; location), we pretend that all live TNs were read, unless (= speed
651 ;;; 3), in which case we mark all the TNs that are live but not
652 ;;; restored as spilled.
653 (defun optimized-emit-saves-block (block saves restores)
654 (declare (type ir2-block block) (type simple-bit-vector saves restores))
655 (let ((1block (ir2-block-block block))
659 (declare (list saves-list restores-list))
660 (clear-bit-vector saves)
661 (clear-bit-vector restores)
662 (do-live-tns (tn (ir2-block-live-in block) block)
663 (when (and (sc-save-p (tn-sc tn))
664 (not (eq (tn-kind tn) :component)))
665 (let ((num (tn-number tn)))
666 (setf (sbit restores num) 1)
667 (push tn restores-list))))
669 (do ((block block (ir2-block-prev block))
671 ((not (eq (ir2-block-block block) 1block))
672 (aver (not skipping))
673 (dolist (save saves-list)
674 (let ((start (ir2-block-start-vop prev)))
675 (save-if-necessary save start start)))
677 (do ((vop (ir2-block-last-vop block) (vop-prev vop)))
679 (let ((info (vop-info vop)))
680 (case (vop-info-name info)
684 (note-environment-start
685 (aver (not skipping))
686 (dolist (save saves-list)
687 (save-if-necessary save (vop-next vop) vop))
688 (return-from optimized-emit-saves-block block)))
691 (do ((write (vop-results vop) (tn-ref-across write)))
693 (let* ((tn (tn-ref-tn write))
694 (num (tn-number tn)))
695 (unless (zerop (sbit restores num))
696 (setf (sbit restores num) 0)
698 (delete tn restores-list :test #'eq)))
699 (unless (zerop (sbit saves num))
700 (setf (sbit saves num) 0)
701 (save-if-necessary tn (vop-next vop) vop)
703 (delete tn saves-list :test #'eq))))))
705 (macrolet ((save-note-read (tn)
707 (num (tn-number tn)))
708 (when (and (sc-save-p (tn-sc tn))
709 (zerop (sbit restores num))
710 (not (eq (tn-kind tn) :component)))
711 (setf (sbit restores num) 1)
712 (push tn restores-list)))))
714 (case (vop-info-save-p info)
716 (dolist (tn restores-list)
717 (restore-tn tn (vop-next vop) vop)
718 (let ((num (tn-number tn)))
719 (when (zerop (sbit saves num))
721 (setf (sbit saves num) 1))))
722 (setq restores-list nil)
723 (clear-bit-vector restores))
725 (cond ((policy (vop-node vop) (= speed 3))
726 (do-live-tns (tn (vop-save-set vop) block)
727 (when (zerop (sbit restores (tn-number tn)))
728 (note-spilled-tn tn vop))))
730 (do-live-tns (tn (vop-save-set vop) block)
731 (save-note-read tn))))))
733 (if (eq (vop-info-move-args info) :local-call)
735 (do ((read (vop-args vop) (tn-ref-across read)))
737 (save-note-read (tn-ref-tn read))))))))))
739 ;;; This is like EMIT-SAVES, only different. We avoid redundant saving
740 ;;; within the block, and don't restore values that aren't used before
741 ;;; the next call. This function is just the top level loop over the
742 ;;; blocks in the component, which locates blocks that need saving
744 (defun optimized-emit-saves (component)
745 (declare (type component component))
746 (let* ((gtn-count (1+ (ir2-component-global-tn-counter
747 (component-info component))))
748 (saves (make-array gtn-count :element-type 'bit))
749 (restores (make-array gtn-count :element-type 'bit))
750 (block (ir2-block-prev (block-info (component-tail component))))
751 (head (block-info (component-head component))))
753 (when (eq block head) (return))
754 (when (do ((vop (ir2-block-start-vop block) (vop-next vop)))
756 (when (eq (vop-info-save-p (vop-info vop)) t)
758 (setq block (optimized-emit-saves-block block saves restores)))
759 (setq block (ir2-block-prev block)))))
761 ;;; Iterate over the normal TNs, finding the cost of packing on the
762 ;;; stack in units of the number of references. We count all
763 ;;; references as +1, and subtract out REGISTER-SAVE-PENALTY for each
764 ;;; place where we would have to save a register.
765 (defun assign-tn-costs (component)
766 (do-ir2-blocks (block component)
767 (do ((vop (ir2-block-start-vop block) (vop-next vop)))
769 (when (eq (vop-info-save-p (vop-info vop)) t)
770 (do-live-tns (tn (vop-save-set vop) block)
771 (decf (tn-cost tn) *backend-register-save-penalty*)))))
773 (do ((tn (ir2-component-normal-tns (component-info component))
776 (let ((cost (tn-cost tn)))
777 (declare (fixnum cost))
778 (do ((ref (tn-reads tn) (tn-ref-next ref)))
781 (do ((ref (tn-writes tn) (tn-ref-next ref)))
784 (setf (tn-cost tn) cost))))
786 ;;; Iterate over the normal TNs, storing the depth of the deepest loop
787 ;;; that the TN is used in TN-LOOP-DEPTH.
788 (defun assign-tn-depths (component)
790 (do-ir2-blocks (block component)
791 (do ((vop (ir2-block-start-vop block)
794 (flet ((find-all-tns (head-fun)
796 (do ((ref (funcall head-fun vop) (tn-ref-across ref)))
798 (tns (tn-ref-tn ref)))
800 (dolist (tn (nconc (find-all-tns #'vop-args)
801 (find-all-tns #'vop-results)
802 (find-all-tns #'vop-temps)
803 ;; What does "references in this VOP
804 ;; mean"? Probably something that isn't
805 ;; useful in this context, since these
806 ;; TN-REFs are linked with TN-REF-NEXT
807 ;; instead of TN-REF-ACROSS. --JES
809 ;; (find-all-tns #'vop-refs)
811 (setf (tn-loop-depth tn)
812 (max (tn-loop-depth tn)
813 (let* ((ir1-block (ir2-block-block (vop-block vop)))
814 (loop (block-loop ir1-block)))
822 ;;; These variables indicate the last location at which we computed
823 ;;; the Live-TNs. They hold the BLOCK and VOP values that were passed
824 ;;; to COMPUTE-LIVE-TNS.
825 (defvar *live-block*)
828 ;;; If we unpack some TNs, then we mark all affected blocks by
829 ;;; sticking them in this hash-table. This is initially null. We
830 ;;; create the hashtable if we do any unpacking.
831 (defvar *repack-blocks*)
832 (declaim (type list *repack-blocks*))
834 ;;; Set the LIVE-TNS vectors in all :FINITE SBs to represent the TNs
835 ;;; live at the end of BLOCK.
836 (defun init-live-tns (block)
837 (dolist (sb *backend-sb-list*)
838 (when (eq (sb-kind sb) :finite)
839 (fill (finite-sb-live-tns sb) nil)))
841 (do-live-tns (tn (ir2-block-live-in block) block)
842 (let* ((sc (tn-sc tn))
844 (when (eq (sb-kind sb) :finite)
845 ;; KLUDGE: we can have "live" TNs that are neither read
846 ;; to nor written from, due to more aggressive (type-
847 ;; directed) constant propagation. Such TNs will never
848 ;; be assigned an offset nor be in conflict with anything.
850 ;; Ideally, it seems to me we could make sure these TNs
851 ;; are never allocated in the first place in
852 ;; ASSIGN-LAMBDA-VAR-TNS.
854 (do ((offset (tn-offset tn) (1+ offset))
855 (end (+ (tn-offset tn) (sc-element-size sc))))
857 (declare (type index offset end))
858 (setf (svref (finite-sb-live-tns sb) offset) tn))
859 (aver (and (null (tn-reads tn)) (null (tn-writes tn))))))))
861 (setq *live-block* block)
862 (setq *live-vop* (ir2-block-last-vop block))
866 ;;; Set the LIVE-TNs in :FINITE SBs to represent the TNs live
867 ;;; immediately after the evaluation of VOP in BLOCK, excluding
868 ;;; results of the VOP. If VOP is null, then compute the live TNs at
869 ;;; the beginning of the block. Sequential calls on the same block
870 ;;; must be in reverse VOP order.
871 (defun compute-live-tns (block vop)
872 (declare (type ir2-block block) (type vop vop))
873 (unless (eq block *live-block*)
874 (init-live-tns block))
876 (do ((current *live-vop* (vop-prev current)))
878 (do ((res (vop-results vop) (tn-ref-across res)))
880 (let* ((tn (tn-ref-tn res))
883 (when (eq (sb-kind sb) :finite)
884 (do ((offset (tn-offset tn) (1+ offset))
885 (end (+ (tn-offset tn) (sc-element-size sc))))
887 (declare (type index offset end))
888 (setf (svref (finite-sb-live-tns sb) offset) nil))))))
889 (do ((ref (vop-refs current) (tn-ref-next-ref ref)))
891 (let ((ltn (tn-ref-load-tn ref)))
893 (let* ((sc (tn-sc ltn))
895 (when (eq (sb-kind sb) :finite)
896 (let ((tns (finite-sb-live-tns sb)))
897 (do ((offset (tn-offset ltn) (1+ offset))
898 (end (+ (tn-offset ltn) (sc-element-size sc))))
900 (declare (type index offset end))
901 (aver (null (svref tns offset)))))))))
903 (let* ((tn (tn-ref-tn ref))
906 (when (eq (sb-kind sb) :finite)
907 (let ((tns (finite-sb-live-tns sb)))
908 (do ((offset (tn-offset tn) (1+ offset))
909 (end (+ (tn-offset tn) (sc-element-size sc))))
911 (declare (type index offset end))
912 (if (tn-ref-write-p ref)
913 (setf (svref tns offset) nil)
914 (let ((old (svref tns offset)))
915 (aver (or (null old) (eq old tn)))
916 (setf (svref tns offset) tn)))))))))
918 (setq *live-vop* vop)
921 ;;; This is kind of like OFFSET-CONFLICTS-IN-SB, except that it uses
922 ;;; the VOP refs to determine whether a Load-TN for OP could be packed
923 ;;; in the specified location, disregarding conflicts with TNs not
924 ;;; referenced by this VOP. There is a conflict if either:
925 ;;; 1. The reference is a result, and the same location is either:
926 ;;; -- Used by some other result.
927 ;;; -- Used in any way after the reference (exclusive).
928 ;;; 2. The reference is an argument, and the same location is either:
929 ;;; -- Used by some other argument.
930 ;;; -- Used in any way before the reference (exclusive).
932 ;;; In 1 (and 2) above, the first bullet corresponds to result-result
933 ;;; (and argument-argument) conflicts. We need this case because there
934 ;;; aren't any TN-REFs to represent the implicit reading of results or
935 ;;; writing of arguments.
937 ;;; The second bullet corresponds to conflicts with temporaries or
938 ;;; between arguments and results.
940 ;;; We consider both the TN-REF-TN and the TN-REF-LOAD-TN (if any) to
941 ;;; be referenced simultaneously and in the same way. This causes
942 ;;; load-TNs to appear live to the beginning (or end) of the VOP, as
945 ;;; We return a conflicting TN if there is a conflict.
946 (defun load-tn-offset-conflicts-in-sb (op sb offset)
947 (declare (type tn-ref op) (type finite-sb sb) (type index offset))
948 (aver (eq (sb-kind sb) :finite))
949 (let ((vop (tn-ref-vop op)))
950 (labels ((tn-overlaps (tn)
951 (let ((sc (tn-sc tn))
952 (tn-offset (tn-offset tn)))
953 (when (and (eq (sc-sb sc) sb)
954 (<= tn-offset offset)
957 (+ tn-offset (sc-element-size sc)))))
960 (let ((tn (tn-ref-tn ref))
961 (ltn (tn-ref-load-tn ref)))
963 (and ltn (tn-overlaps ltn)))))
965 (do ((ops ops (tn-ref-across ops)))
967 (let ((found (same ops)))
968 (when (and found (not (eq ops op)))
971 (do ((refs refs (tn-ref-next-ref refs)))
973 (let ((found (same refs)))
974 (when found (return found))))))
975 (declare (inline is-op is-ref tn-overlaps))
976 (if (tn-ref-write-p op)
977 (or (is-op (vop-results vop))
978 (is-ref (vop-refs vop) op))
979 (or (is-op (vop-args vop))
980 (is-ref (tn-ref-next-ref op) nil))))))
982 ;;; Iterate over all the elements in the SB that would be allocated by
983 ;;; allocating a TN in SC at Offset, checking for conflict with
984 ;;; load-TNs or other TNs (live in the LIVE-TNS, which must be set
985 ;;; up.) We also return true if there aren't enough locations after
986 ;;; Offset to hold a TN in SC. If Ignore-Live is true, then we ignore
987 ;;; the live-TNs, considering only references within Op's VOP.
989 ;;; We return a conflicting TN, or :OVERFLOW if the TN won't fit.
990 (defun load-tn-conflicts-in-sc (op sc offset ignore-live)
991 (let* ((sb (sc-sb sc))
992 (size (finite-sb-current-size sb)))
993 (do ((i offset (1+ i))
994 (end (+ offset (sc-element-size sc))))
996 (declare (type index i end))
997 (let ((res (or (when (>= i size) :overflow)
998 (and (not ignore-live)
999 (svref (finite-sb-live-tns sb) i))
1000 (load-tn-offset-conflicts-in-sb op sb i))))
1001 (when res (return res))))))
1003 ;;; If a load-TN for OP is targeted to a legal location in SC, then
1004 ;;; return the offset, otherwise return NIL. We see whether the target
1005 ;;; of the operand is packed, and try that location. There isn't any
1006 ;;; need to chain down the target path, since everything is packed
1009 ;;; We require the target to be in SC (and not merely to overlap with
1010 ;;; SC). This prevents SC information from being lost in load TNs (we
1011 ;;; won't pack a load TN in ANY-REG when it is targeted to a
1012 ;;; DESCRIPTOR-REG.) This shouldn't hurt the code as long as all
1013 ;;; relevant overlapping SCs are allowed in the operand SC
1015 (defun find-load-tn-target (op sc)
1016 (declare (inline member))
1017 (let ((target (tn-ref-target op)))
1019 (let* ((tn (tn-ref-tn target))
1020 (loc (tn-offset tn)))
1021 (if (and (eq (tn-sc tn) sc)
1022 (member (the index loc) (sc-locations sc))
1023 (not (load-tn-conflicts-in-sc op sc loc nil)))
1027 ;;; Select a legal location for a load TN for Op in SC. We just
1028 ;;; iterate over the SC's locations. If we can't find a legal
1029 ;;; location, return NIL.
1030 (defun select-load-tn-location (op sc)
1031 (declare (type tn-ref op) (type sc sc))
1033 ;; Check any target location first.
1034 (let ((target (tn-ref-target op)))
1036 (let* ((tn (tn-ref-tn target))
1037 (loc (tn-offset tn)))
1038 (when (and (eq (sc-sb sc) (sc-sb (tn-sc tn)))
1039 (member (the index loc) (sc-locations sc))
1040 (not (load-tn-conflicts-in-sc op sc loc nil)))
1041 (return-from select-load-tn-location loc)))))
1043 (dolist (loc (sc-locations sc) nil)
1044 (unless (load-tn-conflicts-in-sc op sc loc nil)
1047 (defevent unpack-tn "Unpacked a TN to satisfy operand SC restriction.")
1049 ;;; Make TN's location the same as for its save TN (allocating a save
1050 ;;; TN if necessary.) Delete any save/restore code that has been
1051 ;;; emitted thus far. Mark all blocks containing references as needing
1053 (defun unpack-tn (tn)
1055 (let ((stn (or (tn-save-tn tn)
1056 (pack-save-tn tn))))
1057 (setf (tn-sc tn) (tn-sc stn))
1058 (setf (tn-offset tn) (tn-offset stn))
1060 (do ((ref refs (tn-ref-next ref)))
1062 (let ((vop (tn-ref-vop ref)))
1063 (if (eq (vop-info-name (vop-info vop)) 'move-operand)
1065 (pushnew (vop-block vop) *repack-blocks*))))))
1067 (zot (tn-writes tn))))
1071 (defevent unpack-fallback "Unpacked some operand TN.")
1073 ;;; This is called by PACK-LOAD-TN where there isn't any location free
1074 ;;; that we can pack into. What we do is move some live TN in one of
1075 ;;; the specified SCs to memory, then mark all blocks that reference
1076 ;;; the TN as needing repacking. If we succeed, we throw to UNPACKED-TN.
1077 ;;; If we fail, we return NIL.
1079 ;;; We can unpack any live TN that appears in the NORMAL-TNs list
1080 ;;; (isn't wired or restricted.) We prefer to unpack TNs that are not
1081 ;;; used by the VOP. If we can't find any such TN, then we unpack some
1082 ;;; argument or result TN. The only way we can fail is if all
1083 ;;; locations in SC are used by load-TNs or temporaries in VOP.
1084 (defun unpack-for-load-tn (sc op)
1085 (declare (type sc sc) (type tn-ref op))
1086 (let ((sb (sc-sb sc))
1087 (normal-tns (ir2-component-normal-tns
1088 (component-info *component-being-compiled*)))
1089 (node (vop-node (tn-ref-vop op)))
1091 (flet ((unpack-em (victims)
1092 (pushnew (vop-block (tn-ref-vop op)) *repack-blocks*)
1093 (dolist (victim victims)
1094 (event unpack-tn node)
1096 (throw 'unpacked-tn nil)))
1097 (dolist (loc (sc-locations sc))
1098 (declare (type index loc))
1100 (collect ((victims nil adjoin))
1102 (end (+ loc (sc-element-size sc))))
1104 (declare (type index i end))
1105 (let ((victim (svref (finite-sb-live-tns sb) i)))
1107 (unless (find-in #'tn-next victim normal-tns)
1111 (let ((conf (load-tn-conflicts-in-sc op sc loc t)))
1113 (unpack-em (victims)))
1114 ((eq conf :overflow))
1116 (cond ((find conf (victims))
1117 (setq fallback (victims)))
1118 ((find-in #'tn-next conf normal-tns)
1119 (setq fallback (list conf))))))))))
1122 (event unpack-fallback node)
1123 (unpack-em fallback))))
1127 ;;; Try to pack a load TN in the SCs indicated by Load-SCs. If we run
1128 ;;; out of SCs, then we unpack some TN and try again. We return the
1131 ;;; Note: we allow a Load-TN to be packed in the target location even
1132 ;;; if that location is in a SC not allowed by the primitive type.
1133 ;;; (The SC must still be allowed by the operand restriction.) This
1134 ;;; makes move VOPs more efficient, since we won't do a move from the
1135 ;;; stack into a non-descriptor any-reg through a descriptor argument
1136 ;;; load-TN. This does give targeting some real semantics, making it
1137 ;;; not a pure advisory to pack. It allows pack to do some packing it
1138 ;;; wouldn't have done before.
1139 (defun pack-load-tn (load-scs op)
1140 (declare (type sc-vector load-scs) (type tn-ref op))
1141 (let ((vop (tn-ref-vop op)))
1142 (compute-live-tns (vop-block vop) vop))
1144 (let* ((tn (tn-ref-tn op))
1145 (ptype (tn-primitive-type tn))
1146 (scs (svref load-scs (sc-number (tn-sc tn)))))
1147 (let ((current-scs scs)
1153 (no-load-scs-allowed-by-primitive-type-error op))
1154 (dolist (sc allowed)
1155 (unpack-for-load-tn sc op))
1156 (failed-to-pack-load-tn-error allowed op))
1158 (let* ((sc (svref *backend-sc-numbers* (pop current-scs)))
1159 (target (find-load-tn-target op sc)))
1160 (when (or target (sc-allowed-by-primitive-type sc ptype))
1161 (let ((loc (or target
1162 (select-load-tn-location op sc))))
1164 (let ((res (make-tn 0 :load nil sc)))
1165 (setf (tn-offset res) loc)
1167 (push sc allowed)))))))))
1169 ;;; Scan a list of load-SCs vectors and a list of TN-REFS threaded by
1170 ;;; TN-REF-ACROSS. When we find a reference whose TN doesn't satisfy
1171 ;;; the restriction, we pack a Load-TN and load the operand into it.
1172 ;;; If a load-tn has already been allocated, we can assume that the
1173 ;;; restriction is satisfied.
1174 #!-sb-fluid (declaim (inline check-operand-restrictions))
1175 (defun check-operand-restrictions (scs ops)
1176 (declare (list scs) (type (or tn-ref null) ops))
1178 ;; Check the targeted operands first.
1179 (do ((scs scs (cdr scs))
1180 (op ops (tn-ref-across op)))
1182 (let ((target (tn-ref-target op)))
1184 (let* ((load-tn (tn-ref-load-tn op))
1185 (load-scs (svref (car scs)
1187 (tn-sc (or load-tn (tn-ref-tn op)))))))
1189 (aver (eq load-scs t))
1190 (unless (eq load-scs t)
1191 (setf (tn-ref-load-tn op)
1192 (pack-load-tn (car scs) op))))))))
1194 (do ((scs scs (cdr scs))
1195 (op ops (tn-ref-across op)))
1197 (let ((target (tn-ref-target op)))
1199 (let* ((load-tn (tn-ref-load-tn op))
1200 (load-scs (svref (car scs)
1202 (tn-sc (or load-tn (tn-ref-tn op)))))))
1204 (aver (eq load-scs t))
1205 (unless (eq load-scs t)
1206 (setf (tn-ref-load-tn op)
1207 (pack-load-tn (car scs) op))))))))
1211 ;;; Scan the VOPs in BLOCK, looking for operands whose SC restrictions
1212 ;;; aren't satisfied. We do the results first, since they are
1213 ;;; evaluated later, and our conflict analysis is a backward scan.
1214 (defun pack-load-tns (block)
1216 (let ((*live-block* nil)
1218 (do ((vop (ir2-block-last-vop block) (vop-prev vop)))
1220 (let ((info (vop-info vop)))
1221 (check-operand-restrictions (vop-info-result-load-scs info)
1223 (check-operand-restrictions (vop-info-arg-load-scs info)
1229 ;;; Link the TN-REFS READ and WRITE together using the TN-REF-TARGET
1230 ;;; when this seems like a good idea. Currently we always do, as this
1231 ;;; increases the success of load-TN targeting.
1232 (defun target-if-desirable (read write)
1233 (declare (type tn-ref read write))
1234 ;; As per the comments at the definition of TN-REF-TARGET, read and
1235 ;; write refs are always paired, with TARGET in the read pointing to
1236 ;; the write and vice versa.
1237 (aver (eq (tn-ref-write-p read)
1238 (not (tn-ref-write-p write))))
1239 (setf (tn-ref-target read) write)
1240 (setf (tn-ref-target write) read))
1242 ;;; If TN can be packed into SC so as to honor a preference to TARGET,
1243 ;;; then return the offset to pack at, otherwise return NIL. TARGET
1244 ;;; must be already packed.
1245 (defun check-ok-target (target tn sc)
1246 (declare (type tn target tn) (type sc sc) (inline member))
1247 (let* ((loc (tn-offset target))
1248 (target-sc (tn-sc target))
1249 (target-sb (sc-sb target-sc)))
1250 (declare (type index loc))
1251 ;; We can honor a preference if:
1252 ;; -- TARGET's location is in SC's locations.
1253 ;; -- The element sizes of the two SCs are the same.
1254 ;; -- TN doesn't conflict with target's location.
1255 (if (and (eq target-sb (sc-sb sc))
1256 (or (eq (sb-kind target-sb) :unbounded)
1257 (member loc (sc-locations sc)))
1258 (= (sc-element-size target-sc) (sc-element-size sc))
1259 (not (conflicts-in-sc tn sc loc))
1260 (zerop (mod loc (sc-alignment sc))))
1264 ;;; Scan along the target path from TN, looking at readers or writers.
1265 ;;; When we find a packed TN, return CHECK-OK-TARGET of that TN. If
1266 ;;; there is no target, or if the TN has multiple readers (writers),
1267 ;;; then we return NIL. We also always return NIL after 10 iterations
1268 ;;; to get around potential circularity problems.
1270 ;;; FIXME: (30 minutes of reverse engineering?) It'd be nice to
1271 ;;; rewrite the header comment here to explain the interface and its
1272 ;;; motivation, and move remarks about implementation details (like
1274 (defun find-ok-target-offset (tn sc)
1275 (declare (type tn tn) (type sc sc))
1276 (flet ((frob-slot (slot-fun)
1277 (declare (type function slot-fun))
1280 (declare (type index count))
1282 (let ((refs (funcall slot-fun current)))
1283 (unless (and (plusp count)
1285 (not (tn-ref-next refs)))
1287 (let ((target (tn-ref-target refs)))
1288 (unless target (return nil))
1289 (setq current (tn-ref-tn target))
1290 (when (tn-offset current)
1291 (return (check-ok-target current tn sc)))
1293 (declare (inline frob-slot)) ; until DYNAMIC-EXTENT works
1294 (or (frob-slot #'tn-reads)
1295 (frob-slot #'tn-writes))))
1297 ;;;; location selection
1299 ;;; Select some location for TN in SC, returning the offset if we
1300 ;;; succeed, and NIL if we fail.
1302 ;;; For :UNBOUNDED SCs just find the smallest correctly aligned offset
1303 ;;; where the TN doesn't conflict with the TNs that have already been
1304 ;;; packed. For :FINITE SCs try to pack the TN into the most heavily
1305 ;;; used locations first (as estimated in FIND-LOCATION-USAGE).
1307 ;;; Historically SELECT-LOCATION tried did the opposite and tried to
1308 ;;; distribute the TNs evenly across the available locations. At least
1309 ;;; on register-starved architectures (x86) this seems to be a bad
1310 ;;; strategy. -- JES 2004-09-11
1311 (defun select-location (tn sc &key use-reserved-locs optimize)
1312 (declare (type tn tn) (type sc sc) (inline member))
1313 (let* ((sb (sc-sb sc))
1314 (element-size (sc-element-size sc))
1315 (alignment (sc-alignment sc))
1316 (align-mask (1- alignment))
1317 (size (finite-sb-current-size sb)))
1318 (flet ((attempt-location (start-offset)
1319 (let ((conflict (conflicts-in-sc tn sc start-offset)))
1321 (logandc2 (+ conflict align-mask 1)
1323 (return-from select-location start-offset)))))
1324 (if (eq (sb-kind sb) :unbounded)
1325 (loop with offset = 0
1326 until (> (+ offset element-size) size) do
1327 (setf offset (attempt-location offset)))
1328 (let ((locations (sc-locations sc)))
1331 (stable-sort (copy-list locations) #'>
1332 :key (lambda (location-offset)
1333 (loop for offset from location-offset
1336 (finite-sb-always-live-count sb)
1338 (dolist (offset locations)
1339 (when (or use-reserved-locs
1341 (sc-reserve-locations sc))))
1342 (attempt-location offset))))))))
1344 ;;; If a save TN, return the saved TN, otherwise return TN. This is
1345 ;;; useful for getting the conflicts of a TN that might be a save TN.
1346 (defun original-tn (tn)
1347 (declare (type tn tn))
1348 (if (member (tn-kind tn) '(:save :save-once :specified-save))
1354 ;;; Attempt to pack TN in all possible SCs, first in the SC chosen by
1355 ;;; representation selection, then in the alternate SCs in the order
1356 ;;; they were specified in the SC definition. If the TN-COST is
1357 ;;; negative, then we don't attempt to pack in SCs that must be saved.
1358 ;;; If Restricted, then we can only pack in TN-SC, not in any
1361 ;;; If we are attempting to pack in the SC of the save TN for a TN
1362 ;;; with a :SPECIFIED-SAVE TN, then we pack in that location, instead
1363 ;;; of allocating a new stack location.
1364 (defun pack-tn (tn restricted optimize &key (allow-unbounded-sc t))
1365 (declare (type tn tn))
1366 (let* ((original (original-tn tn))
1368 (alternates (unless restricted (sc-alternate-scs fsc)))
1369 (save (tn-save-tn tn))
1372 (eq (tn-kind save) :specified-save))
1374 (do ((sc fsc (pop alternates)))
1376 (failed-to-pack-error tn restricted))
1377 (unless (or allow-unbounded-sc
1378 (neq (sb-kind (sc-sb sc)) :unbounded))
1380 (when (eq sc specified-save-sc)
1381 (unless (tn-offset save)
1382 (pack-tn save nil optimize))
1383 (setf (tn-offset tn) (tn-offset save))
1384 (setf (tn-sc tn) (tn-sc save))
1386 (when (or restricted
1387 (not (and (minusp (tn-cost tn)) (sc-save-p sc))))
1388 (let ((loc (or (find-ok-target-offset original sc)
1389 (select-location original sc)
1391 (select-location original sc :use-reserved-locs t))
1392 (when (eq (sb-kind (sc-sb sc)) :unbounded)
1394 (or (select-location original sc)
1395 (error "failed to pack after growing SC?"))))))
1397 (add-location-conflicts original sc loc optimize)
1398 (setf (tn-sc tn) sc)
1399 (setf (tn-offset tn) loc)
1403 ;;; Pack a wired TN, checking that the offset is in bounds for the SB,
1404 ;;; and that the TN doesn't conflict with some other TN already packed
1405 ;;; in that location. If the TN is wired to a location beyond the end
1406 ;;; of a :UNBOUNDED SB, then grow the SB enough to hold the TN.
1408 ;;; ### Checking for conflicts is disabled for :SPECIFIED-SAVE TNs.
1409 ;;; This is kind of a hack to make specifying wired stack save
1410 ;;; locations for local call arguments (such as OLD-FP) work, since
1411 ;;; the caller and callee OLD-FP save locations may conflict when the
1412 ;;; save locations don't really (due to being in different frames.)
1413 (defun pack-wired-tn (tn optimize)
1414 (declare (type tn tn))
1415 (let* ((sc (tn-sc tn))
1417 (offset (tn-offset tn))
1418 (end (+ offset (sc-element-size sc)))
1419 (original (original-tn tn)))
1420 (when (> end (finite-sb-current-size sb))
1421 (unless (eq (sb-kind sb) :unbounded)
1422 (error "~S is wired to a location that is out of bounds." tn))
1425 ;; For non-x86 ports the presence of a save-tn associated with a
1426 ;; tn is used to identify the old-fp and return-pc tns. It depends
1427 ;; on the old-fp and return-pc being passed in registers.
1429 (when (and (not (eq (tn-kind tn) :specified-save))
1430 (conflicts-in-sc original sc offset))
1431 (error "~S is wired to a location that it conflicts with." tn))
1433 ;; Use the above check, but only print a verbose warning. This can
1434 ;; be helpful for debugging the x86 port.
1436 (when (and (not (eq (tn-kind tn) :specified-save))
1437 (conflicts-in-sc original sc offset))
1438 (format t "~&* Pack-wired-tn possible conflict:~% ~
1439 tn: ~S; tn-kind: ~S~% ~
1441 sb: ~S; sb-name: ~S; sb-kind: ~S~% ~
1442 offset: ~S; end: ~S~% ~
1444 tn-save-tn: ~S; tn-kind of tn-save-tn: ~S~%"
1446 sb (sb-name sb) (sb-kind sb)
1449 (tn-save-tn tn) (tn-kind (tn-save-tn tn))))
1451 ;; On the x86 ports the old-fp and return-pc are often passed on
1452 ;; the stack so the above hack for the other ports does not always
1453 ;; work. Here the old-fp and return-pc tns are identified by being
1454 ;; on the stack in their standard save locations.
1456 (when (and (not (eq (tn-kind tn) :specified-save))
1457 (not (and (string= (sb-name sb) "STACK")
1460 (conflicts-in-sc original sc offset))
1461 (error "~S is wired to location ~D in SC ~A of kind ~S that it conflicts with."
1462 tn offset sc (tn-kind tn)))
1464 (add-location-conflicts original sc offset optimize)))
1466 (defevent repack-block "Repacked a block due to TN unpacking.")
1468 ;;; KLUDGE: Prior to SBCL version 0.8.9.xx, this function was known as
1469 ;;; PACK-BEFORE-GC-HOOK, but was non-functional since approximately
1470 ;;; version 0.8.3.xx since the removal of GC hooks from the system.
1471 ;;; This currently (as of 2004-04-12) runs now after every call to
1472 ;;; PACK, rather than -- as was originally intended -- once per GC
1473 ;;; cycle; this is probably non-optimal, and might require tuning,
1474 ;;; maybe to be called when the data structures exceed a certain size,
1475 ;;; or maybe once every N times. The KLUDGE is that this rewrite has
1476 ;;; done nothing to improve the reentrance or threadsafety of the
1477 ;;; compiler; it still fails to be callable from several threads at
1480 ;;; Brief experiments indicate that during a compilation cycle this
1481 ;;; causes about 10% more consing, and takes about 1%-2% more time.
1483 ;;; -- CSR, 2004-04-12
1484 (defun clean-up-pack-structures ()
1485 (dolist (sb *backend-sb-list*)
1486 (unless (eq (sb-kind sb) :non-packed)
1487 (let ((size (sb-size sb)))
1488 (fill (finite-sb-always-live sb) nil)
1489 (setf (finite-sb-always-live sb)
1493 ;; The cross-compiler isn't very good at
1494 ;; dumping specialized arrays, so we delay
1495 ;; construction of this SIMPLE-BIT-VECTOR
1497 #+sb-xc (make-array 0 :element-type 'bit)))
1498 (setf (finite-sb-always-live-count sb)
1503 #+sb-xc (make-array 0 :element-type 'fixnum)))
1505 (fill (finite-sb-conflicts sb) nil)
1506 (setf (finite-sb-conflicts sb)
1507 (make-array size :initial-element '#()))
1509 (fill (finite-sb-live-tns sb) nil)
1510 (setf (finite-sb-live-tns sb)
1511 (make-array size :initial-element nil))))))
1513 (defun tn-lexical-depth (tn)
1514 (let ((path t)) ; dummy initial value
1515 (labels ((path (lambda)
1516 (nreverse (loop while lambda
1518 do (setf lambda (lambda-parent lambda)))))
1519 (register-scope (lambda)
1520 (let ((new-path (path lambda)))
1521 (setf path (if (eql path t)
1524 0 (mismatch path new-path))))))
1526 (do ((ref ref (tn-ref-next ref)))
1528 (binding* ((node (vop-node (tn-ref-vop ref))
1530 (register-scope (lexenv-lambda
1531 (node-lexenv node)))))))
1532 (walk-tn-refs (tn-reads tn))
1533 (walk-tn-refs (tn-writes tn))
1535 most-positive-fixnum
1538 (defun pack (component)
1540 (let ((optimize nil)
1541 (2comp (component-info component)))
1542 (init-sb-vectors component)
1544 ;; Determine whether we want to do more expensive packing by
1545 ;; checking whether any blocks in the component have (> SPEED
1548 ;; FIXME: This means that a declaration can have a minor
1549 ;; effect even outside its scope, and as the packing is done
1550 ;; component-globally it'd be tricky to use strict scoping. I
1551 ;; think this is still acceptable since it's just a tradeoff
1552 ;; between compilation speed and allocation quality and
1553 ;; doesn't affect the semantics of the generated code in any
1554 ;; way. -- JES 2004-10-06
1555 (do-ir2-blocks (block component)
1556 (when (policy (block-last (ir2-block-block block))
1557 (> speed compilation-speed))
1561 ;; Call the target functions.
1562 (do-ir2-blocks (block component)
1563 (do ((vop (ir2-block-start-vop block) (vop-next vop)))
1565 (let ((target-fun (vop-info-target-fun (vop-info vop))))
1567 (funcall target-fun vop)))))
1569 ;; Pack wired TNs first.
1570 (do ((tn (ir2-component-wired-tns 2comp) (tn-next tn)))
1572 (pack-wired-tn tn optimize))
1574 ;; Pack restricted component TNs.
1575 (do ((tn (ir2-component-restricted-tns 2comp) (tn-next tn)))
1577 (when (eq (tn-kind tn) :component)
1578 (pack-tn tn t optimize)))
1580 ;; Pack other restricted TNs.
1581 (do ((tn (ir2-component-restricted-tns 2comp) (tn-next tn)))
1583 (unless (tn-offset tn)
1584 (pack-tn tn t optimize)))
1586 ;; Assign costs to normal TNs so we know which ones should
1587 ;; always be packed on the stack.
1588 (when *pack-assign-costs*
1589 (assign-tn-costs component)
1590 (assign-tn-depths component))
1592 ;; Allocate normal TNs, starting with the TNs that are used
1593 ;; in deep loops. Only allocate in finite SCs (i.e. not on
1596 (do-ir2-blocks (block component)
1597 (let ((ltns (ir2-block-local-tns block)))
1598 (do ((i (1- (ir2-block-local-tn-count block)) (1- i)))
1600 (declare (fixnum i))
1601 (let ((tn (svref ltns i)))
1602 (unless (or (null tn)
1605 ;; If loop analysis has been disabled we might as
1606 ;; well revert to the old behaviour of just
1607 ;; packing TNs linearly as they appear.
1608 (unless *loop-analyze*
1609 (pack-tn tn nil optimize :allow-unbounded-sc nil))
1611 (dolist (tn (stable-sort (tns)
1614 ((> (tn-loop-depth a)
1617 ((= (tn-loop-depth a)
1619 (> (tn-cost a) (tn-cost b)))
1621 (unless (tn-offset tn)
1622 (pack-tn tn nil optimize :allow-unbounded-sc nil))))
1624 ;; Pack any leftover normal TNs that could not be allocated
1625 ;; to finite SCs, or TNs that do not appear in any local TN
1626 ;; map (e.g. :MORE TNs). Since we'll likely be allocating
1627 ;; on the stack, first allocate TNs that are associated with
1628 ;; code at shallow lexical depths: this will allocate long
1629 ;; live ranges (i.e. TNs with more conflicts) first, and
1630 ;; hopefully minimise stack fragmentation.
1632 ;; Collect in reverse order to give priority to older TNs.
1633 (let ((contiguous-tns '())
1635 (do ((tn (ir2-component-normal-tns 2comp) (tn-next tn)))
1637 (unless (tn-offset tn)
1638 (let ((key (cons tn (tn-lexical-depth tn))))
1639 (if (memq (tn-kind tn) '(:environment :debug-environment
1641 (push key contiguous-tns)
1643 (flet ((pack-tns (tns)
1644 (dolist (tn (stable-sort tns #'< :key #'cdr))
1645 (let ((tn (car tn)))
1646 (unless (tn-offset tn)
1647 (pack-tn tn nil optimize))))))
1648 ;; first pack TNs that are known to have simple
1649 ;; live ranges (contiguous lexical scopes)
1650 (pack-tns contiguous-tns)
1653 ;; Do load TN packing and emit saves.
1654 (let ((*repack-blocks* nil))
1655 (cond ((and optimize *pack-optimize-saves*)
1656 (optimized-emit-saves component)
1657 (do-ir2-blocks (block component)
1658 (pack-load-tns block)))
1660 (do-ir2-blocks (block component)
1662 (pack-load-tns block))))
1664 (unless *repack-blocks* (return))
1665 (let ((orpb *repack-blocks*))
1666 (setq *repack-blocks* nil)
1667 (dolist (block orpb)
1668 (event repack-block)
1669 (pack-load-tns block)))))
1672 (clean-up-pack-structures)))