X-Git-Url: http://repo.macrolet.net/gitweb/?a=blobdiff_plain;f=src%2Fruntime%2Fgencgc.c;h=c01841882e1b41ed13661f4bda94d93cb16dd84a;hb=f706a441d7c09cba32701289b63946527fef3c78;hp=b6917abccd6575cfbfd553cc965a91850f70c07a;hpb=2675adcb29d689ee6d270f52658af17f2deeaf77;p=sbcl.git diff --git a/src/runtime/gencgc.c b/src/runtime/gencgc.c index b6917ab..c018418 100644 --- a/src/runtime/gencgc.c +++ b/src/runtime/gencgc.c @@ -27,6 +27,7 @@ #include #include #include +#include #include "runtime.h" #include "sbcl.h" #include "os.h" @@ -43,16 +44,11 @@ #include "genesis/weak-pointer.h" #include "genesis/simple-fun.h" -#ifdef LISP_FEATURE_SB_THREAD -#include -#include /* threading is presently linux-only */ -#endif - /* assembly language stub that executes trap_PendingInterrupt */ void do_pending_interrupt(void); /* forward declarations */ -int gc_find_freeish_pages(int *restart_page_ptr, int nbytes, int unboxed, struct alloc_region *alloc_region); +int gc_find_freeish_pages(int *restart_page_ptr, int nbytes, int unboxed); void gc_set_region_empty(struct alloc_region *region); void gc_alloc_update_all_page_tables(void); static void gencgc_pickup_dynamic(void); @@ -87,7 +83,8 @@ boolean gencgc_unmap_zero = 1; #endif /* the minimum size (in bytes) for a large object*/ -unsigned large_object_size = 4 * 4096; +unsigned large_object_size = 4 * PAGE_BYTES; + /* * debugging @@ -145,12 +142,6 @@ int from_space; int new_space; -/* FIXME: It would be nice to use this symbolic constant instead of - * bare 4096 almost everywhere. We could also use an assertion that - * it's equal to getpagesize(). */ - -#define PAGE_BYTES 4096 - /* An array of page structures is statically allocated. * This helps quickly map between an address its page structure. * NUM_PAGES is set from the size of the dynamic space. */ @@ -165,7 +156,7 @@ static void *heap_base = NULL; inline void * page_address(int page_num) { - return (heap_base + (page_num * 4096)); + return (heap_base + (page_num * PAGE_BYTES)); } /* Find the page index within the page_table for the given @@ -176,7 +167,7 @@ find_page_index(void *addr) int index = addr-heap_base; if (index >= 0) { - index = ((unsigned int)index)/4096; + index = ((unsigned int)index)/PAGE_BYTES; if (index < NUM_PAGES) return (index); } @@ -366,7 +357,7 @@ print_generation_stats(int verbose) /* FIXME: should take FILE argument */ /* Print the heap stats. */ fprintf(stderr, - " Generation Boxed Unboxed LB LUB Alloc Waste Trig WP GCs Mem-age\n"); + " Gen Boxed Unboxed LB LUB !move Alloc Waste Trig WP GCs Mem-age\n"); for (i = 0; i < gens; i++) { int j; @@ -374,6 +365,7 @@ print_generation_stats(int verbose) /* FIXME: should take FILE argument */ int unboxed_cnt = 0; int large_boxed_cnt = 0; int large_unboxed_cnt = 0; + int pinned_cnt=0; for (j = 0; j < last_free_page; j++) if (page_table[j].gen == i) { @@ -386,7 +378,7 @@ print_generation_stats(int verbose) /* FIXME: should take FILE argument */ else boxed_cnt++; } - + if(page_table[j].dont_move) pinned_cnt++; /* Count the number of unboxed pages within the given * generation. */ if (page_table[j].allocated & UNBOXED_PAGE) { @@ -400,11 +392,12 @@ print_generation_stats(int verbose) /* FIXME: should take FILE argument */ gc_assert(generations[i].bytes_allocated == count_generation_bytes_allocated(i)); fprintf(stderr, - " %8d: %5d %5d %5d %5d %8d %5d %8d %4d %3d %7.4f\n", + " %1d: %5d %5d %5d %5d %5d %8d %5d %8d %4d %3d %7.4f\n", i, boxed_cnt, unboxed_cnt, large_boxed_cnt, large_unboxed_cnt, + pinned_cnt, generations[i].bytes_allocated, - (count_generation_pages(i)*4096 + (count_generation_pages(i)*PAGE_BYTES - generations[i].bytes_allocated), generations[i].gc_trigger, count_write_protect_generation_pages(i), @@ -523,9 +516,9 @@ gc_alloc_new_region(int nbytes, int unboxed, struct alloc_region *alloc_region) first_page = generations[gc_alloc_generation].alloc_start_page; } - last_page=gc_find_freeish_pages(&first_page,nbytes,unboxed,alloc_region); - bytes_found=(4096 - page_table[first_page].bytes_used) - + 4096*(last_page-first_page); + last_page=gc_find_freeish_pages(&first_page,nbytes,unboxed); + bytes_found=(PAGE_BYTES - page_table[first_page].bytes_used) + + PAGE_BYTES*(last_page-first_page); /* Set up the alloc_region. */ alloc_region->first_page = first_page; @@ -574,7 +567,7 @@ gc_alloc_new_region(int nbytes, int unboxed, struct alloc_region *alloc_region) if (last_page+1 > last_free_page) { last_free_page = last_page+1; SetSymbolValue(ALLOCATION_POINTER, - (lispobj)(((char *)heap_base) + last_free_page*4096), + (lispobj)(((char *)heap_base) + last_free_page*PAGE_BYTES), 0); } release_spinlock(&free_pages_lock); @@ -647,13 +640,13 @@ add_new_area(int first_page, int offset, int size) gc_abort(); } - new_area_start = 4096*first_page + offset; + new_area_start = PAGE_BYTES*first_page + offset; /* Search backwards for a prior area that this follows from. If found this will save adding a new area. */ for (i = new_areas_index-1, c = 0; (i >= 0) && (c < 8); i--, c++) { unsigned area_end = - 4096*((*new_areas)[i].page) + PAGE_BYTES*((*new_areas)[i].page) + (*new_areas)[i].offset + (*new_areas)[i].size; /*FSHOW((stderr, @@ -705,11 +698,6 @@ gc_alloc_update_page_tables(int unboxed, struct alloc_region *alloc_region) int region_size; int byte_cnt; - /* - FSHOW((stderr, - "/gc_alloc_update_page_tables() to gen %d:\n", - gc_alloc_generation)); - */ first_page = alloc_region->first_page; @@ -748,8 +736,8 @@ gc_alloc_update_page_tables(int unboxed, struct alloc_region *alloc_region) /* Calculate the number of bytes used in this page. This is not * always the number of new bytes, unless it was free. */ more = 0; - if ((bytes_used = (alloc_region->free_pointer - page_address(first_page)))>4096) { - bytes_used = 4096; + if ((bytes_used = (alloc_region->free_pointer - page_address(first_page)))>PAGE_BYTES) { + bytes_used = PAGE_BYTES; more = 1; } page_table[first_page].bytes_used = bytes_used; @@ -775,8 +763,8 @@ gc_alloc_update_page_tables(int unboxed, struct alloc_region *alloc_region) /* Calculate the number of bytes used in this page. */ more = 0; if ((bytes_used = (alloc_region->free_pointer - - page_address(next_page)))>4096) { - bytes_used = 4096; + - page_address(next_page)))>PAGE_BYTES) { + bytes_used = PAGE_BYTES; more = 1; } page_table[next_page].bytes_used = bytes_used; @@ -841,30 +829,6 @@ gc_alloc_large(int nbytes, int unboxed, struct alloc_region *alloc_region) int more; int bytes_used; int next_page; - int large = (nbytes >= large_object_size); - - /* - if (nbytes > 200000) - FSHOW((stderr, "/alloc_large %d\n", nbytes)); - */ - - /* - FSHOW((stderr, - "/gc_alloc_large() for %d bytes from gen %d\n", - nbytes, gc_alloc_generation)); - */ - - /* If the object is small, and there is room in the current region - then allocate it in the current region. */ - if (!large - && ((alloc_region->end_addr-alloc_region->free_pointer) >= nbytes)) - return gc_quick_alloc(nbytes); - - /* To allow the allocation of small objects without the danger of - using a page in the current boxed region, the search starts after - the current boxed free region. XX could probably keep a page - index ahead of the current region and bumped up here to save a - lot of re-scanning. */ get_spinlock(&free_pages_lock,(int) alloc_region); @@ -878,7 +842,7 @@ gc_alloc_large(int nbytes, int unboxed, struct alloc_region *alloc_region) first_page = alloc_region->last_page+1; } - last_page=gc_find_freeish_pages(&first_page,nbytes,unboxed,0); + last_page=gc_find_freeish_pages(&first_page,nbytes,unboxed); gc_assert(first_page > alloc_region->last_page); if (unboxed) @@ -899,7 +863,7 @@ gc_alloc_large(int nbytes, int unboxed, struct alloc_region *alloc_region) page_table[first_page].allocated = BOXED_PAGE; page_table[first_page].gen = gc_alloc_generation; page_table[first_page].first_object_offset = 0; - page_table[first_page].large_object = large; + page_table[first_page].large_object = 1; } if (unboxed) @@ -907,15 +871,15 @@ gc_alloc_large(int nbytes, int unboxed, struct alloc_region *alloc_region) else gc_assert(page_table[first_page].allocated == BOXED_PAGE); gc_assert(page_table[first_page].gen == gc_alloc_generation); - gc_assert(page_table[first_page].large_object == large); + gc_assert(page_table[first_page].large_object == 1); byte_cnt = 0; /* Calc. the number of bytes used in this page. This is not * always the number of new bytes, unless it was free. */ more = 0; - if ((bytes_used = nbytes+orig_first_page_bytes_used) > 4096) { - bytes_used = 4096; + if ((bytes_used = nbytes+orig_first_page_bytes_used) > PAGE_BYTES) { + bytes_used = PAGE_BYTES; more = 1; } page_table[first_page].bytes_used = bytes_used; @@ -934,20 +898,21 @@ gc_alloc_large(int nbytes, int unboxed, struct alloc_region *alloc_region) else page_table[next_page].allocated = BOXED_PAGE; page_table[next_page].gen = gc_alloc_generation; - page_table[next_page].large_object = large; + page_table[next_page].large_object = 1; page_table[next_page].first_object_offset = - orig_first_page_bytes_used - 4096*(next_page-first_page); + orig_first_page_bytes_used - PAGE_BYTES*(next_page-first_page); /* Calculate the number of bytes used in this page. */ more = 0; - if ((bytes_used=(nbytes+orig_first_page_bytes_used)-byte_cnt) > 4096) { - bytes_used = 4096; + if ((bytes_used=(nbytes+orig_first_page_bytes_used)-byte_cnt) > PAGE_BYTES) { + bytes_used = PAGE_BYTES; more = 1; } page_table[next_page].bytes_used = bytes_used; + page_table[next_page].write_protected=0; + page_table[next_page].dont_move=0; byte_cnt += bytes_used; - next_page++; } @@ -964,7 +929,7 @@ gc_alloc_large(int nbytes, int unboxed, struct alloc_region *alloc_region) if (last_page+1 > last_free_page) { last_free_page = last_page+1; SetSymbolValue(ALLOCATION_POINTER, - (lispobj)(((char *)heap_base) + last_free_page*4096),0); + (lispobj)(((char *)heap_base) + last_free_page*PAGE_BYTES),0); } release_spinlock(&free_pages_lock); @@ -972,32 +937,24 @@ gc_alloc_large(int nbytes, int unboxed, struct alloc_region *alloc_region) } int -gc_find_freeish_pages(int *restart_page_ptr, int nbytes, int unboxed, struct alloc_region *alloc_region) +gc_find_freeish_pages(int *restart_page_ptr, int nbytes, int unboxed) { - /* if alloc_region is 0, we assume this is for a potentially large - object */ int first_page; int last_page; int region_size; int restart_page=*restart_page_ptr; int bytes_found; int num_pages; - int large = !alloc_region && (nbytes >= large_object_size); - + int large_p=(nbytes>=large_object_size); gc_assert(free_pages_lock); - /* Search for a contiguous free space of at least nbytes. If it's a - large object then align it on a page boundary by searching for a - free page. */ - /* To allow the allocation of small objects without the danger of - using a page in the current boxed region, the search starts after - the current boxed free region. XX could probably keep a page - index ahead of the current region and bumped up here to save a - lot of re-scanning. */ + /* Search for a contiguous free space of at least nbytes. If it's + * a large object then align it on a page boundary by searching + * for a free page. */ do { first_page = restart_page; - if (large) + if (large_p) while ((first_page < NUM_PAGES) && (page_table[first_page].allocated != FREE_PAGE)) first_page++; @@ -1005,18 +962,15 @@ gc_find_freeish_pages(int *restart_page_ptr, int nbytes, int unboxed, struct all while (first_page < NUM_PAGES) { if(page_table[first_page].allocated == FREE_PAGE) break; - /* I don't know why we need the gen=0 test, but it - * breaks randomly if that's omitted -dan 2003.02.26 - */ if((page_table[first_page].allocated == (unboxed ? UNBOXED_PAGE : BOXED_PAGE)) && (page_table[first_page].large_object == 0) && - (gc_alloc_generation == 0) && (page_table[first_page].gen == gc_alloc_generation) && - (page_table[first_page].bytes_used < (4096-32)) && + (page_table[first_page].bytes_used < (PAGE_BYTES-32)) && (page_table[first_page].write_protected == 0) && - (page_table[first_page].dont_move == 0)) + (page_table[first_page].dont_move == 0)) { break; + } first_page++; } @@ -1031,20 +985,20 @@ gc_find_freeish_pages(int *restart_page_ptr, int nbytes, int unboxed, struct all gc_assert(page_table[first_page].write_protected == 0); last_page = first_page; - bytes_found = 4096 - page_table[first_page].bytes_used; + bytes_found = PAGE_BYTES - page_table[first_page].bytes_used; num_pages = 1; while (((bytes_found < nbytes) - || (alloc_region && (num_pages < 2))) + || (!large_p && (num_pages < 2))) && (last_page < (NUM_PAGES-1)) && (page_table[last_page+1].allocated == FREE_PAGE)) { last_page++; num_pages++; - bytes_found += 4096; + bytes_found += PAGE_BYTES; gc_assert(page_table[last_page].write_protected == 0); } - region_size = (4096 - page_table[first_page].bytes_used) - + 4096*(last_page-first_page); + region_size = (PAGE_BYTES - page_table[first_page].bytes_used) + + PAGE_BYTES*(last_page-first_page); gc_assert(bytes_found == region_size); restart_page = last_page + 1; @@ -1063,8 +1017,7 @@ gc_find_freeish_pages(int *restart_page_ptr, int nbytes, int unboxed, struct all } /* Allocate bytes. All the rest of the special-purpose allocation - * functions will eventually call this (instead of just duplicating - * parts of its code) */ + * functions will eventually call this */ void * gc_alloc_with_region(int nbytes,int unboxed_p, struct alloc_region *my_region, @@ -1072,7 +1025,8 @@ gc_alloc_with_region(int nbytes,int unboxed_p, struct alloc_region *my_region, { void *new_free_pointer; - /* FSHOW((stderr, "/gc_alloc %d\n", nbytes)); */ + if(nbytes>=large_object_size) + return gc_alloc_large(nbytes,unboxed_p,my_region); /* Check whether there is room in the current alloc region. */ new_free_pointer = my_region->free_pointer + nbytes; @@ -1095,48 +1049,18 @@ gc_alloc_with_region(int nbytes,int unboxed_p, struct alloc_region *my_region, return((void *)new_obj); } - /* Else not enough free space in the current region. */ - - /* If there some room left in the current region, enough to be worth - * saving, then allocate a large object. */ - /* FIXME: "32" should be a named parameter. */ - if ((my_region->end_addr-my_region->free_pointer) > 32) - return gc_alloc_large(nbytes, unboxed_p, my_region); + /* Else not enough free space in the current region: retry with a + * new region. */ - /* Else find a new region. */ - - /* Finished with the current region. */ gc_alloc_update_page_tables(unboxed_p, my_region); - - /* Set up a new region. */ gc_alloc_new_region(nbytes, unboxed_p, my_region); - - /* Should now be enough room. */ - - /* Check whether there is room in the current region. */ - new_free_pointer = my_region->free_pointer + nbytes; - - if (new_free_pointer <= my_region->end_addr) { - /* If so then allocate from the current region. */ - void *new_obj = my_region->free_pointer; - my_region->free_pointer = new_free_pointer; - /* Check whether the current region is almost empty. */ - if ((my_region->end_addr - my_region->free_pointer) <= 32) { - /* If so find, finished with the current region. */ - gc_alloc_update_page_tables(unboxed_p, my_region); - - /* Set up a new region. */ - gc_alloc_new_region(32, unboxed_p, my_region); - } - - return((void *)new_obj); - } - - /* shouldn't happen */ - gc_assert(0); - return((void *) NIL); /* dummy value: return something ... */ + return gc_alloc_with_region(nbytes,unboxed_p,my_region,0); } +/* these are only used during GC: all allocation from the mutator calls + * alloc() -> gc_alloc_with_region() with the appropriate per-thread + * region */ + void * gc_general_alloc(int nbytes,int unboxed_p,int quick_p) { @@ -1145,41 +1069,16 @@ gc_general_alloc(int nbytes,int unboxed_p,int quick_p) return gc_alloc_with_region(nbytes,unboxed_p, my_region,quick_p); } - - -static void * -gc_alloc(int nbytes,int unboxed_p) -{ - /* this is the only function that the external interface to - * allocation presently knows how to call: Lisp code will never - * allocate large objects, or to unboxed space, or `quick'ly. - * Any of that stuff will only ever happen inside of GC */ - return gc_general_alloc(nbytes,unboxed_p,0); -} - -/* Allocate space from the boxed_region. If there is not enough free - * space then call gc_alloc to do the job. A pointer to the start of - * the object is returned. */ static inline void * gc_quick_alloc(int nbytes) { return gc_general_alloc(nbytes,ALLOC_BOXED,ALLOC_QUICK); } -/* Allocate space for the possibly large boxed object. If it is a - * large object then do a large alloc else use gc_quick_alloc. Note - * that gc_quick_alloc will eventually fall through to - * gc_general_alloc which may allocate the object in a large way - * anyway, but based on decisions about the free space in the current - * region, not the object size itself */ - static inline void * gc_quick_alloc_large(int nbytes) { - if (nbytes >= large_object_size) - return gc_alloc_large(nbytes, ALLOC_BOXED, &boxed_region); - else - return gc_general_alloc(nbytes,ALLOC_BOXED,ALLOC_QUICK); + return gc_general_alloc(nbytes,ALLOC_BOXED,ALLOC_QUICK); } static inline void * @@ -1194,18 +1093,10 @@ gc_quick_alloc_unboxed(int nbytes) return gc_general_alloc(nbytes,ALLOC_UNBOXED,ALLOC_QUICK); } -/* Allocate space for the object. If it is a large object then do a - * large alloc else allocate from the current region. If there is not - * enough free space then call general gc_alloc_unboxed() to do the job. - * - * A pointer to the start of the object is returned. */ static inline void * gc_quick_alloc_large_unboxed(int nbytes) { - if (nbytes >= large_object_size) - return gc_alloc_large(nbytes,ALLOC_UNBOXED,&unboxed_region); - else - return gc_quick_alloc_unboxed(nbytes); + return gc_general_alloc(nbytes,ALLOC_UNBOXED,ALLOC_QUICK); } /* @@ -1227,7 +1118,6 @@ copy_large_object(lispobj object, int nwords) { int tag; lispobj *new; - lispobj *source, *dest; int first_page; gc_assert(is_lisp_pointer(object)); @@ -1235,7 +1125,7 @@ copy_large_object(lispobj object, int nwords) gc_assert((nwords & 0x01) == 0); - /* Check whether it's a large object. */ + /* Check whether it's in a large object region. */ first_page = find_page_index((void *)object); gc_assert(first_page >= 0); @@ -1258,23 +1148,23 @@ copy_large_object(lispobj object, int nwords) next_page = first_page; remaining_bytes = nwords*4; - while (remaining_bytes > 4096) { + while (remaining_bytes > PAGE_BYTES) { gc_assert(page_table[next_page].gen == from_space); gc_assert(page_table[next_page].allocated == BOXED_PAGE); gc_assert(page_table[next_page].large_object); gc_assert(page_table[next_page].first_object_offset== - -4096*(next_page-first_page)); - gc_assert(page_table[next_page].bytes_used == 4096); + -PAGE_BYTES*(next_page-first_page)); + gc_assert(page_table[next_page].bytes_used == PAGE_BYTES); page_table[next_page].gen = new_space; /* Remove any write-protection. We should be able to rely * on the write-protect flag to avoid redundant calls. */ if (page_table[next_page].write_protected) { - os_protect(page_address(next_page), 4096, OS_VM_PROT_ALL); + os_protect(page_address(next_page), PAGE_BYTES, OS_VM_PROT_ALL); page_table[next_page].write_protected = 0; } - remaining_bytes -= 4096; + remaining_bytes -= PAGE_BYTES; next_page++; } @@ -1295,12 +1185,12 @@ copy_large_object(lispobj object, int nwords) /* Free any remaining pages; needs care. */ next_page++; - while ((old_bytes_used == 4096) && + while ((old_bytes_used == PAGE_BYTES) && (page_table[next_page].gen == from_space) && (page_table[next_page].allocated == BOXED_PAGE) && page_table[next_page].large_object && (page_table[next_page].first_object_offset == - -(next_page - first_page)*4096)) { + -(next_page - first_page)*PAGE_BYTES)) { /* Checks out OK, free the page. Don't need to bother zeroing * pages as this should have been done before shrinking the * object. These pages shouldn't be write-protected as they @@ -1329,17 +1219,7 @@ copy_large_object(lispobj object, int nwords) /* Allocate space. */ new = gc_quick_alloc_large(nwords*4); - dest = new; - source = (lispobj *) native_pointer(object); - - /* Copy the object. */ - while (nwords > 0) { - dest[0] = source[0]; - dest[1] = source[1]; - dest += 2; - source += 2; - nwords -= 2; - } + memcpy(new,native_pointer(object),nwords*4); /* Return Lisp pointer of new object. */ return ((lispobj) new) | tag; @@ -1352,7 +1232,6 @@ copy_unboxed_object(lispobj object, int nwords) { int tag; lispobj *new; - lispobj *source, *dest; gc_assert(is_lisp_pointer(object)); gc_assert(from_space_p(object)); @@ -1364,17 +1243,7 @@ copy_unboxed_object(lispobj object, int nwords) /* Allocate space. */ new = gc_quick_alloc_unboxed(nwords*4); - dest = new; - source = (lispobj *) native_pointer(object); - - /* Copy the object. */ - while (nwords > 0) { - dest[0] = source[0]; - dest[1] = source[1]; - dest += 2; - source += 2; - nwords -= 2; - } + memcpy(new,native_pointer(object),nwords*4); /* Return Lisp pointer of new object. */ return ((lispobj) new) | tag; @@ -1423,18 +1292,18 @@ copy_large_unboxed_object(lispobj object, int nwords) next_page = first_page; remaining_bytes = nwords*4; - while (remaining_bytes > 4096) { + while (remaining_bytes > PAGE_BYTES) { gc_assert(page_table[next_page].gen == from_space); gc_assert((page_table[next_page].allocated == UNBOXED_PAGE) || (page_table[next_page].allocated == BOXED_PAGE)); gc_assert(page_table[next_page].large_object); gc_assert(page_table[next_page].first_object_offset== - -4096*(next_page-first_page)); - gc_assert(page_table[next_page].bytes_used == 4096); + -PAGE_BYTES*(next_page-first_page)); + gc_assert(page_table[next_page].bytes_used == PAGE_BYTES); page_table[next_page].gen = new_space; page_table[next_page].allocated = UNBOXED_PAGE; - remaining_bytes -= 4096; + remaining_bytes -= PAGE_BYTES; next_page++; } @@ -1455,13 +1324,13 @@ copy_large_unboxed_object(lispobj object, int nwords) /* Free any remaining pages; needs care. */ next_page++; - while ((old_bytes_used == 4096) && + while ((old_bytes_used == PAGE_BYTES) && (page_table[next_page].gen == from_space) && ((page_table[next_page].allocated == UNBOXED_PAGE) || (page_table[next_page].allocated == BOXED_PAGE)) && page_table[next_page].large_object && (page_table[next_page].first_object_offset == - -(next_page - first_page)*4096)) { + -(next_page - first_page)*PAGE_BYTES)) { /* Checks out OK, free the page. Don't need to both zeroing * pages as this should have been done before shrinking the * object. These pages shouldn't be write-protected, even if @@ -1734,26 +1603,23 @@ gencgc_apply_code_fixups(struct code *old_code, struct code *new_code) code objects. Check. */ fixups = new_code->constants[0]; - /* It will be 0 or the unbound-marker if there are no fixups, and - * will be an other pointer if it is valid. */ + /* It will be 0 or the unbound-marker if there are no fixups (as + * will be the case if the code object has been purified, for + * example) and will be an other pointer if it is valid. */ if ((fixups == 0) || (fixups == UNBOUND_MARKER_WIDETAG) || !is_lisp_pointer(fixups)) { /* Check for possible errors. */ if (check_code_fixups) sniff_code_object(new_code, displacement); - /*fprintf(stderr,"Fixups for code object not found!?\n"); - fprintf(stderr,"*** Compiled code object at %x: header_words=%d code_words=%d .\n", - new_code, nheader_words, ncode_words); - fprintf(stderr,"*** Const. start = %x; end= %x; Code start = %x; end = %x\n", - constants_start_addr,constants_end_addr, - code_start_addr,code_end_addr);*/ return; } fixups_vector = (struct vector *)native_pointer(fixups); /* Could be pointing to a forwarding pointer. */ + /* FIXME is this always in from_space? if so, could replace this code with + * forwarding_pointer_p/forwarding_pointer_value */ if (is_lisp_pointer(fixups) && (find_page_index((void*)fixups_vector) != -1) && (fixups_vector->header == 0x01)) { @@ -2118,7 +1984,7 @@ search_space(lispobj *start, size_t words, lispobj *pointer) return (NULL); } -static lispobj* +lispobj* search_read_only_space(lispobj *pointer) { lispobj* start = (lispobj*)READ_ONLY_SPACE_START; @@ -2128,7 +1994,7 @@ search_read_only_space(lispobj *pointer) return (search_space(start, (pointer+2)-start, pointer)); } -static lispobj * +lispobj * search_static_space(lispobj *pointer) { lispobj* start = (lispobj*)STATIC_SPACE_START; @@ -2143,7 +2009,7 @@ search_static_space(lispobj *pointer) lispobj * search_dynamic_space(lispobj *pointer) { - int page_index = find_page_index(pointer); + int page_index = find_page_index(pointer); lispobj *start; /* The address may be invalid, so do some checks. */ @@ -2474,21 +2340,21 @@ maybe_adjust_large_object(lispobj *where) next_page = first_page; remaining_bytes = nwords*4; - while (remaining_bytes > 4096) { + while (remaining_bytes > PAGE_BYTES) { gc_assert(page_table[next_page].gen == from_space); gc_assert((page_table[next_page].allocated == BOXED_PAGE) || (page_table[next_page].allocated == UNBOXED_PAGE)); gc_assert(page_table[next_page].large_object); gc_assert(page_table[next_page].first_object_offset == - -4096*(next_page-first_page)); - gc_assert(page_table[next_page].bytes_used == 4096); + -PAGE_BYTES*(next_page-first_page)); + gc_assert(page_table[next_page].bytes_used == PAGE_BYTES); page_table[next_page].allocated = boxed; /* Shouldn't be write-protected at this stage. Essential that the * pages aren't. */ gc_assert(!page_table[next_page].write_protected); - remaining_bytes -= 4096; + remaining_bytes -= PAGE_BYTES; next_page++; } @@ -2510,13 +2376,13 @@ maybe_adjust_large_object(lispobj *where) /* Free any remaining pages; needs care. */ next_page++; - while ((old_bytes_used == 4096) && + while ((old_bytes_used == PAGE_BYTES) && (page_table[next_page].gen == from_space) && ((page_table[next_page].allocated == UNBOXED_PAGE) || (page_table[next_page].allocated == BOXED_PAGE)) && page_table[next_page].large_object && (page_table[next_page].first_object_offset == - -(next_page - first_page)*4096)) { + -(next_page - first_page)*PAGE_BYTES)) { /* It checks out OK, free the page. We don't need to both zeroing * pages as this should have been done before shrinking the * object. These pages shouldn't be write protected as they @@ -2546,11 +2412,8 @@ maybe_adjust_large_object(lispobj *where) * page_table so that it will not be relocated during a GC. * * This involves locating the page it points to, then backing up to - * the first page that has its first object start at offset 0, and - * then marking all pages dont_move from the first until a page that - * ends by being full, or having free gen. - * - * This ensures that objects spanning pages are not broken. + * the start of its region, then marking all pages dont_move from there + * up to the first page that's not full or has a different generation * * It is assumed that all the page static flags have been cleared at * the start of a GC. @@ -2580,10 +2443,8 @@ preserve_pointer(void *addr) /* quick check 2: Check the offset within the page. * - * FIXME: The mask should have a symbolic name, and ideally should - * be derived from page size instead of hardwired to 0xfff. - * (Also fix other uses of 0xfff, elsewhere.) */ - if (((unsigned)addr & 0xfff) > page_table[addr_page_index].bytes_used) + */ + if (((unsigned)addr & (PAGE_BYTES - 1)) > page_table[addr_page_index].bytes_used) return; /* Filter out anything which can't be a pointer to a Lisp object @@ -2594,25 +2455,28 @@ preserve_pointer(void *addr) * a pointer which prevents a page from moving. */ if (!(possibly_valid_dynamic_space_pointer(addr))) return; - first_page = addr_page_index; - /* Work backwards to find a page with a first_object_offset of 0. - * The pages should be contiguous with all bytes used in the same - * gen. Assumes the first_object_offset is negative or zero. */ - - /* this is probably needlessly conservative. The first object in - * the page may not even be the one we were passed a pointer to: - * if this is the case, we will write-protect all the previous - * object's pages too. - */ + /* Find the beginning of the region. Note that there may be + * objects in the region preceding the one that we were passed a + * pointer to: if this is the case, we will write-protect all the + * previous objects' pages too. */ +#if 0 + /* I think this'd work just as well, but without the assertions. + * -dan 2004.01.01 */ + first_page= + find_page_index(page_address(addr_page_index)+ + page_table[addr_page_index].first_object_offset); +#else + first_page = addr_page_index; while (page_table[first_page].first_object_offset != 0) { --first_page; /* Do some checks. */ - gc_assert(page_table[first_page].bytes_used == 4096); + gc_assert(page_table[first_page].bytes_used == PAGE_BYTES); gc_assert(page_table[first_page].gen == from_space); gc_assert(page_table[first_page].allocated == region_allocation); } +#endif /* Adjust any large objects before promotion as they won't be * copied after promotion. */ @@ -2625,7 +2489,7 @@ preserve_pointer(void *addr) if ((page_table[addr_page_index].allocated == FREE_PAGE) || (page_table[addr_page_index].bytes_used == 0) /* Check the offset within the page. */ - || (((unsigned)addr & 0xfff) + || (((unsigned)addr & (PAGE_BYTES - 1)) > page_table[addr_page_index].bytes_used)) { FSHOW((stderr, "weird? ignore ptr 0x%x to freed area of large object\n", @@ -2659,8 +2523,8 @@ preserve_pointer(void *addr) gc_assert(!page_table[i].write_protected); /* Check whether this is the last page in this contiguous block.. */ - if ((page_table[i].bytes_used < 4096) - /* ..or it is 4096 and is the last in the block */ + if ((page_table[i].bytes_used < PAGE_BYTES) + /* ..or it is PAGE_BYTES and is the last in the block */ || (page_table[i+1].allocated == FREE_PAGE) || (page_table[i+1].bytes_used == 0) /* next page free */ || (page_table[i+1].gen != from_space) /* diff. gen */ @@ -2734,7 +2598,7 @@ update_page_write_prot(int page) /*FSHOW((stderr, "/write-protecting page %d gen %d\n", page, gen));*/ os_protect((void *)page_addr, - 4096, + PAGE_BYTES, OS_VM_PROT_READ|OS_VM_PROT_EXECUTE); /* Note the page as protected in the page tables. */ @@ -2747,7 +2611,7 @@ update_page_write_prot(int page) /* Scavenge a generation. * * This will not resolve all pointers when generation is the new - * space, as new objects may be added which are not check here - use + * space, as new objects may be added which are not checked here - use * scavenge_newspace generation. * * Write-protected pages should not have any pointers to the @@ -2792,58 +2656,39 @@ scavenge_generation(int generation) if ((page_table[i].allocated & BOXED_PAGE) && (page_table[i].bytes_used != 0) && (page_table[i].gen == generation)) { - int last_page; + int last_page,j; + int write_protected=1; - /* This should be the start of a contiguous block. */ + /* This should be the start of a region */ gc_assert(page_table[i].first_object_offset == 0); - /* We need to find the full extent of this contiguous - * block in case objects span pages. */ - - /* Now work forward until the end of this contiguous area - * is found. A small area is preferred as there is a - * better chance of its pages being write-protected. */ - for (last_page = i; ; last_page++) - /* Check whether this is the last page in this contiguous - * block. */ - if ((page_table[last_page].bytes_used < 4096) - /* Or it is 4096 and is the last in the block */ + /* Now work forward until the end of the region */ + for (last_page = i; ; last_page++) { + write_protected = + write_protected && page_table[last_page].write_protected; + if ((page_table[last_page].bytes_used < PAGE_BYTES) + /* Or it is PAGE_BYTES and is the last in the block */ || (!(page_table[last_page+1].allocated & BOXED_PAGE)) || (page_table[last_page+1].bytes_used == 0) || (page_table[last_page+1].gen != generation) || (page_table[last_page+1].first_object_offset == 0)) break; - - /* Do a limited check for write_protected pages. If all pages - * are write_protected then there is no need to scavenge. */ - { - int j, all_wp = 1; - for (j = i; j <= last_page; j++) - if (page_table[j].write_protected == 0) { - all_wp = 0; - break; - } -#if !SC_GEN_CK - if (all_wp == 0) -#endif - { - scavenge(page_address(i), (page_table[last_page].bytes_used - + (last_page-i)*4096)/4); - - /* Now scan the pages and write protect those - * that don't have pointers to younger - * generations. */ - if (enable_page_protection) { - for (j = i; j <= last_page; j++) { - num_wp += update_page_write_prot(j); - } - } + } + if (!write_protected) { + scavenge(page_address(i), (page_table[last_page].bytes_used + + (last_page-i)*PAGE_BYTES)/4); + + /* Now scan the pages and write protect those that + * don't have pointers to younger generations. */ + if (enable_page_protection) { + for (j = i; j <= last_page; j++) { + num_wp += update_page_write_prot(j); } + } } i = last_page; } } - if ((gencgc_verbose > 1) && (num_wp != 0)) { FSHOW((stderr, "/write protected %d pages within generation %d\n", @@ -2916,6 +2761,7 @@ scavenge_newspace_generation_one_scan(int generation) * cleared before promotion.) */ || (page_table[i].dont_move == 1))) { int last_page; + int all_wp=1; /* The scavenge will start at the first_object_offset of page i. * @@ -2926,10 +2772,15 @@ scavenge_newspace_generation_one_scan(int generation) * is found. A small area is preferred as there is a * better chance of its pages being write-protected. */ for (last_page = i; ;last_page++) { + /* If all pages are write-protected and movable, + * then no need to scavenge */ + all_wp=all_wp && page_table[last_page].write_protected && + !page_table[last_page].dont_move; + /* Check whether this is the last page in this * contiguous block */ - if ((page_table[last_page].bytes_used < 4096) - /* Or it is 4096 and is the last in the block */ + if ((page_table[last_page].bytes_used < PAGE_BYTES) + /* Or it is PAGE_BYTES and is the last in the block */ || (!(page_table[last_page+1].allocated & BOXED_PAGE)) || (page_table[last_page+1].bytes_used == 0) || (page_table[last_page+1].gen != generation) @@ -2937,41 +2788,20 @@ scavenge_newspace_generation_one_scan(int generation) break; } - /* Do a limited check for write-protected pages. If all - * pages are write-protected then no need to scavenge, - * except if the pages are marked dont_move. */ - { - int j, all_wp = 1; - for (j = i; j <= last_page; j++) - if ((page_table[j].write_protected == 0) - || (page_table[j].dont_move != 0)) { - all_wp = 0; - break; - } - - if (!all_wp) { - int size; - - /* Calculate the size. */ - if (last_page == i) - size = (page_table[last_page].bytes_used - - page_table[i].first_object_offset)/4; - else - size = (page_table[last_page].bytes_used - + (last_page-i)*4096 - - page_table[i].first_object_offset)/4; - - { - new_areas_ignore_page = last_page; - - scavenge(page_address(i) + - page_table[i].first_object_offset, - size); - - } - } + /* Do a limited check for write-protected pages. */ + if (!all_wp) { + int size; + + size = (page_table[last_page].bytes_used + + (last_page-i)*PAGE_BYTES + - page_table[i].first_object_offset)/4; + new_areas_ignore_page = last_page; + + scavenge(page_address(i) + + page_table[i].first_object_offset, + size); + } - i = last_page; } } @@ -2990,7 +2820,7 @@ scavenge_newspace_generation(int generation) struct new_area (*current_new_areas)[] = &new_areas_1; int current_new_areas_index; - /* the new_areas created but the previous scavenge cycle */ + /* the new_areas created by the previous scavenge cycle */ struct new_area (*previous_new_areas)[] = NULL; int previous_new_areas_index; @@ -3125,7 +2955,7 @@ unprotect_oldspace(void) /* Remove any write-protection. We should be able to rely * on the write-protect flag to avoid redundant calls. */ if (page_table[i].write_protected) { - os_protect(page_start, 4096, OS_VM_PROT_ALL); + os_protect(page_start, PAGE_BYTES, OS_VM_PROT_ALL); page_table[i].write_protected = 0; } } @@ -3173,7 +3003,7 @@ free_oldspace(void) void *page_start = (void *)page_address(last_page); if (page_table[last_page].write_protected) { - os_protect(page_start, 4096, OS_VM_PROT_ALL); + os_protect(page_start, PAGE_BYTES, OS_VM_PROT_ALL); page_table[last_page].write_protected = 0; } } @@ -3194,8 +3024,8 @@ free_oldspace(void) page_start = (void *)page_address(first_page); - os_invalidate(page_start, 4096*(last_page-first_page)); - addr = os_validate(page_start, 4096*(last_page-first_page)); + os_invalidate(page_start, PAGE_BYTES*(last_page-first_page)); + addr = os_validate(page_start, PAGE_BYTES*(last_page-first_page)); if (addr == NULL || addr != page_start) { /* Is this an error condition? I couldn't really tell from * the old CMU CL code, which fprintf'ed a message with @@ -3213,7 +3043,7 @@ free_oldspace(void) int *page_start; page_start = (int *)page_address(first_page); - i586_bzero(page_start, 4096*(last_page-first_page)); + i586_bzero(page_start, PAGE_BYTES*(last_page-first_page)); } first_page = last_page; @@ -3316,9 +3146,8 @@ verify_space(lispobj *start, size_t words) } } } else { - if (thing & 0x3) { /* Skip fixnums. FIXME: There should be an - * is_fixnum for this. */ - + if (!(fixnump(thing))) { + /* skip fixnums */ switch(widetag_of(*start)) { /* boxed objects */ @@ -3365,7 +3194,7 @@ verify_space(lispobj *start, size_t words) * there's no byte compiler, but I've got * too much to worry about right now to try * to make sure. -- WHN 2001-10-06 */ - && !(code->trace_table_offset & 0x3) + && fixnump(code->trace_table_offset) /* Only when enabled */ && verify_dynamic_code_check) { FSHOW((stderr, @@ -3514,8 +3343,8 @@ verify_generation(int generation) for (last_page = i; ;last_page++) /* Check whether this is the last page in this contiguous * block. */ - if ((page_table[last_page].bytes_used < 4096) - /* Or it is 4096 and is the last in the block */ + if ((page_table[last_page].bytes_used < PAGE_BYTES) + /* Or it is PAGE_BYTES and is the last in the block */ || (page_table[last_page+1].allocated != region_allocation) || (page_table[last_page+1].bytes_used == 0) || (page_table[last_page+1].gen != generation) @@ -3523,7 +3352,7 @@ verify_generation(int generation) break; verify_space(page_address(i), (page_table[last_page].bytes_used - + (last_page-i)*4096)/4); + + (last_page-i)*PAGE_BYTES)/4); i = last_page; } } @@ -3547,7 +3376,7 @@ verify_zero_fill(void) } } } else { - int free_bytes = 4096 - page_table[page].bytes_used; + int free_bytes = PAGE_BYTES - page_table[page].bytes_used; if (free_bytes > 0) { int *start_addr = (int *)((unsigned)page_address(page) + page_table[page].bytes_used); @@ -3603,7 +3432,7 @@ write_protect_generation_pages(int generation) page_start = (void *)page_address(i); os_protect(page_start, - 4096, + PAGE_BYTES, OS_VM_PROT_READ | OS_VM_PROT_EXECUTE); /* Note the page as protected in the page tables. */ @@ -3720,9 +3549,7 @@ garbage_collect_generation(int generation, int raise) fprintf(stderr, "/non-movable pages due to conservative pointers = %d (%d bytes)\n", num_dont_move_pages, - /* FIXME: 4096 should be symbolic constant here and - * prob'ly elsewhere too. */ - num_dont_move_pages * 4096); + num_dont_move_pages * PAGE_BYTES); } #endif @@ -3888,7 +3715,7 @@ update_x86_dynamic_space_free_pointer(void) last_free_page = last_page+1; SetSymbolValue(ALLOCATION_POINTER, - (lispobj)(((char *)heap_base) + last_free_page*4096),0); + (lispobj)(((char *)heap_base) + last_free_page*PAGE_BYTES),0); return 0; /* dummy value: return something ... */ } @@ -4047,11 +3874,11 @@ gc_free_heap(void) page_start = (void *)page_address(page); /* First, remove any write-protection. */ - os_protect(page_start, 4096, OS_VM_PROT_ALL); + os_protect(page_start, PAGE_BYTES, OS_VM_PROT_ALL); page_table[page].write_protected = 0; - os_invalidate(page_start,4096); - addr = os_validate(page_start,4096); + os_invalidate(page_start,PAGE_BYTES); + addr = os_validate(page_start,PAGE_BYTES); if (addr == NULL || addr != page_start) { lose("gc_free_heap: page moved, 0x%08x ==> 0x%08x", page_start, @@ -4159,32 +3986,35 @@ gc_init(void) /* Pick up the dynamic space from after a core load. * * The ALLOCATION_POINTER points to the end of the dynamic space. - * - * XX A scan is needed to identify the closest first objects for pages. */ + */ + static void gencgc_pickup_dynamic(void) { int page = 0; - int addr = DYNAMIC_SPACE_START; int alloc_ptr = SymbolValue(ALLOCATION_POINTER,0); + lispobj *prev=(lispobj *)page_address(page); - /* Initialize the first region. */ do { + lispobj *first,*ptr= (lispobj *)page_address(page); page_table[page].allocated = BOXED_PAGE; page_table[page].gen = 0; - page_table[page].bytes_used = 4096; + page_table[page].bytes_used = PAGE_BYTES; page_table[page].large_object = 0; + + first=search_space(prev,(ptr+2)-prev,ptr); + if(ptr == first) prev=ptr; page_table[page].first_object_offset = - (void *)DYNAMIC_SPACE_START - page_address(page); - addr += 4096; + (void *)prev - page_address(page); page++; - } while (addr < alloc_ptr); + } while (page_address(page) < alloc_ptr); - generations[0].bytes_allocated = 4096*page; - bytes_allocated = 4096*page; + generations[0].bytes_allocated = PAGE_BYTES*page; + bytes_allocated = PAGE_BYTES*page; } + void gc_initialize_pointers(void) { @@ -4228,9 +4058,9 @@ alloc(int nbytes) fprintf(stderr, "fatal error in thread 0x%x, pid=%d\n", th,getpid()); __asm__("movl %fs,%0" : "=r" (fs) : ); - fprintf(stderr, "fs is %x, th->tls_cookie=%x (should be identical)\n", + fprintf(stderr, "fs is %x, th->tls_cookie=%x \n", debug_get_fs(),th->tls_cookie); - lose("If you see this message before 2003.12.01, mail details to sbcl-devel\n"); + lose("If you see this message before 2004.01.31, mail details to sbcl-devel\n"); } #else gc_assert(SymbolValue(PSEUDO_ATOMIC_ATOMIC,th));