X-Git-Url: http://repo.macrolet.net/gitweb/?a=blobdiff_plain;f=src%2Fruntime%2Fgencgc.c;h=88de20f40602f5547273485531d8f00bf1b6e47f;hb=1479483c5f40fc470053da0fc5cd8e42fc77676e;hp=ca7fa4713f1e150858df7ee11f0df8a35999d7ac;hpb=6e6670a5c26b3594a0eaa8da59db75b48e0db878;p=sbcl.git diff --git a/src/runtime/gencgc.c b/src/runtime/gencgc.c index ca7fa47..88de20f 100644 --- a/src/runtime/gencgc.c +++ b/src/runtime/gencgc.c @@ -1,5 +1,5 @@ /* - * GENerational Conservative Garbage Collector for SBCL x86 + * GENerational Conservative Garbage Collector for SBCL */ /* @@ -43,9 +43,16 @@ #include "thread.h" #include "genesis/vector.h" #include "genesis/weak-pointer.h" +#include "genesis/fdefn.h" #include "genesis/simple-fun.h" #include "save.h" #include "genesis/hash-table.h" +#include "genesis/instance.h" +#include "genesis/layout.h" + +#ifdef LUTEX_WIDETAG +#include "genesis/lutex.h" +#endif /* forward declarations */ page_index_t gc_find_freeish_pages(long *restart_page_ptr, long nbytes, @@ -145,6 +152,9 @@ unsigned long auto_gc_trigger = 0; generation_index_t from_space; generation_index_t new_space; +/* Set to 1 when in GC */ +boolean gc_active_p = 0; + /* should the GC be conservative on stack. If false (only right before * saving a core), don't scan the stack / mark pages dont_move. */ static boolean conservative_stack = 1; @@ -158,12 +168,6 @@ struct page page_table[NUM_PAGES]; * is needed. */ static void *heap_base = NULL; -#if N_WORD_BITS == 32 - #define SIMPLE_ARRAY_WORD_WIDETAG SIMPLE_ARRAY_UNSIGNED_BYTE_32_WIDETAG -#elif N_WORD_BITS == 64 - #define SIMPLE_ARRAY_WORD_WIDETAG SIMPLE_ARRAY_UNSIGNED_BYTE_64_WIDETAG -#endif - /* Calculate the start address for the given page number. */ inline void * page_address(page_index_t page_num) @@ -232,6 +236,14 @@ struct generation { * prevent a GC when a large number of new live objects have been * added, in which case a GC could be a waste of time */ double min_av_mem_age; + + /* A linked list of lutex structures in this generation, used for + * implementing lutex finalization. */ +#ifdef LUTEX_WIDETAG + struct lutex *lutexes; +#else + void *lutexes; +#endif }; /* an array of generation structures. There needs to be one more @@ -299,7 +311,7 @@ count_generation_pages(generation_index_t generation) long count = 0; for (i = 0; i < last_free_page; i++) - if ((page_table[i].allocated != 0) + if ((page_table[i].allocated != FREE_PAGE_FLAG) && (page_table[i].gen == generation)) count++; return count; @@ -312,7 +324,8 @@ count_dont_move_pages(void) page_index_t i; long count = 0; for (i = 0; i < last_free_page; i++) { - if ((page_table[i].allocated != 0) && (page_table[i].dont_move != 0)) { + if ((page_table[i].allocated != FREE_PAGE_FLAG) + && (page_table[i].dont_move != 0)) { ++count; } } @@ -328,7 +341,8 @@ count_generation_bytes_allocated (generation_index_t gen) page_index_t i; long result = 0; for (i = 0; i < last_free_page; i++) { - if ((page_table[i].allocated != 0) && (page_table[i].gen == gen)) + if ((page_table[i].allocated != FREE_PAGE_FLAG) + && (page_table[i].gen == gen)) result += page_table[i].bytes_used; } return result; @@ -346,15 +360,20 @@ gen_av_mem_age(generation_index_t gen) / ((double)generations[gen].bytes_allocated); } -void fpu_save(int *); /* defined in x86-assem.S */ -void fpu_restore(int *); /* defined in x86-assem.S */ /* The verbose argument controls how much to print: 0 for normal * level of detail; 1 for debugging. */ static void print_generation_stats(int verbose) /* FIXME: should take FILE argument */ { generation_index_t i, gens; - int fpu_state[27]; + +#if defined(LISP_FEATURE_X86) || defined(LISP_FEATURE_X86_64) +#define FPU_STATE_SIZE 27 + int fpu_state[FPU_STATE_SIZE]; +#elif defined(LISP_FEATURE_PPC) +#define FPU_STATE_SIZE 32 + long long fpu_state[FPU_STATE_SIZE]; +#endif /* This code uses the FP instructions which may be set up for Lisp * so they need to be saved and reset for C. */ @@ -368,7 +387,7 @@ print_generation_stats(int verbose) /* FIXME: should take FILE argument */ /* Print the heap stats. */ fprintf(stderr, - " Gen Boxed Unboxed LB LUB !move Alloc Waste Trig WP GCs Mem-age\n"); + " Gen StaPg UbSta LaSta LUbSt Boxed Unboxed LB LUB !move Alloc Waste Trig WP GCs Mem-age\n"); for (i = 0; i < gens; i++) { page_index_t j; @@ -403,13 +422,19 @@ print_generation_stats(int verbose) /* FIXME: should take FILE argument */ gc_assert(generations[i].bytes_allocated == count_generation_bytes_allocated(i)); fprintf(stderr, - " %1d: %5ld %5ld %5ld %5ld %5ld %8ld %5ld %8ld %4ld %3d %7.4f\n", + " %1d: %5ld %5ld %5ld %5ld %5ld %5ld %5ld %5ld %5ld %8ld %5ld %8ld %4ld %3d %7.4f\n", i, - boxed_cnt, unboxed_cnt, large_boxed_cnt, large_unboxed_cnt, + generations[i].alloc_start_page, + generations[i].alloc_unboxed_start_page, + generations[i].alloc_large_start_page, + generations[i].alloc_large_unboxed_start_page, + boxed_cnt, + unboxed_cnt, + large_boxed_cnt, + large_unboxed_cnt, pinned_cnt, generations[i].bytes_allocated, - (count_generation_pages(i)*PAGE_BYTES - - generations[i].bytes_allocated), + (count_generation_pages(i)*PAGE_BYTES - generations[i].bytes_allocated), generations[i].gc_trigger, count_write_protect_generation_pages(i), generations[i].num_gc, @@ -421,7 +446,9 @@ print_generation_stats(int verbose) /* FIXME: should take FILE argument */ } +#if defined(LISP_FEATURE_X86) || defined(LISP_FEATURE_X86_64) void fast_bzero(void*, size_t); /* in -assem.S */ +#endif /* Zero the pages from START to END (inclusive), but use mmap/munmap instead * if zeroing it ourselves, i.e. in practice give the memory back to the @@ -454,7 +481,12 @@ zero_pages(page_index_t start, page_index_t end) { if (start > end) return; +#if defined(LISP_FEATURE_X86) || defined(LISP_FEATURE_X86_64) fast_bzero(page_address(start), PAGE_BYTES*(1+end-start)); +#else + bzero(page_address(start), PAGE_BYTES*(1+end-start)); +#endif + } /* Zero the pages from START to END (inclusive), except for those @@ -476,7 +508,6 @@ zero_dirty_pages(page_index_t start, page_index_t end) { page_table[i].need_to_zero = 1; } } -' /* @@ -563,6 +594,7 @@ gc_alloc_new_region(long nbytes, int unboxed, struct alloc_region *alloc_region) page_index_t last_page; long bytes_found; page_index_t i; + int ret; /* FSHOW((stderr, @@ -574,7 +606,8 @@ gc_alloc_new_region(long nbytes, int unboxed, struct alloc_region *alloc_region) gc_assert((alloc_region->first_page == 0) && (alloc_region->last_page == -1) && (alloc_region->free_pointer == alloc_region->end_addr)); - thread_mutex_lock(&free_pages_lock); + ret = thread_mutex_lock(&free_pages_lock); + gc_assert(ret == 0); if (unboxed) { first_page = generations[gc_alloc_generation].alloc_unboxed_start_page; @@ -632,11 +665,11 @@ gc_alloc_new_region(long nbytes, int unboxed, struct alloc_region *alloc_region) /* Bump up last_free_page. */ if (last_page+1 > last_free_page) { last_free_page = last_page+1; - SetSymbolValue(ALLOCATION_POINTER, - (lispobj)(((char *)heap_base) + last_free_page*PAGE_BYTES), - 0); + /* do we only want to call this on special occasions? like for boxed_region? */ + set_alloc_pointer((lispobj)(((char *)heap_base) + last_free_page*PAGE_BYTES)); } - thread_mutex_unlock(&free_pages_lock); + ret = thread_mutex_unlock(&free_pages_lock); + gc_assert(ret == 0); /* we can do this after releasing free_pages_lock */ if (gencgc_zero_check) { @@ -778,6 +811,7 @@ gc_alloc_update_page_tables(int unboxed, struct alloc_region *alloc_region) long orig_first_page_bytes_used; long region_size; long byte_cnt; + int ret; first_page = alloc_region->first_page; @@ -788,7 +822,8 @@ gc_alloc_update_page_tables(int unboxed, struct alloc_region *alloc_region) next_page = first_page+1; - thread_mutex_lock(&free_pages_lock); + ret = thread_mutex_lock(&free_pages_lock); + gc_assert(ret == 0); if (alloc_region->free_pointer != alloc_region->start_addr) { /* some bytes were allocated in the region */ orig_first_page_bytes_used = page_table[first_page].bytes_used; @@ -892,7 +927,9 @@ gc_alloc_update_page_tables(int unboxed, struct alloc_region *alloc_region) page_table[next_page].allocated = FREE_PAGE_FLAG; next_page++; } - thread_mutex_unlock(&free_pages_lock); + ret = thread_mutex_unlock(&free_pages_lock); + gc_assert(ret == 0); + /* alloc_region is per-thread, we're ok to do this unlocked */ gc_set_region_empty(alloc_region); } @@ -910,8 +947,10 @@ gc_alloc_large(long nbytes, int unboxed, struct alloc_region *alloc_region) int more; long bytes_used; page_index_t next_page; + int ret; - thread_mutex_lock(&free_pages_lock); + ret = thread_mutex_lock(&free_pages_lock); + gc_assert(ret == 0); if (unboxed) { first_page = @@ -1009,10 +1048,10 @@ gc_alloc_large(long nbytes, int unboxed, struct alloc_region *alloc_region) /* Bump up last_free_page */ if (last_page+1 > last_free_page) { last_free_page = last_page+1; - SetSymbolValue(ALLOCATION_POINTER, - (lispobj)(((char *)heap_base) + last_free_page*PAGE_BYTES),0); + set_alloc_pointer((lispobj)(((char *)heap_base) + last_free_page*PAGE_BYTES)); } - thread_mutex_unlock(&free_pages_lock); + ret = thread_mutex_unlock(&free_pages_lock); + gc_assert(ret == 0); #ifdef READ_PROTECT_FREE_PAGES os_protect(page_address(first_page), @@ -1027,6 +1066,32 @@ gc_alloc_large(long nbytes, int unboxed, struct alloc_region *alloc_region) static page_index_t gencgc_alloc_start_page = -1; +void +gc_heap_exhausted_error_or_lose (long available, long requested) +{ + /* Write basic information before doing anything else: if we don't + * call to lisp this is a must, and even if we do there is always the + * danger that we bounce back here before the error has been handled, + * or indeed even printed. + */ + fprintf(stderr, "Heap exhausted during %s: %ld bytes available, %ld requested.\n", + gc_active_p ? "garbage collection" : "allocation", available, requested); + if (gc_active_p || (available == 0)) { + /* If we are in GC, or totally out of memory there is no way + * to sanely transfer control to the lisp-side of things. + */ + print_generation_stats(1); + lose("Heap exhausted, game over."); + } + else { + /* FIXME: assert free_pages_lock held */ + thread_mutex_unlock(&free_pages_lock); + funcall2(SymbolFunction(HEAP_EXHAUSTED_ERROR), + make_fixnum(available), make_fixnum(requested)); + lose("HEAP-EXHAUSTED-ERROR fell through"); + } +} + page_index_t gc_find_freeish_pages(page_index_t *restart_page_ptr, long nbytes, int unboxed) { @@ -1069,13 +1134,8 @@ gc_find_freeish_pages(page_index_t *restart_page_ptr, long nbytes, int unboxed) first_page++; } - if (first_page >= NUM_PAGES) { - fprintf(stderr, - "Argh! gc_find_free_space failed (first_page), nbytes=%ld.\n", - nbytes); - print_generation_stats(1); - lose("\n"); - } + if (first_page >= NUM_PAGES) + gc_heap_exhausted_error_or_lose(0, nbytes); gc_assert(page_table[first_page].write_protected == 0); @@ -1100,13 +1160,9 @@ gc_find_freeish_pages(page_index_t *restart_page_ptr, long nbytes, int unboxed) } while ((restart_page < NUM_PAGES) && (bytes_found < nbytes)); /* Check for a failure */ - if ((restart_page >= NUM_PAGES) && (bytes_found < nbytes)) { - fprintf(stderr, - "Argh! gc_find_freeish_pages failed (restart_page), nbytes=%ld.\n", - nbytes); - print_generation_stats(1); - lose("\n"); - } + if ((restart_page >= NUM_PAGES) && (bytes_found < nbytes)) + gc_heap_exhausted_error_or_lose(bytes_found, nbytes); + *restart_page_ptr=first_page; return last_page; @@ -1796,219 +1852,176 @@ trans_unboxed_large(lispobj object) /* - * vector-like objects + * Lutexes. Using the normal finalization machinery for finalizing + * lutexes is tricky, since the finalization depends on working lutexes. + * So we track the lutexes in the GC and finalize them manually. */ +#if defined(LUTEX_WIDETAG) -/* FIXME: What does this mean? */ -int gencgc_hash = 1; +/* + * Start tracking LUTEX in the GC, by adding it to the linked list of + * lutexes in the nursery generation. The caller is responsible for + * locking, and GCs must be inhibited until the registration is + * complete. + */ +void +gencgc_register_lutex (struct lutex *lutex) { + int index = find_page_index(lutex); + generation_index_t gen; + struct lutex *head; -static long -scav_vector(lispobj *where, lispobj object) -{ - unsigned long kv_length; - lispobj *kv_vector; - unsigned long length = 0; /* (0 = dummy to stop GCC warning) */ - struct hash_table *hash_table; - lispobj empty_symbol; - unsigned long *index_vector = NULL; /* (NULL = dummy to stop GCC warning) */ - unsigned long *next_vector = NULL; /* (NULL = dummy to stop GCC warning) */ - unsigned long *hash_vector = NULL; /* (NULL = dummy to stop GCC warning) */ - lispobj weak_p_obj; - unsigned long next_vector_length = 0; - - /* FIXME: A comment explaining this would be nice. It looks as - * though SB-VM:VECTOR-VALID-HASHING-SUBTYPE is set for EQ-based - * hash tables in the Lisp HASH-TABLE code, and nowhere else. */ - if (HeaderValue(object) != subtype_VectorValidHashing) - return 1; + /* This lutex is in static space, so we don't need to worry about + * finalizing it. + */ + if (index == -1) + return; - if (!gencgc_hash) { - /* This is set for backward compatibility. FIXME: Do we need - * this any more? */ - *where = - (subtype_VectorMustRehash<= 0); + gc_assert(gen < NUM_GENERATIONS); - /* Scavenge element 0, which may be a hash-table structure. */ - scavenge(where+2, 1); - if (!is_lisp_pointer(where[2])) { - lose("no pointer at %x in hash table\n", where[2]); - } - hash_table = (struct hash_table *)native_pointer(where[2]); - /*FSHOW((stderr,"/hash_table = %x\n", hash_table));*/ - if (widetag_of(hash_table->header) != INSTANCE_HEADER_WIDETAG) { - lose("hash table not instance (%x at %x)\n", - hash_table->header, - hash_table); - } + head = generations[gen].lutexes; - /* Scavenge element 1, which should be some internal symbol that - * the hash table code reserves for marking empty slots. */ - scavenge(where+3, 1); - if (!is_lisp_pointer(where[3])) { - lose("not empty-hash-table-slot symbol pointer: %x\n", where[3]); - } - empty_symbol = where[3]; - /* fprintf(stderr,"* empty_symbol = %x\n", empty_symbol);*/ - if (widetag_of(*(lispobj *)native_pointer(empty_symbol)) != - SYMBOL_HEADER_WIDETAG) { - lose("not a symbol where empty-hash-table-slot symbol expected: %x\n", - *(lispobj *)native_pointer(empty_symbol)); - } + lutex->gen = gen; + lutex->next = head; + lutex->prev = NULL; + if (head) + head->prev = lutex; + generations[gen].lutexes = lutex; +} - /* Scavenge hash table, which will fix the positions of the other - * needed objects. */ - scavenge((lispobj *)hash_table, - sizeof(struct hash_table) / sizeof(lispobj)); +/* + * Stop tracking LUTEX in the GC by removing it from the appropriate + * linked lists. This will only be called during GC, so no locking is + * needed. + */ +void +gencgc_unregister_lutex (struct lutex *lutex) { + if (lutex->prev) { + lutex->prev->next = lutex->next; + } else { + generations[lutex->gen].lutexes = lutex->next; + } - /* Cross-check the kv_vector. */ - if (where != (lispobj *)native_pointer(hash_table->table)) { - lose("hash_table table!=this table %x\n", hash_table->table); + if (lutex->next) { + lutex->next->prev = lutex->prev; } - /* WEAK-P */ - weak_p_obj = hash_table->weak_p; + lutex->next = NULL; + lutex->prev = NULL; + lutex->gen = -1; +} - /* index vector */ - { - lispobj index_vector_obj = hash_table->index_vector; - - if (is_lisp_pointer(index_vector_obj) && - (widetag_of(*(lispobj *)native_pointer(index_vector_obj)) == - SIMPLE_ARRAY_WORD_WIDETAG)) { - index_vector = - ((unsigned long *)native_pointer(index_vector_obj)) + 2; - /*FSHOW((stderr, "/index_vector = %x\n",index_vector));*/ - length = fixnum_value(((lispobj *)native_pointer(index_vector_obj))[1]); - /*FSHOW((stderr, "/length = %d\n", length));*/ - } else { - lose("invalid index_vector %x\n", index_vector_obj); - } +/* + * Mark all lutexes in generation GEN as not live. + */ +static void +unmark_lutexes (generation_index_t gen) { + struct lutex *lutex = generations[gen].lutexes; + + while (lutex) { + lutex->live = 0; + lutex = lutex->next; } +} - /* next vector */ - { - lispobj next_vector_obj = hash_table->next_vector; - - if (is_lisp_pointer(next_vector_obj) && - (widetag_of(*(lispobj *)native_pointer(next_vector_obj)) == - SIMPLE_ARRAY_WORD_WIDETAG)) { - next_vector = ((unsigned long *)native_pointer(next_vector_obj)) + 2; - /*FSHOW((stderr, "/next_vector = %x\n", next_vector));*/ - next_vector_length = fixnum_value(((lispobj *)native_pointer(next_vector_obj))[1]); - /*FSHOW((stderr, "/next_vector_length = %d\n", next_vector_length));*/ - } else { - lose("invalid next_vector %x\n", next_vector_obj); +/* + * Finalize all lutexes in generation GEN that have not been marked live. + */ +static void +reap_lutexes (generation_index_t gen) { + struct lutex *lutex = generations[gen].lutexes; + + while (lutex) { + struct lutex *next = lutex->next; + if (!lutex->live) { + lutex_destroy(lutex); + gencgc_unregister_lutex(lutex); } + lutex = next; } +} - /* maybe hash vector */ - { - lispobj hash_vector_obj = hash_table->hash_vector; - - if (is_lisp_pointer(hash_vector_obj) && - (widetag_of(*(lispobj *)native_pointer(hash_vector_obj)) == - SIMPLE_ARRAY_WORD_WIDETAG)){ - hash_vector = - ((unsigned long *)native_pointer(hash_vector_obj)) + 2; - /*FSHOW((stderr, "/hash_vector = %x\n", hash_vector));*/ - gc_assert(fixnum_value(((lispobj *)native_pointer(hash_vector_obj))[1]) - == next_vector_length); - } else { - hash_vector = NULL; - /*FSHOW((stderr, "/no hash_vector: %x\n", hash_vector_obj));*/ - } +/* + * Mark LUTEX as live. + */ +static void +mark_lutex (lispobj tagged_lutex) { + struct lutex *lutex = (struct lutex*) native_pointer(tagged_lutex); + + lutex->live = 1; +} + +/* + * Move all lutexes in generation FROM to generation TO. + */ +static void +move_lutexes (generation_index_t from, generation_index_t to) { + struct lutex *tail = generations[from].lutexes; + + /* Nothing to move */ + if (!tail) + return; + + /* Change the generation of the lutexes in FROM. */ + while (tail->next) { + tail->gen = to; + tail = tail->next; } + tail->gen = to; - /* These lengths could be different as the index_vector can be a - * different length from the others, a larger index_vector could help - * reduce collisions. */ - gc_assert(next_vector_length*2 == kv_length); + /* Link the last lutex in the FROM list to the start of the TO list */ + tail->next = generations[to].lutexes; - /* now all set up.. */ + /* And vice versa */ + if (generations[to].lutexes) { + generations[to].lutexes->prev = tail; + } - /* Work through the KV vector. */ - { - long i; - for (i = 1; i < next_vector_length; i++) { - lispobj old_key = kv_vector[2*i]; + /* And update the generations structures to match this */ + generations[to].lutexes = generations[from].lutexes; + generations[from].lutexes = NULL; +} -#if N_WORD_BITS == 32 - unsigned long old_index = (old_key & 0x1fffffff)%length; -#elif N_WORD_BITS == 64 - unsigned long old_index = (old_key & 0x1fffffffffffffff)%length; -#endif +static long +scav_lutex(lispobj *where, lispobj object) +{ + mark_lutex((lispobj) where); - /* Scavenge the key and value. */ - scavenge(&kv_vector[2*i],2); + return CEILING(sizeof(struct lutex)/sizeof(lispobj), 2); +} - /* Check whether the key has moved and is EQ based. */ - { - lispobj new_key = kv_vector[2*i]; -#if N_WORD_BITS == 32 - unsigned long new_index = (new_key & 0x1fffffff)%length; -#elif N_WORD_BITS == 64 - unsigned long new_index = (new_key & 0x1fffffffffffffff)%length; -#endif +static lispobj +trans_lutex(lispobj object) +{ + struct lutex *lutex = native_pointer(object); + lispobj copied; + size_t words = CEILING(sizeof(struct lutex)/sizeof(lispobj), 2); + gc_assert(is_lisp_pointer(object)); + copied = copy_object(object, words); - if ((old_index != new_index) && - ((!hash_vector) || - (hash_vector[i] == MAGIC_HASH_VECTOR_VALUE)) && - ((new_key != empty_symbol) || - (kv_vector[2*i] != empty_symbol))) { - - /*FSHOW((stderr, - "* EQ key %d moved from %x to %x; index %d to %d\n", - i, old_key, new_key, old_index, new_index));*/ - - if (index_vector[old_index] != 0) { - /*FSHOW((stderr, "/P1 %d\n", index_vector[old_index]));*/ - - /* Unlink the key from the old_index chain. */ - if (index_vector[old_index] == i) { - /*FSHOW((stderr, "/P2a %d\n", next_vector[i]));*/ - index_vector[old_index] = next_vector[i]; - /* Link it into the needing rehash chain. */ - next_vector[i] = fixnum_value(hash_table->needing_rehash); - hash_table->needing_rehash = make_fixnum(i); - /*SHOW("P2");*/ - } else { - unsigned long prior = index_vector[old_index]; - unsigned long next = next_vector[prior]; - - /*FSHOW((stderr, "/P3a %d %d\n", prior, next));*/ - - while (next != 0) { - /*FSHOW((stderr, "/P3b %d %d\n", prior, next));*/ - if (next == i) { - /* Unlink it. */ - next_vector[prior] = next_vector[next]; - /* Link it into the needing rehash - * chain. */ - next_vector[next] = - fixnum_value(hash_table->needing_rehash); - hash_table->needing_rehash = make_fixnum(next); - /*SHOW("/P3");*/ - break; - } - prior = next; - next = next_vector[next]; - } - } - } - } - } - } + /* Update the links, since the lutex moved in memory. */ + if (lutex->next) { + lutex->next->prev = native_pointer(copied); } - return (CEILING(kv_length + 2, 2)); + + if (lutex->prev) { + lutex->prev->next = native_pointer(copied); + } else { + generations[lutex->gen].lutexes = native_pointer(copied); + } + + return copied; } +static long +size_lutex(lispobj *where) +{ + return CEILING(sizeof(struct lutex)/sizeof(lispobj), 2); +} +#endif /* LUTEX_WIDETAG */ /* @@ -2348,6 +2361,9 @@ possibly_valid_dynamic_space_pointer(lispobj *pointer) #endif case SAP_WIDETAG: case WEAK_POINTER_WIDETAG: +#ifdef LUTEX_WIDETAG + case LUTEX_WIDETAG: +#endif break; default: @@ -2370,6 +2386,8 @@ possibly_valid_dynamic_space_pointer(lispobj *pointer) return 1; } +#if defined(LISP_FEATURE_X86) || defined(LISP_FEATURE_X86_64) + /* Adjust large bignum and vector objects. This will adjust the * allocated region if the size has shrunk, and move unboxed objects * into unboxed pages. The pages are not promoted here, and the @@ -2544,6 +2562,8 @@ maybe_adjust_large_object(lispobj *where) return; } +#endif + /* Take a possible pointer to a Lisp object and mark its page in the * page_table so that it will not be relocated during a GC. * @@ -2556,6 +2576,9 @@ maybe_adjust_large_object(lispobj *where) * * It is also assumed that the current gc_alloc() region has been * flushed and the tables updated. */ + +#if defined(LISP_FEATURE_X86) || defined(LISP_FEATURE_X86_64) + static void preserve_pointer(void *addr) { @@ -2671,6 +2694,9 @@ preserve_pointer(void *addr) /* Check that the page is now static. */ gc_assert(page_table[addr_page_index].dont_move != 0); } + +#endif + /* If the given page is not write-protected, then scan it for pointers * to younger generations or the top temp. generation, if no @@ -2980,6 +3006,13 @@ scavenge_newspace_generation(generation_index_t generation) /* Record all new areas now. */ record_new_objects = 2; + /* Give a chance to weak hash tables to make other objects live. + * FIXME: The algorithm implemented here for weak hash table gcing + * is O(W^2+N) as Bruno Haible warns in + * http://www.haible.de/bruno/papers/cs/weak/WeakDatastructures-writeup.html + * see "Implementation 2". */ + scav_weak_hash_tables(); + /* Flush the current regions updating the tables. */ gc_alloc_update_all_page_tables(); @@ -3018,8 +3051,8 @@ scavenge_newspace_generation(generation_index_t generation) if (gencgc_verbose) SHOW("new_areas overflow, doing full scavenge"); - /* Don't need to record new areas that get scavenge anyway - * during scavenge_newspace_generation_one_scan. */ + /* Don't need to record new areas that get scavenged + * anyway during scavenge_newspace_generation_one_scan. */ record_new_objects = 1; scavenge_newspace_generation_one_scan(generation); @@ -3027,6 +3060,8 @@ scavenge_newspace_generation(generation_index_t generation) /* Record all new areas now. */ record_new_objects = 2; + scav_weak_hash_tables(); + /* Flush the current regions updating the tables. */ gc_alloc_update_all_page_tables(); @@ -3041,6 +3076,8 @@ scavenge_newspace_generation(generation_index_t generation) scavenge(page_address(page)+offset, size); } + scav_weak_hash_tables(); + /* Flush the current regions updating the tables. */ gc_alloc_update_all_page_tables(); } @@ -3192,7 +3229,12 @@ print_ptr(lispobj *addr) } #endif -extern long undefined_tramp; +#if defined(LISP_FEATURE_PPC) +extern int closure_tramp; +extern int undefined_tramp; +#else +extern int undefined_tramp; +#endif static void verify_space(lispobj *start, size_t words) @@ -3248,8 +3290,14 @@ verify_space(lispobj *start, size_t words) */ } else { /* Verify that it points to another valid space. */ - if (!to_readonly_space && !to_static_space - && (thing != (unsigned long)&undefined_tramp)) { + if (!to_readonly_space && !to_static_space && +#if defined(LISP_FEATURE_PPC) + !((thing == &closure_tramp) || + (thing == &undefined_tramp)) +#else + thing != (unsigned long)&undefined_tramp +#endif + ) { lose("Ptr %x @ %x sees junk.\n", thing, start); } } @@ -3280,11 +3328,24 @@ verify_space(lispobj *start, size_t words) case SINGLE_FLOAT_WIDETAG: #endif case UNBOUND_MARKER_WIDETAG: - case INSTANCE_HEADER_WIDETAG: case FDEFN_WIDETAG: count = 1; break; + case INSTANCE_HEADER_WIDETAG: + { + lispobj nuntagged; + long ntotal = HeaderValue(thing); + lispobj layout = ((struct instance *)start)->slots[0]; + if (!layout) { + count = 1; + break; + } + nuntagged = ((struct layout *)native_pointer(layout))->n_untagged_slots; + verify_space(start + 1, ntotal - fixnum_value(nuntagged)); + count = ntotal + 1; + break; + } case CODE_HEADER_WIDETAG: { lispobj object = *start; @@ -3417,10 +3478,17 @@ verify_space(lispobj *start, size_t words) #endif case SAP_WIDETAG: case WEAK_POINTER_WIDETAG: +#ifdef LUTEX_WIDETAG + case LUTEX_WIDETAG: +#endif count = (sizetab[widetag_of(*start)])(start); break; default: + FSHOW((stderr, + "/Unhandled widetag 0x%x at 0x%x\n", + widetag_of(*start), start)); + fflush(stderr); gc_abort(); } } @@ -3448,7 +3516,7 @@ verify_gc(void) struct thread *th; for_each_thread(th) { long binding_stack_size = - (lispobj*)SymbolValue(BINDING_STACK_POINTER,th) + (lispobj*)get_binding_stack_pointer(th) - (lispobj*)th->binding_stack_start; verify_space(th->binding_stack_start, binding_stack_size); } @@ -3597,6 +3665,186 @@ write_protect_generation_pages(generation_index_t generation) } } +#if !defined(LISP_FEATURE_X86) && !defined(LISP_FEATURE_X86_64) + +static void +scavenge_control_stack() +{ + unsigned long control_stack_size; + + /* This is going to be a big problem when we try to port threads + * to PPC... CLH */ + struct thread *th = arch_os_get_current_thread(); + lispobj *control_stack = + (lispobj *)(th->control_stack_start); + + control_stack_size = current_control_stack_pointer - control_stack; + scavenge(control_stack, control_stack_size); +} + +/* Scavenging Interrupt Contexts */ + +static int boxed_registers[] = BOXED_REGISTERS; + +static void +scavenge_interrupt_context(os_context_t * context) +{ + int i; + +#ifdef reg_LIP + unsigned long lip; + unsigned long lip_offset; + int lip_register_pair; +#endif + unsigned long pc_code_offset; + +#ifdef ARCH_HAS_LINK_REGISTER + unsigned long lr_code_offset; +#endif +#ifdef ARCH_HAS_NPC_REGISTER + unsigned long npc_code_offset; +#endif + +#ifdef reg_LIP + /* Find the LIP's register pair and calculate it's offset */ + /* before we scavenge the context. */ + + /* + * I (RLT) think this is trying to find the boxed register that is + * closest to the LIP address, without going past it. Usually, it's + * reg_CODE or reg_LRA. But sometimes, nothing can be found. + */ + lip = *os_context_register_addr(context, reg_LIP); + lip_offset = 0x7FFFFFFF; + lip_register_pair = -1; + for (i = 0; i < (sizeof(boxed_registers) / sizeof(int)); i++) { + unsigned long reg; + long offset; + int index; + + index = boxed_registers[i]; + reg = *os_context_register_addr(context, index); + if ((reg & ~((1L<uc_mcontext.gregs[2]. But gregs[2] is REG_nPC. Is + * that what we really want? My guess is that that is not what we + * want, so if lip_register_pair is -1, we don't touch reg_LIP at + * all. But maybe it doesn't really matter if LIP is trashed? + */ + if (lip_register_pair >= 0) { + *os_context_register_addr(context, reg_LIP) = + *os_context_register_addr(context, lip_register_pair) + lip_offset; + } +#endif /* reg_LIP */ + + /* Fix the PC if it was in from space */ + if (from_space_p(*os_context_pc_addr(context))) + *os_context_pc_addr(context) = *os_context_register_addr(context, reg_CODE) + pc_code_offset; + +#ifdef ARCH_HAS_LINK_REGISTER + /* Fix the LR ditto; important if we're being called from + * an assembly routine that expects to return using blr, otherwise + * harmless */ + if (from_space_p(*os_context_lr_addr(context))) + *os_context_lr_addr(context) = + *os_context_register_addr(context, reg_CODE) + lr_code_offset; +#endif + +#ifdef ARCH_HAS_NPC_REGISTER + if (from_space_p(*os_context_npc_addr(context))) + *os_context_npc_addr(context) = *os_context_register_addr(context, reg_CODE) + npc_code_offset; +#endif /* ARCH_HAS_NPC_REGISTER */ +} + +void +scavenge_interrupt_contexts(void) +{ + int i, index; + os_context_t *context; + + struct thread *th=arch_os_get_current_thread(); + + index = fixnum_value(SymbolValue(FREE_INTERRUPT_CONTEXT_INDEX,0)); + +#if defined(DEBUG_PRINT_CONTEXT_INDEX) + printf("Number of active contexts: %d\n", index); +#endif + + for (i = 0; i < index; i++) { + context = th->interrupt_contexts[i]; + scavenge_interrupt_context(context); + } +} + +#endif + +#if defined(LISP_FEATURE_SB_THREAD) +static void +preserve_context_registers (os_context_t *c) +{ + void **ptr; + /* On Darwin the signal context isn't a contiguous block of memory, + * so just preserve_pointering its contents won't be sufficient. + */ +#if defined(LISP_FEATURE_DARWIN) +#if defined LISP_FEATURE_X86 + preserve_pointer((void*)*os_context_register_addr(c,reg_EAX)); + preserve_pointer((void*)*os_context_register_addr(c,reg_ECX)); + preserve_pointer((void*)*os_context_register_addr(c,reg_EDX)); + preserve_pointer((void*)*os_context_register_addr(c,reg_EBX)); + preserve_pointer((void*)*os_context_register_addr(c,reg_ESI)); + preserve_pointer((void*)*os_context_register_addr(c,reg_EDI)); + preserve_pointer((void*)*os_context_pc_addr(c)); +#else + #error "preserve_context_registers needs to be tweaked for non-x86 Darwin" +#endif +#endif + for(ptr = ((void **)(c+1))-1; ptr>=(void **)c; ptr--) { + preserve_pointer(*ptr); + } +} +#endif + /* Garbage collect a generation. If raise is 0 then the remains of the * generation are not raised to the next generation. */ static void @@ -3611,9 +3859,16 @@ garbage_collect_generation(generation_index_t generation, int raise) /* The oldest generation can't be raised. */ gc_assert((generation != HIGHEST_NORMAL_GENERATION) || (raise == 0)); + /* Check if weak hash tables were processed in the previous GC. */ + gc_assert(weak_hash_tables == NULL); + /* Initialize the weak pointer list. */ weak_pointers = NULL; +#ifdef LUTEX_WIDETAG + unmark_lutexes(generation); +#endif + /* When a generation is not being raised it is transported to a * temporary generation (NUM_GENERATIONS), and lowered when * done. Set up this new generation. There should be no pages @@ -3665,6 +3920,7 @@ garbage_collect_generation(generation_index_t generation, int raise) * initiates GC. If you ever call GC from inside an altstack * handler, you will lose. */ +#if defined(LISP_FEATURE_X86) || defined(LISP_FEATURE_X86_64) /* And if we're saving a core, there's no point in being conservative. */ if (conservative_stack) { for_each_thread(th) { @@ -3685,20 +3941,20 @@ garbage_collect_generation(generation_index_t generation, int raise) if (esp1>=(void **)th->control_stack_start && esp1<(void **)th->control_stack_end) { if(esp1=(void **)c; ptr--) { - preserve_pointer(*ptr); - } + preserve_context_registers(c); } } } #else esp = (void **)((void *)&raise); #endif - for (ptr = (void **)th->control_stack_end; ptr > esp; ptr--) { + for (ptr = ((void **)th->control_stack_end)-1; ptr > esp; ptr--) { preserve_pointer(*ptr); } } } +#endif + #ifdef QSHOW if (gencgc_verbose > 1) { long num_dont_move_pages = count_dont_move_pages(); @@ -3711,6 +3967,15 @@ garbage_collect_generation(generation_index_t generation, int raise) /* Scavenge all the rest of the roots. */ +#if !defined(LISP_FEATURE_X86) && !defined(LISP_FEATURE_X86_64) + /* + * If not x86, we need to scavenge the interrupt context(s) and the + * control stack. + */ + scavenge_interrupt_contexts(); + scavenge_control_stack(); +#endif + /* Scavenge the Lisp functions of the interrupt handlers, taking * care to avoid SIG_DFL and SIG_IGN. */ for (i = 0; i < NSIG; i++) { @@ -3724,7 +3989,7 @@ garbage_collect_generation(generation_index_t generation, int raise) { struct thread *th; for_each_thread(th) { - long len= (lispobj *)SymbolValue(BINDING_STACK_POINTER,th) - + long len= (lispobj *)get_binding_stack_pointer(th) - th->binding_stack_start; scavenge((lispobj *) th->binding_stack_start,len); #ifdef LISP_FEATURE_SB_THREAD @@ -3804,6 +4069,7 @@ garbage_collect_generation(generation_index_t generation, int raise) } #endif + scan_weak_hash_tables(); scan_weak_pointers(); /* Flush the current regions, updating the tables. */ @@ -3847,6 +4113,12 @@ garbage_collect_generation(generation_index_t generation, int raise) generations[generation].num_gc = 0; else ++generations[generation].num_gc; + +#ifdef LUTEX_WIDETAG + reap_lutexes(generation); + if (raise) + move_lutexes(generation, generation+1); +#endif } /* Update last_free_page, then SymbolValue(ALLOCATION_POINTER). */ @@ -3862,8 +4134,7 @@ update_dynamic_space_free_pointer(void) last_free_page = last_page+1; - SetSymbolValue(ALLOCATION_POINTER, - (lispobj)(((char *)heap_base) + last_free_page*PAGE_BYTES),0); + set_alloc_pointer((lispobj)(((char *)heap_base) + last_free_page*PAGE_BYTES)); return 0; /* dummy value: return something ... */ } @@ -3885,7 +4156,15 @@ remap_free_pages (page_index_t from, page_index_t to) last_page++; } + /* There's a mysterious Solaris/x86 problem with using mmap + * tricks for memory zeroing. See sbcl-devel thread + * "Re: patch: standalone executable redux". + */ +#if defined(LISP_FEATURE_SUNOS) + zero_pages(first_page, last_page-1); +#else zero_pages_with_mmap(first_page, last_page-1); +#endif first_page = last_page; } @@ -3913,6 +4192,8 @@ collect_garbage(generation_index_t last_gen) FSHOW((stderr, "/entering collect_garbage(%d)\n", last_gen)); + gc_active_p = 1; + if (last_gen > HIGHEST_NORMAL_GENERATION+1) { FSHOW((stderr, "/collect_garbage: last_gen = %d, doing a level 0 GC\n", @@ -4014,7 +4295,9 @@ collect_garbage(generation_index_t last_gen) /* Save the high-water mark before updating last_free_page */ if (last_free_page > high_water_mark) high_water_mark = last_free_page; + update_dynamic_space_free_pointer(); + auto_gc_trigger = bytes_allocated + bytes_consed_between_gcs; if(gencgc_verbose) fprintf(stderr,"Next gc when %ld bytes have been consed\n", @@ -4030,6 +4313,8 @@ collect_garbage(generation_index_t last_gen) high_water_mark = 0; } + gc_active_p = 0; + SHOW("returning from collect_garbage"); } @@ -4104,6 +4389,7 @@ gc_free_heap(void) generations[page].gc_trigger = 2000000; generations[page].num_gc = 0; generations[page].cum_sum_bytes_allocated = 0; + generations[page].lutexes = NULL; } if (gencgc_verbose > 1) @@ -4116,7 +4402,7 @@ gc_free_heap(void) gc_set_region_empty(&unboxed_region); last_free_page = 0; - SetSymbolValue(ALLOCATION_POINTER, (lispobj)((char *)heap_base),0); + set_alloc_pointer((lispobj)((char *)heap_base)); if (verify_after_free_heap) { /* Check whether purify has left any bad pointers. */ @@ -4132,10 +4418,15 @@ gc_init(void) page_index_t i; gc_init_tables(); - scavtab[SIMPLE_VECTOR_WIDETAG] = scav_vector; scavtab[WEAK_POINTER_WIDETAG] = scav_weak_pointer; transother[SIMPLE_ARRAY_WIDETAG] = trans_boxed_large; +#ifdef LUTEX_WIDETAG + scavtab[LUTEX_WIDETAG] = scav_lutex; + transother[LUTEX_WIDETAG] = trans_lutex; + sizetab[LUTEX_WIDETAG] = size_lutex; +#endif + heap_base = (void*)DYNAMIC_SPACE_START; /* Initialize each page structure. */ @@ -4166,6 +4457,7 @@ gc_init(void) generations[i].bytes_consed_between_gc = 2000000; generations[i].trigger_age = 1; generations[i].min_av_mem_age = 0.75; + generations[i].lutexes = NULL; } /* Initialize gc_alloc. */ @@ -4185,7 +4477,7 @@ static void gencgc_pickup_dynamic(void) { page_index_t page = 0; - long alloc_ptr = SymbolValue(ALLOCATION_POINTER,0); + long alloc_ptr = get_alloc_pointer(); lispobj *prev=(lispobj *)page_address(page); generation_index_t gen = PSEUDO_STATIC_GENERATION; @@ -4209,6 +4501,13 @@ gencgc_pickup_dynamic(void) page++; } while ((long)page_address(page) < alloc_ptr); +#ifdef LUTEX_WIDETAG + /* Lutexes have been registered in generation 0 by coreparse, and + * need to be moved to the right one manually. + */ + move_lutexes(0, PSEUDO_STATIC_GENERATION); +#endif + last_free_page = page; generations[gen].bytes_allocated = PAGE_BYTES*page; @@ -4252,16 +4551,18 @@ alloc(long nbytes) void *new_obj; void *new_free_pointer; gc_assert(nbytes>0); + /* Check for alignment allocation problems. */ gc_assert((((unsigned long)region->free_pointer & LOWTAG_MASK) == 0) && ((nbytes & LOWTAG_MASK) == 0)); + #if 0 if(all_threads) /* there are a few places in the C code that allocate data in the * heap before Lisp starts. This is before interrupts are enabled, * so we don't need to check for pseudo-atomic */ #ifdef LISP_FEATURE_SB_THREAD - if(!SymbolValue(PSEUDO_ATOMIC_ATOMIC,th)) { + if(!get_psuedo_atomic_atomic(th)) { register u32 fs; fprintf(stderr, "fatal error in thread 0x%x, tid=%ld\n", th,th->os_thread); @@ -4271,7 +4572,7 @@ alloc(long nbytes) lose("If you see this message before 2004.01.31, mail details to sbcl-devel\n"); } #else - gc_assert(SymbolValue(PSEUDO_ATOMIC_ATOMIC,th)); + gc_assert(get_pseudo_atomic_atomic(th)); #endif #endif @@ -4287,7 +4588,7 @@ alloc(long nbytes) * we should GC in the near future */ if (auto_gc_trigger && bytes_allocated > auto_gc_trigger) { - gc_assert(fixnum_value(SymbolValue(PSEUDO_ATOMIC_ATOMIC,thread))); + gc_assert(get_pseudo_atomic_atomic(thread)); /* Don't flood the system with interrupts if the need to gc is * already noted. This can happen for example when SUB-GC * allocates or after a gc triggered in a WITHOUT-GCING. */ @@ -4296,7 +4597,7 @@ alloc(long nbytes) * section */ SetSymbolValue(GC_PENDING,T,thread); if (SymbolValue(GC_INHIBIT,thread) == NIL) - arch_set_pseudo_atomic_interrupted(0); + set_pseudo_atomic_interrupted(thread); } } new_obj = gc_alloc_with_region(nbytes,0,region,0); @@ -4354,7 +4655,8 @@ gencgc_handle_wp_violation(void* fault_addr) * does this test after the first one has already set wp=0 */ if(page_table[page_index].write_protected_cleared != 1) - lose("fault in heap page not marked as write-protected\n"); + lose("fault in heap page %d not marked as write-protected\nboxed_region.first_page: %d, boxed_region.last_page %d\n", + page_index, boxed_region.first_page, boxed_region.last_page); } /* Don't worry, we can handle it. */ return 1; @@ -4434,13 +4736,17 @@ prepare_for_final_gc () * function being set to the value of the static symbol * SB!VM:RESTART-LISP-FUNCTION */ void -gc_and_save(char *filename) +gc_and_save(char *filename, int prepend_runtime) { - FILE *file = open_core_for_saving(filename); - if (!file) { - perror(filename); - return; - } + FILE *file; + void *runtime_bytes = NULL; + size_t runtime_size; + + file = prepare_to_save(filename, prepend_runtime, &runtime_bytes, + &runtime_size); + if (file == NULL) + return; + conservative_stack = 0; /* The filename might come from Lisp, and be moved by the now @@ -4459,9 +4765,13 @@ gc_and_save(char *filename) gencgc_alloc_start_page = -1; collect_garbage(HIGHEST_NORMAL_GENERATION+1); + if (prepend_runtime) + save_runtime_to_filehandle(file, runtime_bytes, runtime_size); + /* The dumper doesn't know that pages need to be zeroed before use. */ zero_all_free_pages(); - save_to_filehandle(file, filename, SymbolValue(RESTART_LISP_FUNCTION,0)); + save_to_filehandle(file, filename, SymbolValue(RESTART_LISP_FUNCTION,0), + prepend_runtime); /* Oops. Save still managed to fail. Since we've mangled the stack * beyond hope, there's not much we can do. * (beyond FUNCALLing RESTART_LISP_FUNCTION, but I suspect that's