X-Git-Url: http://repo.macrolet.net/gitweb/?a=blobdiff_plain;f=src%2Fruntime%2Fgencgc.c;h=a6891d0149f3ff6a9df1fe9d8ac324373b46acc7;hb=7fb597b585fc715537ea644f7d84440eca217ca1;hp=973152b3a5871918b86fccba7d083f141b21dfa3;hpb=e6c3f1d3e416bd473a9e476e6f95235c783794a3;p=sbcl.git diff --git a/src/runtime/gencgc.c b/src/runtime/gencgc.c index 973152b..a6891d0 100644 --- a/src/runtime/gencgc.c +++ b/src/runtime/gencgc.c @@ -61,7 +61,7 @@ page_index_t gc_find_freeish_pages(long *restart_page_ptr, long nbytes, */ enum { HIGHEST_NORMAL_GENERATION = 5, - PSEUDO_STATIC_GENERATION = 5, + PSEUDO_STATIC_GENERATION, SCRATCH_GENERATION, NUM_GENERATIONS }; @@ -94,8 +94,6 @@ unsigned long large_object_size = 4 * PAGE_BYTES; * debugging */ - - /* the verbosity level. All non-error messages are disabled at level 0; * and only a few rare messages are printed at level 1. */ #ifdef QSHOW @@ -135,6 +133,12 @@ boolean gencgc_enable_verify_zero_fill = 0; /* Should we check that free pages are zero filled during gc_free_heap * called after Lisp PURIFY? */ boolean gencgc_zero_check_during_free_heap = 0; + +/* When loading a core, don't do a full scan of the memory for the + * memory region boundaries. (Set to true by coreparse.c if the core + * contained a pagetable entry). + */ +boolean gencgc_partial_pickup = 0; /* * GC structures and variables @@ -341,7 +345,7 @@ count_generation_bytes_allocated (generation_index_t gen) /* Return the average age of the memory in a generation. */ static double -gen_av_mem_age(int gen) +gen_av_mem_age(generation_index_t gen) { if (generations[gen].bytes_allocated == 0) return 0.0; @@ -358,7 +362,7 @@ void fpu_restore(int *); /* defined in x86-assem.S */ static void print_generation_stats(int verbose) /* FIXME: should take FILE argument */ { - int i, gens; + generation_index_t i, gens; int fpu_state[27]; /* This code uses the FP instructions which may be set up for Lisp @@ -598,13 +602,12 @@ gc_alloc_new_region(long nbytes, int unboxed, struct alloc_region *alloc_region) * (long) in code like this, so that it is less likely to * break randomly when running on a machine with different * word sizes. -- WHN 19991129 */ - lose("The new region at %x is not zero.", p); + lose("The new region at %x is not zero.\n", p); } + } } } -} - /* If the record_new_objects flag is 2 then all new regions created * are recorded. * @@ -952,8 +955,10 @@ gc_alloc_large(long nbytes, int unboxed, struct alloc_region *alloc_region) return((void *)(page_address(first_page)+orig_first_page_bytes_used)); } -long -gc_find_freeish_pages(long *restart_page_ptr, long nbytes, int unboxed) +static page_index_t gencgc_alloc_start_page = -1; + +page_index_t +gc_find_freeish_pages(page_index_t *restart_page_ptr, long nbytes, int unboxed) { page_index_t first_page; page_index_t last_page; @@ -968,6 +973,10 @@ gc_find_freeish_pages(long *restart_page_ptr, long nbytes, int unboxed) * a large object then align it on a page boundary by searching * for a free page. */ + if (gencgc_alloc_start_page != -1) { + restart_page = gencgc_alloc_start_page; + } + do { first_page = restart_page; if (large_p) @@ -995,7 +1004,7 @@ gc_find_freeish_pages(long *restart_page_ptr, long nbytes, int unboxed) "Argh! gc_find_free_space failed (first_page), nbytes=%ld.\n", nbytes); print_generation_stats(1); - lose(NULL); + lose("\n"); } gc_assert(page_table[first_page].write_protected == 0); @@ -1026,9 +1035,10 @@ gc_find_freeish_pages(long *restart_page_ptr, long nbytes, int unboxed) "Argh! gc_find_freeish_pages failed (restart_page), nbytes=%ld.\n", nbytes); print_generation_stats(1); - lose(NULL); + lose("\n"); } *restart_page_ptr=first_page; + return last_page; } @@ -1413,6 +1423,7 @@ static lispobj trans_boxed(lispobj object); void sniff_code_object(struct code *code, unsigned long displacement) { +#ifdef LISP_FEATURE_X86 long nheader_words, ncode_words, nwords; void *p; void *constants_start_addr = NULL, *constants_end_addr; @@ -1578,11 +1589,14 @@ sniff_code_object(struct code *code, unsigned long displacement) "/code start = %x, end = %x\n", code_start_addr, code_end_addr)); } +#endif } void gencgc_apply_code_fixups(struct code *old_code, struct code *new_code) { +/* x86-64 uses pc-relative addressing instead of this kludge */ +#ifndef LISP_FEATURE_X86_64 long nheader_words, ncode_words, nwords; void *constants_start_addr, *constants_end_addr; void *code_start_addr, *code_end_addr; @@ -1673,6 +1687,7 @@ gencgc_apply_code_fixups(struct code *old_code, struct code *new_code) if (check_code_fixups) { sniff_code_object(new_code,displacement); } +#endif } @@ -1699,7 +1714,6 @@ trans_unboxed_large(lispobj object) lispobj header; unsigned long length; - gc_assert(is_lisp_pointer(object)); header = *((lispobj *) native_pointer(object)); @@ -1754,12 +1768,12 @@ scav_vector(lispobj *where, lispobj object) /* Scavenge element 0, which may be a hash-table structure. */ scavenge(where+2, 1); if (!is_lisp_pointer(where[2])) { - lose("no pointer at %x in hash table", where[2]); + lose("no pointer at %x in hash table\n", where[2]); } hash_table = (struct hash_table *)native_pointer(where[2]); /*FSHOW((stderr,"/hash_table = %x\n", hash_table));*/ if (widetag_of(hash_table->header) != INSTANCE_HEADER_WIDETAG) { - lose("hash table not instance (%x at %x)", + lose("hash table not instance (%x at %x)\n", hash_table->header, hash_table); } @@ -1768,13 +1782,13 @@ scav_vector(lispobj *where, lispobj object) * the hash table code reserves for marking empty slots. */ scavenge(where+3, 1); if (!is_lisp_pointer(where[3])) { - lose("not empty-hash-table-slot symbol pointer: %x", where[3]); + lose("not empty-hash-table-slot symbol pointer: %x\n", where[3]); } empty_symbol = where[3]; /* fprintf(stderr,"* empty_symbol = %x\n", empty_symbol);*/ if (widetag_of(*(lispobj *)native_pointer(empty_symbol)) != SYMBOL_HEADER_WIDETAG) { - lose("not a symbol where empty-hash-table-slot symbol expected: %x", + lose("not a symbol where empty-hash-table-slot symbol expected: %x\n", *(lispobj *)native_pointer(empty_symbol)); } @@ -1785,7 +1799,7 @@ scav_vector(lispobj *where, lispobj object) /* Cross-check the kv_vector. */ if (where != (lispobj *)native_pointer(hash_table->table)) { - lose("hash_table table!=this table %x", hash_table->table); + lose("hash_table table!=this table %x\n", hash_table->table); } /* WEAK-P */ @@ -1804,7 +1818,7 @@ scav_vector(lispobj *where, lispobj object) length = fixnum_value(((lispobj *)native_pointer(index_vector_obj))[1]); /*FSHOW((stderr, "/length = %d\n", length));*/ } else { - lose("invalid index_vector %x", index_vector_obj); + lose("invalid index_vector %x\n", index_vector_obj); } } @@ -1820,7 +1834,7 @@ scav_vector(lispobj *where, lispobj object) next_vector_length = fixnum_value(((lispobj *)native_pointer(next_vector_obj))[1]); /*FSHOW((stderr, "/next_vector_length = %d\n", next_vector_length));*/ } else { - lose("invalid next_vector %x", next_vector_obj); + lose("invalid next_vector %x\n", next_vector_obj); } } @@ -2661,11 +2675,9 @@ update_page_write_prot(page_index_t page) return (wp_it); } -/* Scavenge a generation. - * - * This will not resolve all pointers when generation is the new - * space, as new objects may be added which are not checked here - use - * scavenge_newspace generation. +/* Scavenge all generations from FROM to TO, inclusive, except for + * new_space which needs special handling, as new objects may be + * added which are not checked here - use scavenge_newspace generation. * * Write-protected pages should not have any pointers to the * from_space so do need scavenging; thus write-protected pages are @@ -2693,7 +2705,7 @@ update_page_write_prot(page_index_t page) * pointers as the objects contain a link to the next and are written * if a weak pointer is scavenged. Still it's a useful check. */ static void -scavenge_generation(generation_index_t generation) +scavenge_generations(generation_index_t from, generation_index_t to) { page_index_t i; int num_wp = 0; @@ -2706,9 +2718,12 @@ scavenge_generation(generation_index_t generation) #endif for (i = 0; i < last_free_page; i++) { + generation_index_t generation = page_table[i].gen; if ((page_table[i].allocated & BOXED_PAGE_FLAG) && (page_table[i].bytes_used != 0) - && (page_table[i].gen == generation)) { + && (generation != new_space) + && (generation >= from) + && (generation <= to)) { page_index_t last_page,j; int write_protected=1; @@ -2739,15 +2754,15 @@ scavenge_generation(generation_index_t generation) num_wp += update_page_write_prot(j); } } + if ((gencgc_verbose > 1) && (num_wp != 0)) { + FSHOW((stderr, + "/write protected %d pages within generation %d\n", + num_wp, generation)); + } } i = last_page; } } - if ((gencgc_verbose > 1) && (num_wp != 0)) { - FSHOW((stderr, - "/write protected %d pages within generation %d\n", - num_wp, generation)); - } #if SC_GEN_CK /* Check that none of the write_protected pages in this generation @@ -2763,7 +2778,7 @@ scavenge_generation(generation_index_t generation) page_table[i].bytes_used, page_table[i].first_object_offset, page_table[i].dont_move)); - lose("write to protected page %d in scavenge_generation()", i); + lose("write to protected page %d in scavenge_generation()\n", i); } } #endif @@ -2979,7 +2994,7 @@ scavenge_newspace_generation(generation_index_t generation) && (page_table[i].gen == generation) && (page_table[i].write_protected_cleared != 0) && (page_table[i].dont_move == 0)) { - lose("write protected page %d written to in scavenge_newspace_generation\ngeneration=%d dont_move=%d", + lose("write protected page %d written to in scavenge_newspace_generation\ngeneration=%d dont_move=%d\n", i, generation, page_table[i].dont_move); } } @@ -3078,8 +3093,8 @@ free_oldspace(void) os_invalidate(page_start, PAGE_BYTES*(last_page-first_page)); addr = os_validate(page_start, PAGE_BYTES*(last_page-first_page)); if (addr == NULL || addr != page_start) { - lose("free_oldspace: page moved, 0x%08x ==> 0x%08x",page_start, - addr); + lose("free_oldspace: page moved, 0x%08x ==> 0x%08x\n", + page_start, addr); } } else { long *page_start; @@ -3155,15 +3170,15 @@ verify_space(lispobj *start, size_t words) * page. XX Could check the offset too. */ if ((page_table[page_index].allocated != FREE_PAGE_FLAG) && (page_table[page_index].bytes_used == 0)) - lose ("Ptr %x @ %x sees free page.", thing, start); + lose ("Ptr %x @ %x sees free page.\n", thing, start); /* Check that it doesn't point to a forwarding pointer! */ if (*((lispobj *)native_pointer(thing)) == 0x01) { - lose("Ptr %x @ %x sees forwarding ptr.", thing, start); + lose("Ptr %x @ %x sees forwarding ptr.\n", thing, start); } /* Check that its not in the RO space as it would then be a * pointer from the RO to the dynamic space. */ if (is_in_readonly_space) { - lose("ptr to dynamic space %x from RO space %x", + lose("ptr to dynamic space %x from RO space %x\n", thing, start); } /* Does it point to a plausible object? This check slows @@ -3177,14 +3192,14 @@ verify_space(lispobj *start, size_t words) * dynamically. */ /* if (!possibly_valid_dynamic_space_pointer((lispobj *)thing)) { - lose("ptr %x to invalid object %x", thing, start); + lose("ptr %x to invalid object %x\n", thing, start); } */ } else { /* Verify that it points to another valid space. */ if (!to_readonly_space && !to_static_space && (thing != (unsigned long)&undefined_tramp)) { - lose("Ptr %x @ %x sees junk.", thing, start); + lose("Ptr %x @ %x sees junk.\n", thing, start); } } } else { @@ -3442,7 +3457,7 @@ verify_zero_fill(void) long i; for (i = 0; i < size; i++) { if (start_addr[i] != 0) { - lose("free page not zero at %x", start_addr + i); + lose("free page not zero at %x\n", start_addr + i); } } } else { @@ -3454,7 +3469,7 @@ verify_zero_fill(void) long i; for (i = 0; i < size; i++) { if (start_addr[i] != 0) { - lose("free region not zero at %x", start_addr + i); + lose("free region not zero at %x\n", start_addr + i); } } } @@ -3488,26 +3503,39 @@ verify_dynamic_space(void) static void write_protect_generation_pages(generation_index_t generation) { - page_index_t i; + page_index_t start; gc_assert(generation < SCRATCH_GENERATION); - for (i = 0; i < last_free_page; i++) - if ((page_table[i].allocated == BOXED_PAGE_FLAG) - && (page_table[i].bytes_used != 0) - && !page_table[i].dont_move - && (page_table[i].gen == generation)) { + for (start = 0; start < last_free_page; start++) { + if ((page_table[start].allocated == BOXED_PAGE_FLAG) + && (page_table[start].bytes_used != 0) + && !page_table[start].dont_move + && (page_table[start].gen == generation)) { void *page_start; + page_index_t last; - page_start = (void *)page_address(i); + /* Note the page as protected in the page tables. */ + page_table[start].write_protected = 1; + + for (last = start + 1; last < last_free_page; last++) { + if ((page_table[last].allocated != BOXED_PAGE_FLAG) + || (page_table[last].bytes_used == 0) + || page_table[last].dont_move + || (page_table[last].gen != generation)) + break; + page_table[last].write_protected = 1; + } + + page_start = (void *)page_address(start); os_protect(page_start, - PAGE_BYTES, + PAGE_BYTES * (last - start), OS_VM_PROT_READ | OS_VM_PROT_EXECUTE); - /* Note the page as protected in the page tables. */ - page_table[i].write_protected = 1; + start = last; } + } if (gencgc_verbose > 1) { FSHOW((stderr, @@ -3585,38 +3613,41 @@ garbage_collect_generation(generation_index_t generation, int raise) /* we assume that none of the preceding applies to the thread that * initiates GC. If you ever call GC from inside an altstack * handler, you will lose. */ - for_each_thread(th) { - void **ptr; - void **esp=(void **)-1; + + /* And if we're saving a core, there's no point in being conservative. */ + if (conservative_stack) { + for_each_thread(th) { + void **ptr; + void **esp=(void **)-1; #ifdef LISP_FEATURE_SB_THREAD - long i,free; - if(th==arch_os_get_current_thread()) { - /* Somebody is going to burn in hell for this, but casting - * it in two steps shuts gcc up about strict aliasing. */ - esp = (void **)((void *)&raise); - } else { - void **esp1; - free=fixnum_value(SymbolValue(FREE_INTERRUPT_CONTEXT_INDEX,th)); - for(i=free-1;i>=0;i--) { - os_context_t *c=th->interrupt_contexts[i]; - esp1 = (void **) *os_context_register_addr(c,reg_SP); - if (esp1>=(void **)th->control_stack_start && - esp1<(void **)th->control_stack_end) { - if(esp1=(void **)c; ptr--) { - preserve_pointer(*ptr); + long i,free; + if(th==arch_os_get_current_thread()) { + /* Somebody is going to burn in hell for this, but casting + * it in two steps shuts gcc up about strict aliasing. */ + esp = (void **)((void *)&raise); + } else { + void **esp1; + free=fixnum_value(SymbolValue(FREE_INTERRUPT_CONTEXT_INDEX,th)); + for(i=free-1;i>=0;i--) { + os_context_t *c=th->interrupt_contexts[i]; + esp1 = (void **) *os_context_register_addr(c,reg_SP); + if (esp1>=(void **)th->control_stack_start && + esp1<(void **)th->control_stack_end) { + if(esp1=(void **)c; ptr--) { + preserve_pointer(*ptr); + } } } } - } #else - esp = (void **)((void *)&raise); + esp = (void **)((void *)&raise); #endif - for (ptr = (void **)th->control_stack_end; ptr > esp; ptr--) { - preserve_pointer(*ptr); + for (ptr = (void **)th->control_stack_end; ptr > esp; ptr--) { + preserve_pointer(*ptr); + } } } - #ifdef QSHOW if (gencgc_verbose > 1) { long num_dont_move_pages = count_dont_move_pages(); @@ -3687,11 +3718,7 @@ garbage_collect_generation(generation_index_t generation, int raise) /* All generations but the generation being GCed need to be * scavenged. The new_space generation needs special handling as * objects may be moved in - it is handled separately below. */ - for (i = 0; i <= PSEUDO_STATIC_GENERATION; i++) { - if ((i != generation) && (i != new_space)) { - scavenge_generation(i); - } - } + scavenge_generations(generation+1, PSEUDO_STATIC_GENERATION); /* Finally scavenge the new_space generation. Keep going until no * more objects are moved into the new generation */ @@ -3720,7 +3747,7 @@ garbage_collect_generation(generation_index_t generation, int raise) bytes_allocated = bytes_allocated - old_bytes_allocated; if (bytes_allocated != 0) { - lose("Rescan of new_space allocated %d more bytes.", + lose("Rescan of new_space allocated %d more bytes.\n", bytes_allocated); } } @@ -3894,7 +3921,7 @@ collect_garbage(generation_index_t last_gen) /* Check that they are all empty. */ for (i = 0; i < gen_to_wp; i++) { if (generations[i].bytes_allocated) - lose("trying to write-protect gen. %d when gen. %d nonempty", + lose("trying to write-protect gen. %d when gen. %d nonempty\n", gen_to_wp, i); } write_protect_generation_pages(gen_to_wp); @@ -3939,6 +3966,7 @@ gc_free_heap(void) page_table[page].allocated = FREE_PAGE_FLAG; page_table[page].bytes_used = 0; +#ifndef LISP_FEATURE_WIN32 /* Pages already zeroed on win32? Not sure about this change. */ /* Zero the page. */ page_start = (void *)page_address(page); @@ -3949,10 +3977,13 @@ gc_free_heap(void) os_invalidate(page_start,PAGE_BYTES); addr = os_validate(page_start,PAGE_BYTES); if (addr == NULL || addr != page_start) { - lose("gc_free_heap: page moved, 0x%08x ==> 0x%08x", + lose("gc_free_heap: page moved, 0x%08x ==> 0x%08x\n", page_start, addr); } +#else + page_table[page].write_protected = 0; +#endif } else if (gencgc_zero_check_during_free_heap) { /* Double-check that the page is zero filled. */ long *page_start; @@ -3962,7 +3993,7 @@ gc_free_heap(void) page_start = (long *)page_address(page); for (i=0; i<1024; i++) { if (page_start[i] != 0) { - lose("free region not zero at %x", page_start + i); + lose("free region not zero at %x\n", page_start + i); } } } @@ -4050,7 +4081,6 @@ gc_init(void) gc_set_region_empty(&unboxed_region); last_free_page = 0; - } /* Pick up the dynamic space from after a core load. @@ -4064,24 +4094,34 @@ gencgc_pickup_dynamic(void) page_index_t page = 0; long alloc_ptr = SymbolValue(ALLOCATION_POINTER,0); lispobj *prev=(lispobj *)page_address(page); + generation_index_t gen = PSEUDO_STATIC_GENERATION; do { lispobj *first,*ptr= (lispobj *)page_address(page); page_table[page].allocated = BOXED_PAGE_FLAG; - page_table[page].gen = 0; + page_table[page].gen = gen; page_table[page].bytes_used = PAGE_BYTES; page_table[page].large_object = 0; - - first=gc_search_space(prev,(ptr+2)-prev,ptr); - if(ptr == first) prev=ptr; - page_table[page].first_object_offset = - (void *)prev - page_address(page); + page_table[page].write_protected = 0; + page_table[page].write_protected_cleared = 0; + page_table[page].dont_move = 0; + + if (!gencgc_partial_pickup) { + first=gc_search_space(prev,(ptr+2)-prev,ptr); + if(ptr == first) prev=ptr; + page_table[page].first_object_offset = + (void *)prev - page_address(page); + } page++; } while ((long)page_address(page) < alloc_ptr); - generations[0].bytes_allocated = PAGE_BYTES*page; + last_free_page = page; + + generations[gen].bytes_allocated = PAGE_BYTES*page; bytes_allocated = PAGE_BYTES*page; + gc_alloc_update_all_page_tables(); + write_protect_generation_pages(gen); } void @@ -4220,7 +4260,7 @@ gencgc_handle_wp_violation(void* fault_addr) * does this test after the first one has already set wp=0 */ if(page_table[page_index].write_protected_cleared != 1) - lose("fault in heap page not marked as write-protected"); + lose("fault in heap page not marked as write-protected\n"); } /* Don't worry, we can handle it. */ return 1; @@ -4254,3 +4294,64 @@ gc_set_region_empty(struct alloc_region *region) region->end_addr = page_address(0); } +/* Things to do before doing a final GC before saving a core (without + * purify). + * + * + Pages in large_object pages aren't moved by the GC, so we need to + * unset that flag from all pages. + * + The pseudo-static generation isn't normally collected, but it seems + * reasonable to collect it at least when saving a core. So move the + * pages to a normal generation. + */ +static void +prepare_for_final_gc () +{ + page_index_t i; + for (i = 0; i < last_free_page; i++) { + page_table[i].large_object = 0; + if (page_table[i].gen == PSEUDO_STATIC_GENERATION) { + int used = page_table[i].bytes_used; + page_table[i].gen = HIGHEST_NORMAL_GENERATION; + generations[PSEUDO_STATIC_GENERATION].bytes_allocated -= used; + generations[HIGHEST_NORMAL_GENERATION].bytes_allocated += used; + } + } +} + + +/* Do a non-conservative GC, and then save a core with the initial + * function being set to the value of the static symbol + * SB!VM:RESTART-LISP-FUNCTION */ +void +gc_and_save(char *filename) +{ + FILE *file = open_core_for_saving(filename); + if (!file) { + perror(filename); + return; + } + conservative_stack = 0; + + /* The filename might come from Lisp, and be moved by the now + * non-conservative GC. */ + filename = strdup(filename); + + /* Collect twice: once into relatively high memory, and then back + * into low memory. This compacts the retained data into the lower + * pages, minimizing the size of the core file. + */ + prepare_for_final_gc(); + gencgc_alloc_start_page = last_free_page; + collect_garbage(HIGHEST_NORMAL_GENERATION+1); + + prepare_for_final_gc(); + gencgc_alloc_start_page = -1; + collect_garbage(HIGHEST_NORMAL_GENERATION+1); + + save_to_filehandle(file, filename, SymbolValue(RESTART_LISP_FUNCTION,0)); + /* Oops. Save still managed to fail. Since we've mangled the stack + * beyond hope, there's not much we can do. + * (beyond FUNCALLing RESTART_LISP_FUNCTION, but I suspect that's + * going to be rather unsatisfactory too... */ + lose("Attempt to save core after non-conservative GC failed.\n"); +}