From: Paul Khuong Date: Sun, 28 Aug 2011 03:24:14 +0000 (-0400) Subject: Optionally mark pages from cores as de-duplicatable X-Git-Url: http://repo.macrolet.net/gitweb/?a=commitdiff_plain;h=77bb7c43dae7507aacae795af427548cb4616bc8;p=sbcl.git Optionally mark pages from cores as de-duplicatable Linux supports optimistic virtual-memory-level de-duplication for identical pages. By default, only enable this for compressed cores. Runtime options --[no-]merge-core-pages can be used to override the default. Very much experimental and a hack, but compressed cores would sometimes be gratuitously inefficient otherwise. If/when support is generalized to more OSes, figure out how to do this right. Thanks to David Lichteblau for the idea. --- diff --git a/NEWS b/NEWS index bf53f51..60e5f9c 100644 --- a/NEWS +++ b/NEWS @@ -6,6 +6,10 @@ changes relative to sbcl-1.0.50: * enhancement: when built with :sb-core-compression, core files (regular or executable) can be compressed with zlib. Use the :COMPRESSION argument to SAVE-LISP-AND-DIE to specify a compression level. + * enhancement: --[no-]merge-core-pages determines whether the runtime + hints the operating system that identical core pages between SBCL + processes should share the same physical memory. Default is to only + enable this for compressed cores. * optimization: SLEEP no longer conses. * optimization: *PRINT-PRETTY* no longer slows down printing of strings or bit-vectors when using the standard pretty-print dispatch table. diff --git a/doc/manual/start-stop.texinfo b/doc/manual/start-stop.texinfo index f97aef4..ece3c90 100644 --- a/doc/manual/start-stop.texinfo +++ b/doc/manual/start-stop.texinfo @@ -245,6 +245,26 @@ description of @code{--script} as a toplevel option below. If there are no other commandline arguments following @code{--script}, the filename argument can be omitted. + +@item --merge-core-pages +When platform support is present, provide hints to the operating system +that identical pages may be shared between processes until they are +written to. This can be useful to reduce the memory usage on systems +with multiple SBCL processes started from similar but differently-named +core files, or from compressed cores. Without platform support, do +nothing. + + +@item --no-merge-core-pages +Ensures that no sharing hint is provided to the operating system. + + +@item --default-merge-core-pages +Reverts the sharing hint policy to the default: only compressed cores +trigger hinting. Uncompressed cores are mapped directly from the core +file, which is usually enough to ensure sharing. + + @item --help Print some basic information about SBCL, then exit. diff --git a/doc/sbcl.1 b/doc/sbcl.1 index a877657..4a85831 100644 --- a/doc/sbcl.1 +++ b/doc/sbcl.1 @@ -122,6 +122,22 @@ As a runtime option equivalent to \-\-noinform \-\-disable\-ldb . See the description of \-\-script as a toplevel option below. .TP 3 +.B \-\-merge\-core\-pages +When platform support is present, provide hints to the operating +system that identical pages may be shared between processes until they +are written to. This can be useful to reduce the memory usage on +systems with multiple SBCL processes started from similar but +differently\-named core files, or from compressed cores. Without +platform support, do nothing. +.TP 3 +.B \-\-no-merge\-core\-pages +Ensures that no sharing hint is provided to the operating system. +.TP 3 +.B \-\-default\-merge\-core\-pages +Reverts the sharing hint policy to the default: only compressed cores +trigger hinting. Uncompressed cores are mapped directly from the core +file, which is usually enough to ensure sharing. +.TP 3 .B \-\-help Print some basic information about SBCL, then exit. .TP 3 diff --git a/src/runtime/core.h b/src/runtime/core.h index 57df1cb..3f05cd5 100644 --- a/src/runtime/core.h +++ b/src/runtime/core.h @@ -31,6 +31,16 @@ struct ndir_entry { #endif }; +/* Tri-state flag to determine whether we attempt to mark pages + * as targets for virtual memory deduplication (ala MADV_MERGEABLE + * on Linux). + * + * 1: Yes + * 0: No + * -1: default, yes for compressed cores, no otherwise. + */ +extern int merge_core_pages; + extern lispobj load_core_file(char *file, os_vm_offset_t offset); extern os_vm_offset_t search_for_embedded_core(char *file); diff --git a/src/runtime/coreparse.c b/src/runtime/coreparse.c index a794c39..bc562ba 100644 --- a/src/runtime/coreparse.c +++ b/src/runtime/coreparse.c @@ -22,6 +22,11 @@ #include #include #include +#ifdef LISP_FEATURE_LINUX +/* For madvise */ +# define _BSD_SOURCE +#endif +#include #include "sbcl.h" #include "os.h" @@ -262,6 +267,8 @@ os_vm_address_t inflate_core_bytes(int fd, os_vm_offset_t offset, # undef ZLIB_BUFFER_SIZE #endif +int merge_core_pages = -1; + static void process_directory(int fd, lispobj *ptr, int count, os_vm_offset_t file_offset) { @@ -308,6 +315,13 @@ process_directory(int fd, lispobj *ptr, int count, os_vm_offset_t file_offset) } } +#ifdef MADV_MERGEABLE + if ((merge_core_pages == 1) + || ((merge_core_pages == -1) && compressed)) { + madvise(addr, len, MADV_MERGEABLE); + } +#endif + FSHOW((stderr, "/space id = %ld, free pointer = 0x%lx\n", id, (unsigned long)free_pointer)); diff --git a/src/runtime/runtime.c b/src/runtime/runtime.c index 1396573..b84d6af 100644 --- a/src/runtime/runtime.c +++ b/src/runtime/runtime.c @@ -452,6 +452,15 @@ main(int argc, char *argv[], char *envp[]) end_runtime_options = 1; ++argi; break; + } else if (0 == strcmp(arg, "--merge-core-pages")) { + ++argi; + merge_core_pages = 1; + } else if (0 == strcmp(arg, "--no-merge-core-pages")) { + ++argi; + merge_core_pages = 0; + } else if (0 == strcmp(arg, "--default-merge-core-pages")) { + ++argi; + merge_core_pages = -1; } else { /* This option was unrecognized as a runtime option, * so it must be a toplevel option or a user option,