Optionally mark pages from cores as de-duplicatable

 Linux supports optimistic virtual-memory-level de-duplication for
 identical pages.  By default, only enable this for compressed cores.
 Runtime options --[no-]merge-core-pages can be used to override the
 default.

 Very much experimental and a hack, but compressed cores would sometimes
 be gratuitously inefficient otherwise.  If/when support is generalized
 to more OSes, figure out how to do this right.

 Thanks to David Lichteblau for the idea.

diff --git a/NEWS b/NEWS
index bf53f51..60e5f9c 100644 (file)
--- a/NEWS
+++ b/NEWS
@@ -6,6 +6,10 @@ changes relative to sbcl-1.0.50:
   * enhancement: when built with :sb-core-compression, core files (regular
     or executable) can be compressed with zlib.  Use the :COMPRESSION
     argument to SAVE-LISP-AND-DIE to specify a compression level.
+  * enhancement: --[no-]merge-core-pages determines whether the runtime
+    hints the operating system that identical core pages between SBCL
+    processes should share the same physical memory.  Default is to only
+    enable this for compressed cores.
   * optimization: SLEEP no longer conses.
   * optimization: *PRINT-PRETTY* no longer slows down printing of strings
     or bit-vectors when using the standard pretty-print dispatch table.

diff --git a/doc/manual/start-stop.texinfo b/doc/manual/start-stop.texinfo
index f97aef4..ece3c90 100644 (file)
--- a/doc/manual/start-stop.texinfo
+++ b/doc/manual/start-stop.texinfo
@@ -245,6 +245,26 @@ description of @code{--script} as a toplevel option below. If there
 are no other commandline arguments following @code{--script}, the
 filename argument can be omitted.
 
+
+@item --merge-core-pages
+When platform support is present, provide hints to the operating system
+that identical pages may be shared between processes until they are
+written to.  This can be useful to reduce the memory usage on systems
+with multiple SBCL processes started from similar but differently-named
+core files, or from compressed cores.  Without platform support, do
+nothing.
+
+
+@item --no-merge-core-pages
+Ensures that no sharing hint is provided to the operating system.
+
+
+@item --default-merge-core-pages
+Reverts the sharing hint policy to the default: only compressed cores
+trigger hinting.  Uncompressed cores are mapped directly from the core
+file, which is usually enough to ensure sharing.
+
+
 @item --help
 Print some basic information about SBCL, then exit.
 

diff --git a/doc/sbcl.1 b/doc/sbcl.1
index a877657..4a85831 100644 (file)
--- a/doc/sbcl.1
+++ b/doc/sbcl.1
@@ -122,6 +122,22 @@ As a runtime option equivalent to \-\-noinform \-\-disable\-ldb
 <filename>. See the description of \-\-script as a toplevel option
 below.
 .TP 3
+.B \-\-merge\-core\-pages
+When platform support is present, provide hints to the operating
+system that identical pages may be shared between processes until they
+are written to. This can be useful to reduce the memory usage on
+systems with multiple SBCL processes started from similar but
+differently\-named core files, or from compressed cores. Without
+platform support, do nothing.
+.TP 3
+.B \-\-no-merge\-core\-pages
+Ensures that no sharing hint is provided to the operating system.
+.TP 3
+.B \-\-default\-merge\-core\-pages
+Reverts the sharing hint policy to the default: only compressed cores
+trigger hinting. Uncompressed cores are mapped directly from the core
+file, which is usually enough to ensure sharing.
+.TP 3
 .B \-\-help
 Print some basic information about SBCL, then exit.
 .TP 3

diff --git a/src/runtime/core.h b/src/runtime/core.h
index 57df1cb..3f05cd5 100644 (file)
--- a/src/runtime/core.h
+++ b/src/runtime/core.h
@@ -31,6 +31,16 @@ struct ndir_entry {
 #endif
 };
 
+/* Tri-state flag to determine whether we attempt to mark pages
+ * as targets for virtual memory deduplication (ala MADV_MERGEABLE
+ * on Linux).
+ *
+ * 1: Yes
+ * 0: No
+ * -1: default, yes for compressed cores, no otherwise.
+ */
+extern int merge_core_pages;
+
 extern lispobj load_core_file(char *file, os_vm_offset_t offset);
 extern os_vm_offset_t search_for_embedded_core(char *file);
 

diff --git a/src/runtime/coreparse.c b/src/runtime/coreparse.c
index a794c39..bc562ba 100644 (file)
--- a/src/runtime/coreparse.c
+++ b/src/runtime/coreparse.c
@@ -22,6 +22,11 @@
 #include <sys/stat.h>
 #include <fcntl.h>
 #include <unistd.h>
+#ifdef LISP_FEATURE_LINUX
+/* For madvise */
+# define _BSD_SOURCE
+#endif
+#include <sys/mman.h>
 
 #include "sbcl.h"
 #include "os.h"
@@ -262,6 +267,8 @@ os_vm_address_t inflate_core_bytes(int fd, os_vm_offset_t offset,
 # undef ZLIB_BUFFER_SIZE
 #endif
 
+int merge_core_pages = -1;
+
 static void
 process_directory(int fd, lispobj *ptr, int count, os_vm_offset_t file_offset)
 {
@@ -308,6 +315,13 @@ process_directory(int fd, lispobj *ptr, int count, os_vm_offset_t file_offset)
             }
         }
 
+#ifdef MADV_MERGEABLE
+        if ((merge_core_pages == 1)
+            || ((merge_core_pages == -1) && compressed)) {
+                madvise(addr, len, MADV_MERGEABLE);
+        }
+#endif
+
         FSHOW((stderr, "/space id = %ld, free pointer = 0x%lx\n",
                id, (unsigned long)free_pointer));
 

diff --git a/src/runtime/runtime.c b/src/runtime/runtime.c
index 1396573..b84d6af 100644 (file)
--- a/src/runtime/runtime.c
+++ b/src/runtime/runtime.c
@@ -452,6 +452,15 @@ main(int argc, char *argv[], char *envp[])
                 end_runtime_options = 1;
                 ++argi;
                 break;
+            } else if (0 == strcmp(arg, "--merge-core-pages")) {
+                ++argi;
+                merge_core_pages = 1;
+            } else if (0 == strcmp(arg, "--no-merge-core-pages")) {
+                ++argi;
+                merge_core_pages = 0;
+            } else if (0 == strcmp(arg, "--default-merge-core-pages")) {
+                ++argi;
+                merge_core_pages = -1;
             } else {
                 /* This option was unrecognized as a runtime option,
                  * so it must be a toplevel option or a user option,