From b7eed59f1877263e1af5ad80299e641e8276f77d Mon Sep 17 00:00:00 2001 From: Daniel Barlow Date: Tue, 2 Sep 2003 00:08:14 +0000 Subject: [PATCH] 0.8.3.23 Exciting gencgc hacking: SB-SYS:WITH-PINNED-OBJECTS When passing Lisp objects to foreign code, it is important that GC doesn't change the object's address as the foreign code's pointers to it won't be updated. Traditionally this is done using WITHOUT-GCING, but that's not ideal in a threaded system where one thread may be doing a blocking alien call and other threads would like to continue as normal. So, instead we shove pointers to the important objects on the C stack where they will cause the pages pointed to to be pinned in place using preserve_pointers. Additionally we make a small change to gencgc to ensure that they _stay_ pinned in subsequent GCs On ports that use cheneygc we can't do this, so WITH-PINNED-OBJECTS is just an alias for WITHOUT-GCING. As these ports don't have threads anyway, that's not a major inconvenience. Small updates to doc/internals-notes/threading-specials, slightly larger updates to comments regarding ALLOCATION and PSEUDO-ATOMIC macros Documentation update to the FFI chapter --- contrib/sb-bsd-sockets/foreign-glue.lisp | 4 +- contrib/sb-bsd-sockets/name-service.lisp | 2 +- contrib/sb-bsd-sockets/sockets.lisp | 44 ++++++------ doc/beyond-ansi.sgml | 20 +++--- doc/ffi.sgml | 112 +++++++++++++++++++++++------- doc/internals-notes/threading-specials | 11 +-- doc/intro.sgml | 5 +- package-data-list.lisp-expr | 2 +- src/compiler/alpha/macros.lisp | 9 +++ src/compiler/fndb.lisp | 7 ++ src/compiler/hppa/macros.lisp | 9 +++ src/compiler/mips/macros.lisp | 9 +++ src/compiler/ppc/macros.lisp | 9 +++ src/compiler/sparc/macros.lisp | 10 +++ src/compiler/x86/c-call.lisp | 23 ++++++ src/compiler/x86/macros.lisp | 75 ++++++++++---------- src/runtime/gencgc.c | 7 +- version.lisp-expr | 2 +- 18 files changed, 253 insertions(+), 107 deletions(-) diff --git a/contrib/sb-bsd-sockets/foreign-glue.lisp b/contrib/sb-bsd-sockets/foreign-glue.lisp index b205364..958497f 100644 --- a/contrib/sb-bsd-sockets/foreign-glue.lisp +++ b/contrib/sb-bsd-sockets/foreign-glue.lisp @@ -50,12 +50,12 @@ ;;(declaim (inline ,el (setf ,el))) (defun ,el (ptr &optional (index 0)) (declare (optimize (speed 3))) - (sb-sys:without-gcing + (sb-sys:with-pinned-objects (ptr) ,(template 'prog1 nil))) (defconstant ,(intern (format nil "OFFSET-OF-~A" el)) ,offset) (defun (setf ,el) (newval ptr &optional (index 0)) (declare (optimize (speed 3))) - (sb-sys:without-gcing + (sb-sys:with-pinned-objects (ptr) ,(template 'setf 'newval))))))) diff --git a/contrib/sb-bsd-sockets/name-service.lisp b/contrib/sb-bsd-sockets/name-service.lisp index 23f65a1..46094d2 100644 --- a/contrib/sb-bsd-sockets/name-service.lisp +++ b/contrib/sb-bsd-sockets/name-service.lisp @@ -42,7 +42,7 @@ grisly details." (loop for i from 0 to 3 do (setf (sockint::in-addr-addr packed-addr i) (elt address i))) (make-host-ent - (sb-sys:without-gcing + (sb-sys:with-pinned-objects (packed-addr) (sockint::gethostbyaddr (sockint::array-data-address packed-addr) 4 sockint::af-inet))))) diff --git a/contrib/sb-bsd-sockets/sockets.lisp b/contrib/sb-bsd-sockets/sockets.lisp index ed03482..3134e7d 100644 --- a/contrib/sb-bsd-sockets/sockets.lisp +++ b/contrib/sb-bsd-sockets/sockets.lisp @@ -62,7 +62,7 @@ See also bind(2)")) (defmethod socket-bind ((socket socket) &rest address) (let ((sockaddr (apply #'make-sockaddr-for socket nil address))) - (if (= (sb-sys:without-gcing + (if (= (sb-sys:with-pinned-objects (sockaddr) (sockint::bind (socket-file-descriptor socket) (sockint::array-data-address sockaddr) (size-of-sockaddr socket))) @@ -76,28 +76,28 @@ newly-created connected socket and the peer address as multiple values")) (defmethod socket-accept ((socket socket)) - (let* ((sockaddr (make-sockaddr-for socket)) - (fd (sb-sys:without-gcing - (sockint::accept (socket-file-descriptor socket) - (sockint::array-data-address sockaddr) - (size-of-sockaddr socket))))) - (apply #'values - (if (= fd -1) - (socket-error "accept") - (let ((s (make-instance (class-of socket) - :type (socket-type socket) - :protocol (socket-protocol socket) - :descriptor fd))) - (sb-ext:finalize s (lambda () (sockint::close fd))))) - (multiple-value-list (bits-of-sockaddr socket sockaddr))))) - + (let ((sockaddr (make-sockaddr-for socket))) + (sb-ext::with-pointers-preserved (sockaddr) + (let ((fd (sockint::accept (socket-file-descriptor socket) + (sockint::array-data-address sickint) + (size-of-sockaddr socket)))) + (apply #'values + (if (= fd -1) + (socket-error "accept") + (let ((s (make-instance (class-of socket) + :type (socket-type socket) + :protocol (socket-protocol socket) + :descriptor fd))) + (sb-ext:finalize s (lambda () (sockint::close fd))))) + (multiple-value-list (bits-of-sockaddr socket sockaddr))))))) + (defgeneric socket-connect (socket &rest address) (:documentation "Perform the connect(2) call to connect SOCKET to a remote PEER. No useful return value.")) (defmethod socket-connect ((socket socket) &rest peer) (let* ((sockaddr (apply #'make-sockaddr-for socket nil peer))) - (if (= (sb-sys:without-gcing + (if (= (sb-sys:with-pinned-objects (sockaddr) (sockint::connect (socket-file-descriptor socket) (sockint::array-data-address sockaddr) (size-of-sockaddr socket))) @@ -110,7 +110,7 @@ values")) (defmethod socket-peername ((socket socket)) (let* ((sockaddr (make-sockaddr-for socket))) - (when (= (sb-sys:without-gcing + (when (= (sb-sys:with-pinned-objects (sockaddr) (sockint::getpeername (socket-file-descriptor socket) (sockint::array-data-address sockaddr) (size-of-sockaddr socket))) @@ -124,7 +124,7 @@ values")) (defmethod socket-name ((socket socket)) (let* ((sockaddr (make-sockaddr-for socket))) - (when (= (sb-sys:without-gcing + (when (= (sb-sys:with-pinned-objects (sockaddr) (sockint::getsockname (socket-file-descriptor socket) (sockint::array-data-address sockaddr) (size-of-sockaddr socket))) @@ -169,7 +169,7 @@ small")) (setf buffer (make-array length :element-type element-type))) (sb-alien:with-alien ((sa-len (array (sb-alien:unsigned 32) 2))) (setf (sb-alien:deref sa-len 0) (size-of-sockaddr socket)) - (sb-sys:without-gcing + (sb-sys:with-pinned-objects (buffer sockaddr) (let ((len (sockint::recvfrom (socket-file-descriptor socket) (sockint::array-data-address buffer) @@ -230,7 +230,9 @@ SB-SYS:MAKE-FD-STREAM.")) (and (slot-boundp socket 'stream) (slot-value socket 'stream)))) (unless stream (setf stream (apply #'sb-sys:make-fd-stream - (socket-file-descriptor socket) args)) + (socket-file-descriptor socket) + :name "a constant string" + args)) (setf (slot-value socket 'stream) stream) (sb-ext:cancel-finalization socket)) stream)) diff --git a/doc/beyond-ansi.sgml b/doc/beyond-ansi.sgml index 694898a..9138489 100644 --- a/doc/beyond-ansi.sgml +++ b/doc/beyond-ansi.sgml @@ -142,7 +142,7 @@ current bugs) are: Threading (a.k.a Multiprocessing)</> -<para>&SBCL; (as of version 0.x.y, on Linux x86 only) supports a +<para>&SBCL; (as of version 0.8.3, on Linux x86 only) supports a fairly low-level threading interface that maps onto the host operating system's concept of threads or lightweight processes. @@ -214,20 +214,20 @@ and woken with SIGCONT. <para>&SBCL; at present will alway have at least two tasks running as seen from Linux: when the first process has done startup initialization (mapping files in place, installing signal handlers -etc) it creates a new thread to run the Lisp startup and initial listener. -The original thread is then used to run GC and to reap dead subthreads -when they exit. +etc) it creates a new thread to run the Lisp startup and initial +listener. The original thread stays around to reap dead subthreads +and deallocate their resources (e.g. stacks) when they exit. <para>Garbage collection is done with the existing Conservative Generational GC. Allocation is done in small (typically 8k) regions : each thread has its own region so this involves no stopping. However, when a region fills, a lock must be obtained while another is allocated, and when a collection is required, all processes are -stopped. This is achieved using <function>ptrace()</function>, so you -should be very careful if you wish to examine an &SBCL; worker thread -using <command>strace</command>, <command>truss</command>, -<command>gdb</command> or similar. It may be prudent to disable GC -before doing so. +stopped. This is achieved by sending them signals, which may make for +interesting behaviour if they are interrupted in system calls. The +streams interface is believed to handle the required system call +restarting correctly, but this may be a consideration when making +other blocking calls e.g. from foreign library code. <para>Large amounts of the &SBCL; library have not been inspected for thread-safety. Some of the obviously unsafe areas have large locks @@ -241,7 +241,7 @@ on keyboard interrupt handling: pressing your terminal's intr key process group, including Lisp threads that &SBCL; considers to be notionally `background'. This is undesirable, so background threads are set to ignore the SIGINT signal. Arbitration for the input stream -is managed by locking on sb-thread::*session-lock* +is managed by locking on <varname>sb-thread::*session-lock*</varname> <para>A thread can be created in a new Lisp 'session' (new terminal or window) using <function>sb-thread:make-listener-thread</function>. diff --git a/doc/ffi.sgml b/doc/ffi.sgml index e48d16c..5f11eab 100644 --- a/doc/ffi.sgml +++ b/doc/ffi.sgml @@ -42,13 +42,13 @@ are three common approaches to establishing communication: </para></listitem> </itemizedlist> -<para>&SBCL;, like &CMUCL; before it, -relies primarily on the automatic conversion and direct manipulation -approaches. Foreign values of simple scalar types are automatically -converted, complex types are directly manipulated in their foreign -representation. Furthermore, Lisp strings are represented internally -with null termination bytes so that they can be passed directly to -C interfaces without allocating new zero-terminated copies.</para> +<para>&SBCL;, like &CMUCL; before it, relies primarily on the +automatic conversion and direct manipulation approaches. The SB-ALIEN +package provices a facility wherein foreign values of simple scalar +types are automatically converted and complex types are directly +manipulated in their foreign representation. Additionally the +lower-level System Area Pointers (or SAPs) can be used where +necessary to provide untyped access to foreign memory.</para> <para>Any foreign objects that can't automatically be converted into Lisp values are represented by objects of type <type>alien-value</>. @@ -294,6 +294,9 @@ These are the basic foreign type specifiers: null-terminated string, and is automatically converted into a Lisp string when accessed; or if the pointer is C <literal>NULL</> or <literal>0</>, then accessing it gives Lisp <literal>nil</>. + Lisp strings are stored with a trailing NUL termination, so no + copying (either by the user or the implementation) is necessary + when passing them to foreign code. </para> <para> Assigning a Lisp string to a <type>c-string</> structure field or @@ -363,7 +366,41 @@ a new value. Note that <varname>slot-name</> is evaluated, and need not be a compile-time constant (but only constant slot accesses are efficiently compiled.)</para> -</sect2> +<sect3><title>Untyped memory</> + +<para>As noted at the beginning of the chapter, the System Area +Pointer facilities allow untyped access to foreign memory. SAPs can +be converted to and from the usual typed foreign values using +<function>sap-alien</function> and <function>alien-sap</function> +(described elsewhere), and also to and from integers - raw machine +addresses. They should thus be used with caution; corrupting the Lisp +heap or other memory with SAPs is trivial.</para> + +<synopsis>(sb-sys:int-sap machine-address)</> + +<para>Creates a SAP pointing at the virtual address +<varname>machine-address</varname>. </para> + +<synopsis>(sb-sys:sap-ref-32 sap offset)</> + +<para>Access the value of the memory location at +<varname>offset</varname> bytes from <varname>sap</varname>. This form +may also be used with <function>setf</function> to alter the memory at +that location.</para> + +<synopsis>(sb-sys:sap= sap1 sap2)</> + +<para>Compare <varname>sap1</varname> and <varname>sap2</varname> for +equality.</para> + +<para>Similarly named functions exist for accessing other sizes of +word, other comparisons, and other conversions. The reader is invited +to use <function>apropos</function> and <function>describe</function> +for more details</para> +<programlisting> +(apropos "sap" :sb-sys) +</programlisting> +</sect3></sect2> <sect2><title>Coercing Foreign Values</> @@ -392,8 +429,6 @@ argument, but it does refer to the same foreign data bits.</para> <para>The <function>sb-alien:sap-alien</> function converts <varname>sap</> (a system area pointer) to a foreign value with the specified <varname>type</>. <varname>type</> is not evaluated. -As of &SBCL; 0.7.6, it looks as though this and other SAP functionality -may become deprecated, since it shouldn't be needed by user code. </para> <para>The <varname>type</> must be some foreign pointer, array, or @@ -403,8 +438,6 @@ record type.</para> <para>The <function>sb-alien:alien-sap</> function returns the SAP which points to <varname>alien-value</>'s data. -As of &SBCL; 0.7.6, it looks as though this and other SAP functionality -may become deprecated, since it shouldn't be needed by user code. </para> <para>The <varname>foreign-value</> must be of some foreign pointer, @@ -688,19 +721,23 @@ code when a saved core is loaded.</para></note> <para> The foreign function call interface allows a Lisp program to call -functions written in other languages using the C calling convention. +many functions written in languages that use the C calling convention. </para> <para> -Lisp sets up various interrupt handling routines and other environment +Lisp sets up various signal handling routines and other environment information when it first starts up, and expects these to be in place -at all times. The C functions called by Lisp should either not change -the environment, especially the interrupt entry points, or should make -sure that these entry points are restored when the C function returns -to Lisp. If a C function makes changes without restoring things to the -way they were when the C function was entered, there is no telling -what will happen. -</para> +at all times. The C functions called by Lisp should not change the +environment, especially the signal handlers: the signal handlers +installed by Lisp typically have interesting flags set (e.g to request +machine context information, or for signal delivery on an alternate +stack) which the Lisp runtime relies on for correct operation. +Precise details of how this works may change without notice between +versions; the source, or the brain of a friendly &SBCL; developer, +is the only documentation. Users of a Lisp built with the :sb-thread +feature should also read the Threading section +<!-- FIXME I'm sure docbook has some syntax for internal links --> +of this manual</para> <sect2><title>The <function>alien-funcall</> Primitive @@ -864,15 +901,30 @@ the runtime system. The arguments must be valid &SBCL; object descriptors (so that e.g. fixnums must be left-shifted by 2.) As of &SBCL; 0.7.5, the format -of object descriptors is documented only by the source code and +of object descriptors is documented only by the source code and, in parts, by the old &CMUCL; "INTERNALS" documentation. Note that the garbage collector moves objects, and won't be -able to fix up any references in C variables, so either turn GC off or -don't keep Lisp pointers in C data unless they are to statically -allocated objects. It is possible to use the -sb-ext:purify function to place live data structures in -static space so that they won't move during GC. +able to fix up any references in C variables. There are three +mechanisms for coping with this: + + +The sb-ext:purify moves all live Lisp +data into static or read-only areas such that it will never be moved +(or freed) again in the life of the Lisp session + +sb-sys:with-pinned-objects is a +macro which arranges for some set of objects to be pinned in memory +for the dynamic extent of its body forms. On ports which use the +generational garbage collector (as of &SBCL; 0.8.3, only the x86) this +has a page granularity - i.e. the entire 4k page or pages containing +the objects will be locked down. On other ports it is implemented by +turning off GC for the duration (so could be said to have a +whole-world granularity). + +Disable GC, using the without-gcing +macro or gc-off call. +