2 * very-low-level utilities for runtime support
6 * This software is part of the SBCL system. See the README file for
9 * This software is derived from the CMU CL system, which was
10 * written at Carnegie Mellon University and released into the
11 * public domain. The software is in the public domain and is
12 * provided with absolutely no warranty. See the COPYING and CREDITS
13 * files for more information.
16 #define LANGUAGE_ASSEMBLY
17 #include "genesis/config.h"
20 #include "genesis/closure.h"
21 #include "genesis/funcallable-instance.h"
22 #include "genesis/fdefn.h"
23 #include "genesis/static-symbols.h"
24 #include "genesis/symbol.h"
25 #include "genesis/thread.h"
27 /* Minimize conditionalization for different OS naming schemes. */
28 #if defined __linux__ || defined __FreeBSD__ || defined __OpenBSD__ || defined __NetBSD__ || defined __sun
29 #define GNAME(var) var
31 #define GNAME(var) _##var
34 /* Get the right type of alignment. Linux, FreeBSD and OpenBSD
35 * want alignment in bytes. */
36 #if defined(__linux__) || defined(__FreeBSD__) || defined(__OpenBSD__) || defined __NetBSD__ || defined(__sun)
39 #define align_16byte 16
40 #define align_32byte 32
41 #define align_page 32768
45 #define align_16byte 4
50 * The assembler used for win32 doesn't like .type or .size directives,
51 * so we want to conditionally kill them out. So let's wrap them in macros
52 * that are defined to be no-ops on win32. Hopefully this still works on
55 #if !defined(LISP_FEATURE_WIN32) && !defined(LISP_FEATURE_DARWIN)
56 #define TYPE(name) .type name,@function
57 #define SIZE(name) .size name,.-name
58 #define DOLLAR(name) $(name)
65 * x86/darwin (as of MacOS X 10.4.5) doesn't reliably fire signal
66 * handlers (SIGTRAP or Mach exception handlers) for 0xCC, wo we have
67 * to use ud2 instead. ud2 is an undefined opcode, #x0b0f, or
68 * 0F 0B in low-endian notation, that causes SIGILL to fire. We check
69 * for this instruction in the SIGILL handler and if we see it, we
70 * advance the EIP by two bytes to skip over ud2 instruction and
71 * call sigtrap_handler. */
72 #if defined(LISP_FEATURE_UD2_BREAKPOINTS)
79 * More Apple assembler hacks
82 #if defined(LISP_FEATURE_DARWIN)
83 /* global symbol x86-64 sym(%rip) hack:*/
84 #define GSYM(name) name(%rip)
87 #define GSYM(name) $name
93 .globl GNAME(all_threads)
97 /* From lower to higher-numbered addresses, the stack contains
98 * return address, arg 0, arg 1, arg 2 ...
99 * rax contains the address of the function to call
100 * Lisp expects return value in rax, whic is already consistent with C
101 * XXXX correct floating point handling is unimplemented so far
102 * Based on comments cleaned from x86-assem.S, we believe that
103 * Lisp is expecting us to preserve rsi, rdi, rsp (no idea about r8-15)
106 .align align_16byte,0x90
107 .globl GNAME(call_into_c)
108 TYPE(GNAME(call_into_c))
110 push %rbp # Save old frame pointer.
111 mov %rsp,%rbp # Establish new frame.
113 push %rsi # args are going in here
126 SIZE(GNAME(call_into_c))
130 .globl GNAME(call_into_lisp_first_time)
131 TYPE(GNAME(call_into_lisp_first_time))
133 /* We don't worry too much about saving registers
134 * here, because we never expect to return from the initial call to lisp
137 .align align_16byte,0x90
138 GNAME(call_into_lisp_first_time):
139 push %rbp # Save old frame pointer.
140 mov %rsp,%rbp # Establish new frame.
141 #if defined(LISP_FEATURE_DARWIN)
142 movq GSYM(GNAME(all_threads)),%rax
144 movq GNAME(all_threads),%rax
146 mov THREAD_CONTROL_STACK_END_OFFSET(%rax) ,%rsp
150 .globl GNAME(call_into_lisp)
151 TYPE(GNAME(call_into_lisp))
154 * amd64 calling convention: C expects that
155 * arguments go in rdi rsi rdx rcx r8 r9
156 * return values in rax rdx
157 * callee saves rbp rbx r12-15 if it uses them
160 .align align_16byte,0x90
161 GNAME(call_into_lisp):
162 push %rbp # Save old frame pointer.
163 mov %rsp,%rbp # Establish new frame.
165 /* FIXME x86 saves FPU state here */
166 push %rbx # these regs are callee-saved according to C
167 push %r12 # so must be preserved and restored when
168 push %r13 # the lisp function returns
172 mov %rsp,%rbx # remember current stack
173 push %rbx # Save entry stack on (maybe) new stack.
175 push %rdi # args from C
178 #ifdef LISP_FEATURE_SB_THREAD
179 #ifdef LISP_FEATURE_GCC_TLS
181 movq GNAME(current_thread)@TPOFF(%rax), %r12
183 #ifdef LISP_FEATURE_DARWIN
184 mov GSYM(GNAME(specials)),%rdi
188 call GNAME(pthread_getspecific)
193 pop %rbx # arg vector
194 pop %rax # function ptr/lexenv
196 xor %rdx,%rdx # clear any descriptor registers
197 xor %rdi,%rdi # that we can't be sure we'll
198 xor %rsi,%rsi # initialise properly. XX do r8-r15 too?
199 shl $(N_FIXNUM_TAG_BITS),%rcx # (fixnumize num-args)
202 mov 0(%rbx),%rdx # arg0
205 mov 8(%rbx),%rdi # arg1
208 mov 16(%rbx),%rsi # arg2
210 /* Registers rax, rcx, rdx, rdi, and rsi are now live. */
211 xor %rbx,%rbx # available
213 /* Alloc new frame. */
214 push %rbp # Dummy for return address
215 push %rbp # fp in save location S1
216 mov %rsp,%rbp # The current sp marks start of new frame.
217 sub $8,%rsp # Ensure 3 slots are allocated, two above.
220 call *CLOSURE_FUN_OFFSET(%rax)
222 /* If the function returned multiple values, it will return to
223 this point. Lose them */
228 /* Restore the stack, in case there was a stack change. */
238 /* FIXME Restore the NPX state. */
240 mov %rdx,%rax # c-val
243 SIZE(GNAME(call_into_lisp))
245 /* support for saving and restoring the NPX state from C */
247 .globl GNAME(fpu_save)
248 TYPE(GNAME(fpu_save))
249 .align align_16byte,0x90
251 fnsave (%rdi) # Save the NPX state. (resets NPX)
253 SIZE(GNAME(fpu_save))
255 .globl GNAME(fpu_restore)
256 TYPE(GNAME(fpu_restore))
257 .align align_16byte,0x90
259 frstor (%rdi) # Restore the NPX state.
261 SIZE(GNAME(fpu_restore))
264 * the undefined-function trampoline
267 .align align_16byte,0x90
268 .globl GNAME(undefined_tramp)
269 TYPE(GNAME(undefined_tramp))
270 GNAME(undefined_tramp):
271 pop 8(%rbp) # Save return PC for backtrace.
275 .byte UNDEFINED_FUN_ERROR
276 .byte sc_DescriptorReg # eax in the Descriptor-reg SC
278 SIZE(GNAME(undefined_tramp))
280 /* KLUDGE: FIND-ESCAPED-FRAME (SYS:SRC;CODE;DEBUG-INT.LISP) needs
281 * to know the name of the function immediately following the
282 * undefined-function trampoline. */
285 .align align_16byte,0x90
286 .globl GNAME(alloc_tramp)
287 TYPE(GNAME(alloc_tramp))
289 push %rbp # Save old frame pointer.
290 mov %rsp,%rbp # Establish new frame.
314 SIZE(GNAME(alloc_tramp))
318 * the closure trampoline
321 .align align_16byte,0x90
322 .globl GNAME(closure_tramp)
323 TYPE(GNAME(closure_tramp))
324 GNAME(closure_tramp):
325 mov FDEFN_FUN_OFFSET(%rax),%rax
326 /* FIXME: The '*' after "jmp" in the next line is from PVE's
327 * patch posted to the CMU CL mailing list Oct 6, 1999. It looks
328 * reasonable, and it certainly seems as though if CMU CL needs it,
329 * SBCL needs it too, but I haven't actually verified that it's
330 * right. It would be good to find a way to force the flow of
331 * control through here to test it. */
332 jmp *CLOSURE_FUN_OFFSET(%rax)
333 SIZE(GNAME(closure_tramp))
336 .align align_16byte,0x90
337 .globl GNAME(funcallable_instance_tramp)
338 #if !defined(LISP_FEATURE_DARWIN)
339 .type GNAME(funcallable_instance_tramp),@function
341 GNAME(funcallable_instance_tramp):
342 mov FUNCALLABLE_INSTANCE_FUNCTION_OFFSET(%rax),%rax
343 /* KLUDGE: on this platform, whatever kind of function is in %rax
344 * now, the first word of it contains the address to jump to. */
345 jmp *CLOSURE_FUN_OFFSET(%rax)
346 #if !defined(LISP_FEATURE_DARWIN)
347 .size GNAME(funcallable_instance_tramp), .-GNAME(funcallable_instance_tramp)
350 * fun-end breakpoint magic
354 * For an explanation of the magic involved in function-end
355 * breakpoints, see the implementation in ppc-assem.S.
359 .globl GNAME(fun_end_breakpoint_guts)
361 GNAME(fun_end_breakpoint_guts):
362 /* Multiple Value return */
363 jc multiple_value_return
364 /* Single value return: The eventual return will now use the
365 multiple values return convention but with a return values
367 mov %rsp,%rbx # Setup ebx - the ofp.
368 sub $8,%rsp # Allocate one stack slot for the return value
369 mov $8,%rcx # Setup ecx for one return value.
370 #if defined(LISP_FEATURE_DARWIN)
371 mov GSYM(NIL),%rdi # default second value
372 mov GSYM(NIL),%rsi # default third value
374 mov $NIL,%rdi # default second value
375 mov $NIL,%rsi # default third value
377 multiple_value_return:
379 .globl GNAME(fun_end_breakpoint_trap)
380 .align align_16byte,0x90
381 GNAME(fun_end_breakpoint_trap):
383 .byte trap_FunEndBreakpoint
384 hlt # We should never return here.
386 .globl GNAME(fun_end_breakpoint_end)
387 GNAME(fun_end_breakpoint_end):
390 .globl GNAME(do_pending_interrupt)
391 TYPE(GNAME(do_pending_interrupt))
392 .align align_16byte,0x90
393 GNAME(do_pending_interrupt):
395 .byte trap_PendingInterrupt
397 SIZE(GNAME(do_pending_interrupt))
399 .globl GNAME(post_signal_tramp)
400 TYPE(GNAME(post_signal_tramp))
401 .align align_16byte,0x90
402 GNAME(post_signal_tramp):
403 /* this is notionally the second half of a function whose first half
404 * doesn't exist. This is where call_into_lisp returns when called
405 * using return_to_lisp_function */
416 /* skip RBP and RSP */
424 SIZE(GNAME(post_signal_tramp))
427 .align align_16byte,0x90
428 .globl GNAME(fast_bzero)
429 TYPE(GNAME(fast_bzero))
432 /* A fast routine for zero-filling blocks of memory that are
433 * guaranteed to start and end at a 4096-byte aligned address.
435 shr $6, %rsi /* Amount of 64-byte blocks to copy */
436 jz Lend /* If none, stop */
437 mov %rsi, %rcx /* Save start address */
438 movups %xmm7, -16(%rsp) /* Save XMM register */
439 xorps %xmm7, %xmm7 /* Zero the XMM register */
444 /* Copy the 16 zeroes from xmm7 to memory, 4 times. MOVNTDQ is the
445 * non-caching double-quadword moving variant, i.e. the memory areas
446 * we're touching are not fetched into the L1 cache, since we're just
447 * going to overwrite the memory soon anyway.
449 movntdq %xmm7, 0(%rdi)
450 movntdq %xmm7, 16(%rdi)
451 movntdq %xmm7, 32(%rdi)
452 movntdq %xmm7, 48(%rdi)
454 add $64, %rdi /* Advance pointer */
455 dec %rsi /* Decrement 64-byte block count */
457 mfence /* Ensure that the writes are globally visible, since
458 * MOVNTDQ is weakly ordered */
459 movups -16(%rsp), %xmm7 /* Restore the XMM register */
460 prefetcht0 0(%rcx) /* Prefetch the start of the block into cache,
461 * since it's likely to be used immediately. */
464 SIZE(GNAME(fast_bzero))
467 /* When LISP_FEATURE_C_STACK_IS_CONTROL_STACK, we cannot safely scrub
468 * the control stack from C, largely due to not knowing where the
469 * active stack frame ends. On such platforms, we reimplement the
470 * core scrubbing logic in assembly, in this case here:
473 .align align_16byte,0x90
474 .globl GNAME(arch_scrub_control_stack)
475 TYPE(GNAME(arch_scrub_control_stack))
476 GNAME(arch_scrub_control_stack):
477 /* We are passed three parameters:
478 * A (struct thread *) in RDI,
479 * the address of the guard page in RSI, and
480 * the address of the hard guard page in RDX.
481 * We may trash RAX, RCX, and R8-R11 with impunity.
482 * [RSP] is our return address, [RSP-8] is the first
483 * stack slot to scrub. */
485 /* We start by setting up our scrub pointer in RAX, our
486 * guard page upper bound in R8, and our hard guard
487 * page upper bound in R9. */
489 #ifdef LISP_FEATURE_DARWIN
490 mov GSYM(GNAME(os_vm_page_size)),%r9
492 mov os_vm_page_size,%r9
497 /* Now we begin our main scrub loop. */
500 /* If we're about to scrub the hard guard page, exit. */
502 jae ascs_check_guard_page
506 ascs_check_guard_page:
507 /* If we're about to scrub the guard page, and the guard
508 * page is protected, exit. */
513 cmpq $(NIL), THREAD_CONTROL_STACK_GUARD_PAGE_PROTECTED_OFFSET(%rdi)
516 /* Clear memory backwards to the start of the (4KiB) page */
523 /* If we're about to hit the hard guard page, exit. */
527 /* If the next (previous?) 4KiB page contains a non-zero
528 * word, continue scrubbing. */
538 SIZE(GNAME(arch_scrub_control_stack))
540 .globl GNAME(gc_safepoint_page)
543 GNAME(gc_safepoint_page):