/* $NetBSD: copy.S,v 1.12.4.1 2009/02/26 20:38:00 snj Exp $ */ /* * Copyright (c) 2001 Wasabi Systems, Inc. * All rights reserved. * * Written by Frank van der Linden for Wasabi Systems, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed for the NetBSD Project by * Wasabi Systems, Inc. * 4. The name of Wasabi Systems, Inc. may not be used to endorse * or promote products derived from this software without specific prior * written permission. * * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include "assym.h" #include <sys/errno.h> #include <sys/syscall.h> #include <machine/asm.h> #include <machine/frameasm.h> #define GET_CURPCB(reg) \ movq CPUVAR(CURLWP), reg; \ movq L_ADDR(reg), reg /* * These are arranged so that the abnormal case is a forwards * conditional branch - which will be predicted not-taken by * both Intel and AMD processors. */ #define DEFERRED_SWITCH_CHECK \ CHECK_DEFERRED_SWITCH ; \ jnz 99f ; \ 98: #define DEFERRED_SWITCH_CALL \ 99: ; \ call _C_LABEL(do_pmap_load) ; \ jmp 98b .text x86_copyfunc_start: .globl x86_copyfunc_start /* * Handle deferred pmap switch. We must re-enable preemption without * making a function call, so that the program counter is visible to * cpu_kpreempt_exit(). It can then know if it needs to restore the * pmap on returning, because a preemption occurred within one of the * copy functions. */ ENTRY(do_pmap_load) pushq %rbp movq %rsp,%rbp pushq %rdi pushq %rsi pushq %rdx pushq %rcx pushq %rbx movq CPUVAR(CURLWP), %rbx 1: incl L_NOPREEMPT(%rbx) call _C_LABEL(pmap_load) decl L_NOPREEMPT(%rbx) jnz 2f cmpl $0, L_DOPREEMPT(%rbx) jz 2f xorq %rdi, %rdi call _C_LABEL(kpreempt) 2: cmpl $0, CPUVAR(WANT_PMAPLOAD) jnz 1b popq %rbx popq %rcx popq %rdx popq %rsi popq %rdi leaveq ret /* * Copy routines from and to userland, plus a few more. See the * section 9 manpages for info. Some cases can be optimized more. * * I wonder if it's worthwhile to make these use SSE2 registers? * (dsl) Not from info I've read from the AMD guides. * * Also note that the setup time for 'rep movs' is horrid - especially on P4 * netburst - but on my AMD X2 it manages one copy (read+write) per clock * which can be achieved with a code loop, but is probably impossible to beat. * Howver the use of 'rep movsb' for the final bytes should be killed. * * Newer Intel cpus have a much lower setup time, and may (someday) * be ably to do cache-line size copies.... */ ENTRY(kcopy) GET_CURPCB(%rax) pushq PCB_ONFAULT(%rax) leaq _C_LABEL(kcopy_fault)(%rip),%r11 movq %r11, PCB_ONFAULT(%rax) xchgq %rdi,%rsi movq %rdx,%rcx movq %rdi,%rax subq %rsi,%rax cmpq %rcx,%rax # overlapping? jb 1f # nope, copy forward shrq $3,%rcx # copy by 64-bit words rep movsq movq %rdx,%rcx andl $7,%ecx # any bytes left? rep movsb GET_CURPCB(%rdx) popq PCB_ONFAULT(%rdx) xorq %rax,%rax ret # Using 'rep movs' to copy backwards is not as fast as for forwards copies # and ought not be done when the copy doesn't acually overlap. # However kcopy() isn't used any that looks even vaguely used often. # I'm also not sure it is ever asked to do overlapping copies! 1: addq %rcx,%rdi # copy backward addq %rcx,%rsi std andq $7,%rcx # any fractional bytes? decq %rdi decq %rsi rep movsb movq %rdx,%rcx # copy remainder by 64-bit words shrq $3,%rcx subq $7,%rsi subq $7,%rdi rep movsq cld GET_CURPCB(%rdx) popq PCB_ONFAULT(%rdx) xorq %rax,%rax ret ENTRY(copyout) DEFERRED_SWITCH_CHECK pushq $0 # Restored to PCB_ONFAULT(%rdx) xchgq %rdi,%rsi # kernel address to %rsi, user to %rdi movq %rdx,%rax # save transfer length (bytes) addq %rdi,%rdx # end address to %rdx jc _C_LABEL(copy_efault) # jump if wraps movq $VM_MAXUSER_ADDRESS,%r8 cmpq %r8,%rdx ja _C_LABEL(copy_efault) # jump if end in kernel space GET_CURPCB(%rdx) leaq _C_LABEL(copy_fault)(%rip),%r11 movq %r11,PCB_ONFAULT(%rdx) # prime fault handler movq %rax,%rcx # length shrq $3,%rcx # count of 8-byte words rep movsq # copy from %rsi to %rdi movb %al,%cl andb $7,%cl # remaining number of bytes rep movsb # copy remaining bytes popq PCB_ONFAULT(%rdx) xorl %eax,%eax ret DEFERRED_SWITCH_CALL ENTRY(copyin) DEFERRED_SWITCH_CHECK GET_CURPCB(%rax) pushq $0 leaq _C_LABEL(copy_fault)(%rip),%r11 movq %r11,PCB_ONFAULT(%rax) xchgq %rdi,%rsi movq %rdx,%rax addq %rsi,%rdx # Check source address not wrapped jc _C_LABEL(copy_efault) movq $VM_MAXUSER_ADDRESS,%r8 cmpq %r8,%rdx ja _C_LABEL(copy_efault) # j if end in kernel space 3: /* bcopy(%rsi, %rdi, %rax); */ movq %rax,%rcx shrq $3,%rcx rep movsq movb %al,%cl andb $7,%cl rep movsb GET_CURPCB(%rdx) popq PCB_ONFAULT(%rdx) xorl %eax,%eax ret DEFERRED_SWITCH_CALL NENTRY(copy_efault) movq $EFAULT,%rax /* * kcopy_fault is used by kcopy and copy_fault is used by copyin/out. * * they're distinguished for lazy pmap switching. see trap(). */ NENTRY(kcopy_fault) GET_CURPCB(%rdx) popq PCB_ONFAULT(%rdx) ret NENTRY(copy_fault) GET_CURPCB(%rdx) popq PCB_ONFAULT(%rdx) ret ENTRY(copyoutstr) DEFERRED_SWITCH_CHECK xchgq %rdi,%rsi movq %rdx,%r8 movq %rcx,%r9 5: GET_CURPCB(%rax) leaq _C_LABEL(copystr_fault)(%rip),%r11 movq %r11,PCB_ONFAULT(%rax) /* * Get min(%rdx, VM_MAXUSER_ADDRESS-%rdi). */ movq $VM_MAXUSER_ADDRESS,%rax subq %rdi,%rax jc _C_LABEL(copystr_efault) cmpq %rdx,%rax jae 1f movq %rax,%rdx movq %rax,%r8 1: incq %rdx 1: decq %rdx jz 2f lodsb stosb testb %al,%al jnz 1b /* Success -- 0 byte reached. */ decq %rdx xorq %rax,%rax jmp copystr_return 2: /* rdx is zero -- return EFAULT or ENAMETOOLONG. */ movq $VM_MAXUSER_ADDRESS,%r11 cmpq %r11,%rdi jae _C_LABEL(copystr_efault) movq $ENAMETOOLONG,%rax jmp copystr_return DEFERRED_SWITCH_CALL ENTRY(copyinstr) DEFERRED_SWITCH_CHECK xchgq %rdi,%rsi movq %rdx,%r8 movq %rcx,%r9 GET_CURPCB(%rcx) leaq _C_LABEL(copystr_fault)(%rip),%r11 movq %r11,PCB_ONFAULT(%rcx) /* * Get min(%rdx, VM_MAXUSER_ADDRESS-%rsi). */ movq $VM_MAXUSER_ADDRESS,%rax subq %rsi,%rax jc _C_LABEL(copystr_efault) cmpq %rdx,%rax jae 1f movq %rax,%rdx movq %rax,%r8 1: incq %rdx 1: decq %rdx jz 2f lodsb stosb testb %al,%al jnz 1b /* Success -- 0 byte reached. */ decq %rdx xorq %rax,%rax jmp copystr_return 2: /* edx is zero -- return EFAULT or ENAMETOOLONG. */ movq $VM_MAXUSER_ADDRESS,%r11 cmpq %r11,%rsi jae _C_LABEL(copystr_efault) movq $ENAMETOOLONG,%rax jmp copystr_return DEFERRED_SWITCH_CALL ENTRY(copystr_efault) movl $EFAULT,%eax ENTRY(copystr_fault) copystr_return: /* Set *lencopied and return %eax. */ GET_CURPCB(%rcx) movq $0,PCB_ONFAULT(%rcx) testq %r9,%r9 jz 8f subq %rdx,%r8 movq %r8,(%r9) 8: ret ENTRY(copystr) xchgq %rdi,%rsi movq %rdx,%r8 incq %rdx 1: decq %rdx jz 4f lodsb stosb testb %al,%al jnz 1b /* Success -- 0 byte reached. */ decq %rdx xorl %eax,%eax jmp 6f 4: /* edx is zero -- return ENAMETOOLONG. */ movl $ENAMETOOLONG,%eax 6: /* Set *lencopied and return %eax. */ testq %rcx,%rcx jz 7f subq %rdx,%r8 movq %r8,(%rcx) 7: ret ENTRY(fuword) DEFERRED_SWITCH_CHECK movq $VM_MAXUSER_ADDRESS-4,%r11 cmpq %r11,%rdi ja _C_LABEL(fusuaddrfault) GET_CURPCB(%rcx) leaq _C_LABEL(fusufailure)(%rip),%r11 movq %r11,PCB_ONFAULT(%rcx) movl (%rdi),%eax movq $0,PCB_ONFAULT(%rcx) ret DEFERRED_SWITCH_CALL ENTRY(fusword) DEFERRED_SWITCH_CHECK movq $VM_MAXUSER_ADDRESS-2,%r11 cmpq %r11,%rdi ja _C_LABEL(fusuaddrfault) GET_CURPCB(%rcx) leaq _C_LABEL(fusufailure)(%rip),%r11 movq %r11,PCB_ONFAULT(%rcx) movzwl (%rdi),%eax movq $0,PCB_ONFAULT(%rcx) ret DEFERRED_SWITCH_CALL ENTRY(fuswintr) cmpq $TLBSTATE_VALID, CPUVAR(TLBSTATE) jnz _C_LABEL(fusuaddrfault) movq $VM_MAXUSER_ADDRESS-2,%r11 cmpq %r11,%rdi ja _C_LABEL(fusuaddrfault) GET_CURPCB(%rcx) leaq _C_LABEL(fusuintrfailure)(%rip),%r11 movq %r11,PCB_ONFAULT(%rcx) movzwl (%rdi),%eax movq $0,PCB_ONFAULT(%rcx) ret ENTRY(fubyte) DEFERRED_SWITCH_CHECK movq $VM_MAXUSER_ADDRESS-1,%r11 cmpq %r11,%rdi ja _C_LABEL(fusuaddrfault) GET_CURPCB(%rcx) leaq _C_LABEL(fusufailure)(%rip),%r11 movq %r11,PCB_ONFAULT(%rcx) movzbl (%rdi),%eax movq $0,PCB_ONFAULT(%rcx) ret DEFERRED_SWITCH_CALL ENTRY(suword) DEFERRED_SWITCH_CHECK movq $VM_MAXUSER_ADDRESS-4,%r11 cmpq %r11,%rdi ja _C_LABEL(fusuaddrfault) GET_CURPCB(%rcx) leaq _C_LABEL(fusufailure)(%rip),%r11 movq %r11,PCB_ONFAULT(%rcx) movq %rsi,(%rdi) xorq %rax,%rax movq %rax,PCB_ONFAULT(%rcx) ret DEFERRED_SWITCH_CALL ENTRY(susword) DEFERRED_SWITCH_CHECK movq $VM_MAXUSER_ADDRESS-2,%r11 cmpq %r11,%rdi ja _C_LABEL(fusuaddrfault) GET_CURPCB(%rcx) leaq _C_LABEL(fusufailure)(%rip),%r11 movq %r11,PCB_ONFAULT(%rcx) movw %si,(%rdi) xorq %rax,%rax movq %rax,PCB_ONFAULT(%rcx) ret DEFERRED_SWITCH_CALL ENTRY(suswintr) cmpq $TLBSTATE_VALID, CPUVAR(TLBSTATE) jnz _C_LABEL(fusuaddrfault) movq $VM_MAXUSER_ADDRESS-2,%r11 cmpq %r11,%rdi ja _C_LABEL(fusuaddrfault) GET_CURPCB(%rcx) leaq _C_LABEL(fusuintrfailure)(%rip),%r11 movq %r11,PCB_ONFAULT(%rcx) movw %si,(%rdi) xorq %rax,%rax movq %rax,PCB_ONFAULT(%rcx) ret ENTRY(subyte) DEFERRED_SWITCH_CHECK movq $VM_MAXUSER_ADDRESS-1,%r11 cmpq %r11,%rdi ja _C_LABEL(fusuaddrfault) GET_CURPCB(%rcx) leaq _C_LABEL(fusufailure)(%rip),%r11 movq %r11,PCB_ONFAULT(%rcx) movb %sil,(%rdi) xorq %rax,%rax movq %rax,PCB_ONFAULT(%rcx) ret DEFERRED_SWITCH_CALL /* * These are the same, but must reside at different addresses, * because trap.c checks for them. */ ENTRY(fusuintrfailure) movq $0,PCB_ONFAULT(%rcx) movl $-1,%eax ret ENTRY(fusufailure) movq $0,PCB_ONFAULT(%rcx) movl $-1,%eax ret ENTRY(fusuaddrfault) movl $-1,%eax ret /* * Compare-and-swap the 64-bit integer in the user-space. * * int ucas_64(volatile int64_t *uptr, int64_t old, int64_t new, int64_t *ret); */ ENTRY(ucas_64) DEFERRED_SWITCH_CHECK /* Fail if kernel-space */ movq $VM_MAXUSER_ADDRESS-8, %r8 cmpq %r8, %rdi ja 1f /* Set the fault handler */ GET_CURPCB(%r8) leaq _C_LABEL(ucas_fault)(%rip), %r9 movq %r9, PCB_ONFAULT(%r8) /* Perform the CAS */ lock cmpxchgq %rdx, (%rdi) /* Clear the fault handler */ movq %rax, PCB_ONFAULT(%r8) /* * Note: %rax is "old" value. * Set the return values. */ movq %rax, (%rcx) xorq %rax, %rax /* Clear the fault handler */ movq %rax, PCB_ONFAULT(%r8) 1: /* Failure case */ movq $EFAULT, %rax ret DEFERRED_SWITCH_CALL /* * int ucas_32(volatile int32_t *uptr, int32_t old, int32_t new, int32_t *ret); */ ENTRY(ucas_32) DEFERRED_SWITCH_CHECK /* Fail if kernel-space */ movq $VM_MAXUSER_ADDRESS-4, %r8 cmpq %r8, %rdi ja 1f /* Set the fault handler */ GET_CURPCB(%r8) leaq _C_LABEL(ucas_fault)(%rip), %r9 movq %r9, PCB_ONFAULT(%r8) /* Perform the CAS */ lock cmpxchgl %edx, (%rdi) /* * Note: %eax is "old" value. * Set the return values. */ movl %eax, (%rcx) xorq %rax, %rax 1: /* Failure case */ movq $EFAULT, %rax ret DEFERRED_SWITCH_CALL /* * Fault handler for ucas_32() and ucas_64(). * Unset the handler and return the failure. */ NENTRY(ucas_fault) movq $0, PCB_ONFAULT(%r8) movq $EFAULT, %rax ret /* * int ucas_ptr(volatile void *uptr, void *old, void *new, void *ret); */ STRONG_ALIAS(ucas_ptr, ucas_64) STRONG_ALIAS(ucas_int, ucas_32) x86_copyfunc_end: .globl x86_copyfunc_end