FreeBSD-5.3/sys/amd64/amd64/apic_vector.S
/*-
* Copyright (c) 1989, 1990 William F. Jolitz.
* Copyright (c) 1990 The Regents of the University of California.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 4. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* from: vector.s, 386BSD 0.1 unknown origin
* $FreeBSD: src/sys/amd64/amd64/apic_vector.S,v 1.100 2004/05/24 12:08:56 bde Exp $
*/
/*
* Interrupt entry points for external interrupts triggered by I/O APICs
* as well as IPI handlers.
*/
#include <machine/asmacros.h>
#include <machine/apicreg.h>
#include "assym.s"
/*
* Macros to create and destroy a trap frame.
*/
#define PUSH_FRAME \
subq $TF_RIP,%rsp ; /* skip dummy tf_err and tf_trapno */ \
testb $SEL_RPL_MASK,TF_CS(%rsp) ; /* come from kernel? */ \
jz 1f ; /* Yes, dont swapgs again */ \
swapgs ; \
1: movq %rdi,TF_RDI(%rsp) ; \
movq %rsi,TF_RSI(%rsp) ; \
movq %rdx,TF_RDX(%rsp) ; \
movq %rcx,TF_RCX(%rsp) ; \
movq %r8,TF_R8(%rsp) ; \
movq %r9,TF_R9(%rsp) ; \
movq %rax,TF_RAX(%rsp) ; \
movq %rbx,TF_RBX(%rsp) ; \
movq %rbp,TF_RBP(%rsp) ; \
movq %r10,TF_R10(%rsp) ; \
movq %r11,TF_R11(%rsp) ; \
movq %r12,TF_R12(%rsp) ; \
movq %r13,TF_R13(%rsp) ; \
movq %r14,TF_R14(%rsp) ; \
movq %r15,TF_R15(%rsp)
#define POP_FRAME \
movq TF_RDI(%rsp),%rdi ; \
movq TF_RSI(%rsp),%rsi ; \
movq TF_RDX(%rsp),%rdx ; \
movq TF_RCX(%rsp),%rcx ; \
movq TF_R8(%rsp),%r8 ; \
movq TF_R9(%rsp),%r9 ; \
movq TF_RAX(%rsp),%rax ; \
movq TF_RBX(%rsp),%rbx ; \
movq TF_RBP(%rsp),%rbp ; \
movq TF_R10(%rsp),%r10 ; \
movq TF_R11(%rsp),%r11 ; \
movq TF_R12(%rsp),%r12 ; \
movq TF_R13(%rsp),%r13 ; \
movq TF_R14(%rsp),%r14 ; \
movq TF_R15(%rsp),%r15 ; \
testb $SEL_RPL_MASK,TF_CS(%rsp) ; /* come from kernel? */ \
jz 1f ; /* keep kernel GS.base */ \
cli ; \
swapgs ; \
1: addq $TF_RIP,%rsp /* skip over tf_err, tf_trapno */
/*
* I/O Interrupt Entry Point. Rather than having one entry point for
* each interrupt source, we use one entry point for each 32-bit word
* in the ISR. The handler determines the highest bit set in the ISR,
* translates that into a vector, and passes the vector to the
* lapic_handle_intr() function.
*/
#define ISR_VEC(index, vec_name) \
.text ; \
SUPERALIGN_TEXT ; \
IDTVEC(vec_name) ; \
PUSH_FRAME ; \
FAKE_MCOUNT(TF_RIP(%rsp)) ; \
movq lapic, %rdx ; /* pointer to local APIC */ \
movl LA_ISR + 16 * (index)(%rdx), %eax ; /* load ISR */ \
bsrl %eax, %eax ; /* index of highset set bit in ISR */ \
jz 2f ; \
addl $(32 * index),%eax ; \
1: ; \
movq %rax, %rdi ; /* pass the IRQ */ \
call lapic_handle_intr ; \
MEXITCOUNT ; \
jmp doreti ; \
2: movl $-1, %eax ; /* send a vector of -1 */ \
jmp 1b
/*
* Handle "spurious INTerrupts".
* Notes:
* This is different than the "spurious INTerrupt" generated by an
* 8259 PIC for missing INTs. See the APIC documentation for details.
* This routine should NOT do an 'EOI' cycle.
*/
.text
SUPERALIGN_TEXT
IDTVEC(spuriousint)
/* No EOI cycle used here */
iretq
ISR_VEC(1, apic_isr1)
ISR_VEC(2, apic_isr2)
ISR_VEC(3, apic_isr3)
ISR_VEC(4, apic_isr4)
ISR_VEC(5, apic_isr5)
ISR_VEC(6, apic_isr6)
ISR_VEC(7, apic_isr7)
#ifdef SMP
/*
* Global address space TLB shootdown.
*/
.text
SUPERALIGN_TEXT
IDTVEC(invltlb)
pushq %rax
movq %cr3, %rax /* invalidate the TLB */
movq %rax, %cr3
movq lapic, %rax
movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
lock
incl smp_tlb_wait
popq %rax
iretq
/*
* Single page TLB shootdown
*/
.text
SUPERALIGN_TEXT
IDTVEC(invlpg)
pushq %rax
movq smp_tlb_addr1, %rax
invlpg (%rax) /* invalidate single page */
movq lapic, %rax
movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
lock
incl smp_tlb_wait
popq %rax
iretq
/*
* Page range TLB shootdown.
*/
.text
SUPERALIGN_TEXT
IDTVEC(invlrng)
pushq %rax
pushq %rdx
movq smp_tlb_addr1, %rdx
movq smp_tlb_addr2, %rax
1: invlpg (%rdx) /* invalidate single page */
addq $PAGE_SIZE, %rdx
cmpq %rax, %rdx
jb 1b
movq lapic, %rax
movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
lock
incl smp_tlb_wait
popq %rdx
popq %rax
iretq
/*
* Forward hardclock to another CPU. Pushes a clockframe and calls
* forwarded_hardclock().
*/
.text
SUPERALIGN_TEXT
IDTVEC(hardclock)
PUSH_FRAME
movq lapic, %rdx
movl $0, LA_EOI(%rdx) /* End Of Interrupt to APIC */
call forwarded_hardclock
MEXITCOUNT
jmp doreti
/*
* Forward statclock to another CPU. Pushes a clockframe and calls
* forwarded_statclock().
*/
.text
SUPERALIGN_TEXT
IDTVEC(statclock)
PUSH_FRAME
movq lapic, %rdx
movl $0, LA_EOI(%rdx) /* End Of Interrupt to APIC */
FAKE_MCOUNT(TF_RIP(%rsp))
call forwarded_statclock
MEXITCOUNT
jmp doreti
/*
* Executed by a CPU when it receives an Xcpuast IPI from another CPU,
*
* The other CPU has already executed aston() or need_resched() on our
* current process, so we simply need to ack the interrupt and return
* via doreti to run ast().
*/
.text
SUPERALIGN_TEXT
IDTVEC(cpuast)
PUSH_FRAME
movq lapic, %rdx
movl $0, LA_EOI(%rdx) /* End Of Interrupt to APIC */
FAKE_MCOUNT(TF_RIP(%rsp))
MEXITCOUNT
jmp doreti
/*
* Executed by a CPU when it receives an Xcpustop IPI from another CPU,
*
* - Signals its receipt.
* - Waits for permission to restart.
* - Signals its restart.
*/
.text
SUPERALIGN_TEXT
IDTVEC(cpustop)
PUSH_FRAME
movq lapic, %rax
movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
movl PCPU(CPUID), %eax
imull $PCB_SIZE, %eax
leaq stoppcbs(%rax), %rdi
call savectx /* Save process context */
movl PCPU(CPUID), %eax
lock
btsl %eax, stopped_cpus /* stopped_cpus |= (1<<id) */
1:
btl %eax, started_cpus /* while (!(started_cpus & (1<<id))) */
jnc 1b
lock
btrl %eax, started_cpus /* started_cpus &= ~(1<<id) */
lock
btrl %eax, stopped_cpus /* stopped_cpus &= ~(1<<id) */
test %eax, %eax
jnz 2f
movq cpustop_restartfunc, %rax
testq %rax, %rax
jz 2f
movq $0, cpustop_restartfunc /* One-shot */
call *%rax
2:
POP_FRAME
iretq
/*
* Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU.
*
* - Calls the generic rendezvous action function.
*/
.text
SUPERALIGN_TEXT
IDTVEC(rendezvous)
PUSH_FRAME
call smp_rendezvous_action
movq lapic, %rax
movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */
POP_FRAME /* Why not doreti? */
iretq
#endif /* SMP */