summaryrefslogtreecommitdiffstats
path: root/arch/x86_64
diff options
context:
space:
mode:
Diffstat (limited to 'arch/x86_64')
-rw-r--r--arch/x86_64/kernel/asm-offsets.c4
-rw-r--r--arch/x86_64/kernel/entry.S26
-rw-r--r--arch/x86_64/kernel/setup64.c22
-rw-r--r--arch/x86_64/kernel/traps.c35
4 files changed, 76 insertions, 11 deletions
diff --git a/arch/x86_64/kernel/asm-offsets.c b/arch/x86_64/kernel/asm-offsets.c
index aaa6d3833517..00a08d13230a 100644
--- a/arch/x86_64/kernel/asm-offsets.c
+++ b/arch/x86_64/kernel/asm-offsets.c
@@ -64,5 +64,9 @@ int main(void)
DEFINE(pbe_address, offsetof(struct pbe, address));
DEFINE(pbe_orig_address, offsetof(struct pbe, orig_address));
DEFINE(pbe_next, offsetof(struct pbe, next));
+ BLANK();
+#if DEBUG_STKSZ > EXCEPTION_STKSZ
+ DEFINE(DEBUG_IST, DEBUG_STACK);
+#endif
return 0;
}
diff --git a/arch/x86_64/kernel/entry.S b/arch/x86_64/kernel/entry.S
index bd21ebafaf66..28b3c8e6b044 100644
--- a/arch/x86_64/kernel/entry.S
+++ b/arch/x86_64/kernel/entry.S
@@ -673,7 +673,10 @@ ENTRY(spurious_interrupt)
/* error code is on the stack already */
/* handle NMI like exceptions that can happen everywhere */
- .macro paranoidentry sym
+#ifndef DEBUG_IST
+# define DEBUG_IST 0
+#endif
+ .macro paranoidentry sym, ist=0
SAVE_ALL
cld
movl $1,%ebx
@@ -683,10 +686,20 @@ ENTRY(spurious_interrupt)
js 1f
swapgs
xorl %ebx,%ebx
-1: movq %rsp,%rdi
+1:
+ .if \ist
+ movq %gs:pda_data_offset, %rbp
+ .endif
+ movq %rsp,%rdi
movq ORIG_RAX(%rsp),%rsi
movq $-1,ORIG_RAX(%rsp)
+ .if \ist
+ subq $EXCEPTION_STACK_SIZE, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
+ .endif
call \sym
+ .if \ist
+ addq $EXCEPTION_STACK_SIZE, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
+ .endif
cli
.endm
@@ -904,7 +917,7 @@ KPROBE_ENTRY(debug)
INTR_FRAME
pushq $0
CFI_ADJUST_CFA_OFFSET 8
- paranoidentry do_debug
+ paranoidentry do_debug, DEBUG_IST
jmp paranoid_exit
CFI_ENDPROC
.previous .text
@@ -959,7 +972,12 @@ paranoid_schedule:
CFI_ENDPROC
KPROBE_ENTRY(int3)
- zeroentry do_int3
+ INTR_FRAME
+ pushq $0
+ CFI_ADJUST_CFA_OFFSET 8
+ paranoidentry do_int3, DEBUG_IST
+ jmp paranoid_exit
+ CFI_ENDPROC
.previous .text
ENTRY(overflow)
diff --git a/arch/x86_64/kernel/setup64.c b/arch/x86_64/kernel/setup64.c
index 7b7131d8bc90..39e728cfe6a0 100644
--- a/arch/x86_64/kernel/setup64.c
+++ b/arch/x86_64/kernel/setup64.c
@@ -145,7 +145,7 @@ void pda_init(int cpu)
pda->irqstackptr += IRQSTACKSIZE-64;
}
-char boot_exception_stacks[N_EXCEPTION_STACKS * EXCEPTION_STKSZ]
+char boot_exception_stacks[(N_EXCEPTION_STACKS - 2) * EXCEPTION_STKSZ + DEBUG_STKSZ]
__attribute__((section(".bss.page_aligned")));
/* May not be marked __init: used by software suspend */
@@ -236,13 +236,27 @@ void __cpuinit cpu_init (void)
*/
for (v = 0; v < N_EXCEPTION_STACKS; v++) {
if (cpu) {
- estacks = (char *)__get_free_pages(GFP_ATOMIC,
- EXCEPTION_STACK_ORDER);
+ static const unsigned int order[N_EXCEPTION_STACKS] = {
+ [0 ... N_EXCEPTION_STACKS - 1] = EXCEPTION_STACK_ORDER,
+ [DEBUG_STACK - 1] = DEBUG_STACK_ORDER
+ };
+
+ estacks = (char *)__get_free_pages(GFP_ATOMIC, order[v]);
if (!estacks)
panic("Cannot allocate exception stack %ld %d\n",
v, cpu);
}
- estacks += EXCEPTION_STKSZ;
+ switch (v + 1) {
+#if DEBUG_STKSZ > EXCEPTION_STKSZ
+ case DEBUG_STACK:
+ cpu_pda[cpu].debugstack = (unsigned long)estacks;
+ estacks += DEBUG_STKSZ;
+ break;
+#endif
+ default:
+ estacks += EXCEPTION_STKSZ;
+ break;
+ }
t->ist[v] = (unsigned long)estacks;
}
diff --git a/arch/x86_64/kernel/traps.c b/arch/x86_64/kernel/traps.c
index bd71ddac0dc5..1a9094dab682 100644
--- a/arch/x86_64/kernel/traps.c
+++ b/arch/x86_64/kernel/traps.c
@@ -121,19 +121,31 @@ int printk_address(unsigned long address)
static unsigned long *in_exception_stack(unsigned cpu, unsigned long stack,
unsigned *usedp, const char **idp)
{
- static const char ids[N_EXCEPTION_STACKS][8] = {
+ static char ids[][8] = {
[DEBUG_STACK - 1] = "#DB",
[NMI_STACK - 1] = "NMI",
[DOUBLEFAULT_STACK - 1] = "#DF",
[STACKFAULT_STACK - 1] = "#SS",
[MCE_STACK - 1] = "#MC",
+#if DEBUG_STKSZ > EXCEPTION_STKSZ
+ [N_EXCEPTION_STACKS ... N_EXCEPTION_STACKS + DEBUG_STKSZ / EXCEPTION_STKSZ - 2] = "#DB[?]"
+#endif
};
unsigned k;
for (k = 0; k < N_EXCEPTION_STACKS; k++) {
unsigned long end;
- end = per_cpu(init_tss, cpu).ist[k];
+ switch (k + 1) {
+#if DEBUG_STKSZ > EXCEPTION_STKSZ
+ case DEBUG_STACK:
+ end = cpu_pda[cpu].debugstack + DEBUG_STKSZ;
+ break;
+#endif
+ default:
+ end = per_cpu(init_tss, cpu).ist[k];
+ break;
+ }
if (stack >= end)
continue;
if (stack >= end - EXCEPTION_STKSZ) {
@@ -143,6 +155,22 @@ static unsigned long *in_exception_stack(unsigned cpu, unsigned long stack,
*idp = ids[k];
return (unsigned long *)end;
}
+#if DEBUG_STKSZ > EXCEPTION_STKSZ
+ if (k == DEBUG_STACK - 1 && stack >= end - DEBUG_STKSZ) {
+ unsigned j = N_EXCEPTION_STACKS - 1;
+
+ do {
+ ++j;
+ end -= EXCEPTION_STKSZ;
+ ids[j][4] = '1' + (j - N_EXCEPTION_STACKS);
+ } while (stack < end - EXCEPTION_STKSZ);
+ if (*usedp & (1U << j))
+ break;
+ *usedp |= 1U << j;
+ *idp = ids[j];
+ return (unsigned long *)end;
+ }
+#endif
}
return NULL;
}
@@ -613,6 +641,7 @@ asmlinkage void default_do_nmi(struct pt_regs *regs)
io_check_error(reason, regs);
}
+/* runs on IST stack. */
asmlinkage void __kprobes do_int3(struct pt_regs * regs, long error_code)
{
if (notify_die(DIE_INT3, "int3", regs, error_code, 3, SIGTRAP) == NOTIFY_STOP) {
@@ -894,7 +923,7 @@ void __init trap_init(void)
set_intr_gate(0,&divide_error);
set_intr_gate_ist(1,&debug,DEBUG_STACK);
set_intr_gate_ist(2,&nmi,NMI_STACK);
- set_system_gate(3,&int3);
+ set_system_gate_ist(3,&int3,DEBUG_STACK); /* int3 can be called from all */
set_system_gate(4,&overflow); /* int4 can be called from all */
set_intr_gate(5,&bounds);
set_intr_gate(6,&invalid_op);
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-27 22:52:31 +0100