From c15ac4fd60d5ffdb151bb2c7805f377fd7f90363 Mon Sep 17 00:00:00 2001 From: Alan Kao Date: Tue, 13 Feb 2018 13:13:17 +0800 Subject: riscv/ftrace: Add dynamic function tracer support We now have dynamic ftrace with the following added items: * ftrace_make_call, ftrace_make_nop (in kernel/ftrace.c) The two functions turn each recorded call site of filtered functions into a call to ftrace_caller or nops * ftracce_update_ftrace_func (in kernel/ftrace.c) turns the nops at ftrace_call into a call to a generic entry for function tracers. * ftrace_caller (in kernel/mcount-dyn.S) The entry where each _mcount call sites calls to once they are filtered to be traced. Also, this patch fixes the semantic problems in mcount.S, which will be treated as only a reference implementation once we have the dynamic ftrace. Cc: Greentime Hu Signed-off-by: Alan Kao Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/ftrace.c | 103 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 102 insertions(+), 1 deletion(-) (limited to 'arch/riscv/kernel/ftrace.c') diff --git a/arch/riscv/kernel/ftrace.c b/arch/riscv/kernel/ftrace.c index d0de68d144cb..be4b24332d97 100644 --- a/arch/riscv/kernel/ftrace.c +++ b/arch/riscv/kernel/ftrace.c @@ -6,9 +6,109 @@ */ #include +#include +#include +#ifdef CONFIG_DYNAMIC_FTRACE +static int ftrace_check_current_call(unsigned long hook_pos, + unsigned int *expected) +{ + unsigned int replaced[2]; + unsigned int nops[2] = {NOP4, NOP4}; + + /* we expect nops at the hook position */ + if (!expected) + expected = nops; + + /* + * Read the text we want to modify; + * return must be -EFAULT on read error + */ + if (probe_kernel_read(replaced, (void *)hook_pos, MCOUNT_INSN_SIZE)) + return -EFAULT; + + /* + * Make sure it is what we expect it to be; + * return must be -EINVAL on failed comparison + */ + if (memcmp(expected, replaced, sizeof(replaced))) { + pr_err("%p: expected (%08x %08x) but get (%08x %08x)", + (void *)hook_pos, expected[0], expected[1], replaced[0], + replaced[1]); + return -EINVAL; + } + + return 0; +} + +static int __ftrace_modify_call(unsigned long hook_pos, unsigned long target, + bool enable) +{ + unsigned int call[2]; + unsigned int nops[2] = {NOP4, NOP4}; + int ret = 0; + + make_call(hook_pos, target, call); + + /* replace the auipc-jalr pair at once */ + ret = probe_kernel_write((void *)hook_pos, enable ? call : nops, + MCOUNT_INSN_SIZE); + /* return must be -EPERM on write error */ + if (ret) + return -EPERM; + + smp_mb(); + flush_icache_range((void *)hook_pos, (void *)hook_pos + MCOUNT_INSN_SIZE); + + return 0; +} + +int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) +{ + int ret = ftrace_check_current_call(rec->ip, NULL); + + if (ret) + return ret; + + return __ftrace_modify_call(rec->ip, addr, true); +} + +int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, + unsigned long addr) +{ + unsigned int call[2]; + int ret; + + make_call(rec->ip, addr, call); + ret = ftrace_check_current_call(rec->ip, call); + + if (ret) + return ret; + + return __ftrace_modify_call(rec->ip, addr, false); +} + +int ftrace_update_ftrace_func(ftrace_func_t func) +{ + int ret = __ftrace_modify_call((unsigned long)&ftrace_call, + (unsigned long)func, true); + if (!ret) { + ret = __ftrace_modify_call((unsigned long)&ftrace_regs_call, + (unsigned long)func, true); + } + + return ret; +} + +int __init ftrace_dyn_arch_init(void) +{ + return 0; +} +#endif + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER /* - * Most of this file is copied from arm64. + * Most of this function is copied from arm64. */ void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, unsigned long frame_pointer) @@ -39,3 +139,4 @@ void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, return; *parent = return_hooker; } +#endif -- cgit v1.2.3 From bc1a4c3a842556852bb02039887d73899d513532 Mon Sep 17 00:00:00 2001 From: Alan Kao Date: Tue, 13 Feb 2018 13:13:18 +0800 Subject: riscv/ftrace: Add dynamic function graph tracer support Once the function_graph tracer is enabled, a filtered function has the following call sequence: * ftracer_caller ==> on/off by ftrace_make_call/ftrace_make_nop * ftrace_graph_caller * ftrace_graph_call ==> on/off by ftrace_en/disable_ftrace_graph_caller * prepare_ftrace_return Considering the following DYNAMIC_FTRACE_WITH_REGS feature, it would be more extendable to have a ftrace_graph_caller function, instead of calling prepare_ftrace_return directly in ftrace_caller. Cc: Greentime Hu Signed-off-by: Alan Kao Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/ftrace.c | 55 +++++++++++++++++++++++++++++++++++- arch/riscv/kernel/mcount-dyn.S | 64 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 118 insertions(+), 1 deletion(-) (limited to 'arch/riscv/kernel/ftrace.c') diff --git a/arch/riscv/kernel/ftrace.c b/arch/riscv/kernel/ftrace.c index be4b24332d97..5bbe1afd9463 100644 --- a/arch/riscv/kernel/ftrace.c +++ b/arch/riscv/kernel/ftrace.c @@ -139,4 +139,57 @@ void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, return; *parent = return_hooker; } -#endif + +#ifdef CONFIG_DYNAMIC_FTRACE +extern void ftrace_graph_call(void); +int ftrace_enable_ftrace_graph_caller(void) +{ + unsigned int call[2]; + static int init_graph = 1; + int ret; + + make_call(&ftrace_graph_call, &ftrace_stub, call); + + /* + * When enabling graph tracer for the first time, ftrace_graph_call + * should contains a call to ftrace_stub. Once it has been disabled, + * the 8-bytes at the position becomes NOPs. + */ + if (init_graph) { + ret = ftrace_check_current_call((unsigned long)&ftrace_graph_call, + call); + init_graph = 0; + } else { + ret = ftrace_check_current_call((unsigned long)&ftrace_graph_call, + NULL); + } + + if (ret) + return ret; + + return __ftrace_modify_call((unsigned long)&ftrace_graph_call, + (unsigned long)&prepare_ftrace_return, true); +} + +int ftrace_disable_ftrace_graph_caller(void) +{ + unsigned int call[2]; + int ret; + + make_call(&ftrace_graph_call, &prepare_ftrace_return, call); + + /* + * This is to make sure that ftrace_enable_ftrace_graph_caller + * did the right thing. + */ + ret = ftrace_check_current_call((unsigned long)&ftrace_graph_call, + call); + + if (ret) + return ret; + + return __ftrace_modify_call((unsigned long)&ftrace_graph_call, + (unsigned long)&prepare_ftrace_return, false); +} +#endif /* CONFIG_DYNAMIC_FTRACE */ +#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ diff --git a/arch/riscv/kernel/mcount-dyn.S b/arch/riscv/kernel/mcount-dyn.S index a3ebeadbe698..739e07a6fd85 100644 --- a/arch/riscv/kernel/mcount-dyn.S +++ b/arch/riscv/kernel/mcount-dyn.S @@ -14,18 +14,62 @@ .text .macro SAVE_ABI_STATE +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + addi sp, sp, -48 + sd s0, 32(sp) + sd ra, 40(sp) + addi s0, sp, 48 + sd t0, 24(sp) + sd t1, 16(sp) +#ifdef HAVE_FUNCTION_GRAPH_FP_TEST + sd t2, 8(sp) +#endif +#else addi sp, sp, -16 sd s0, 0(sp) sd ra, 8(sp) addi s0, sp, 16 +#endif .endm .macro RESTORE_ABI_STATE +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + ld s0, 32(sp) + ld ra, 40(sp) + addi sp, sp, 48 +#else ld ra, 8(sp) ld s0, 0(sp) addi sp, sp, 16 +#endif .endm + .macro RESTORE_GRAPH_ARGS + ld a0, 24(sp) + ld a1, 16(sp) +#ifdef HAVE_FUNCTION_GRAPH_FP_TEST + ld a2, 8(sp) +#endif + .endm + +ENTRY(ftrace_graph_caller) + addi sp, sp, -16 + sd s0, 0(sp) + sd ra, 8(sp) + addi s0, sp, 16 +ftrace_graph_call: + .global ftrace_graph_call + /* + * Calling ftrace_enable/disable_ftrace_graph_caller would overwrite the + * call below. Check ftrace_modify_all_code for details. + */ + call ftrace_stub + ld ra, 8(sp) + ld s0, 0(sp) + addi sp, sp, 16 + ret +ENDPROC(ftrace_graph_caller) + ENTRY(ftrace_caller) /* * a0: the address in the caller when calling ftrace_caller @@ -33,6 +77,20 @@ ENTRY(ftrace_caller) */ ld a1, -8(s0) addi a0, ra, -MCOUNT_INSN_SIZE + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + /* + * the graph tracer (specifically, prepare_ftrace_return) needs these + * arguments but for now the function tracer occupies the regs, so we + * save them in temporary regs to recover later. + */ + addi t0, s0, -8 + mv t1, a0 +#ifdef HAVE_FUNCTION_GRAPH_FP_TEST + ld t2, -16(s0) +#endif +#endif + SAVE_ABI_STATE ftrace_call: .global ftrace_call @@ -45,6 +103,12 @@ ftrace_call: * Check ftrace_modify_all_code for details. */ call ftrace_stub + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + RESTORE_GRAPH_ARGS + call ftrace_graph_caller +#endif + RESTORE_ABI_STATE ret ENDPROC(ftrace_caller) -- cgit v1.2.3 From aea4c671fb985e6a9ffc365c43ea6f5e0d737fea Mon Sep 17 00:00:00 2001 From: Alan Kao Date: Tue, 13 Feb 2018 13:13:20 +0800 Subject: riscv/ftrace: Add DYNAMIC_FTRACE_WITH_REGS support Cc: Greentime Hu Signed-off-by: Alan Kao Signed-off-by: Palmer Dabbelt --- arch/riscv/Kconfig | 1 + arch/riscv/include/asm/ftrace.h | 1 + arch/riscv/kernel/ftrace.c | 17 ++++++ arch/riscv/kernel/mcount-dyn.S | 122 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 141 insertions(+) (limited to 'arch/riscv/kernel/ftrace.c') diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 1e9d878c1ac4..61dd82709898 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -116,6 +116,7 @@ config ARCH_RV64I select HAVE_FUNCTION_GRAPH_TRACER select HAVE_FTRACE_MCOUNT_RECORD select HAVE_DYNAMIC_FTRACE + select HAVE_DYNAMIC_FTRACE_WITH_REGS endchoice diff --git a/arch/riscv/include/asm/ftrace.h b/arch/riscv/include/asm/ftrace.h index fedadc40e358..c6dcc5291f97 100644 --- a/arch/riscv/include/asm/ftrace.h +++ b/arch/riscv/include/asm/ftrace.h @@ -8,6 +8,7 @@ #if defined(CONFIG_FUNCTION_GRAPH_TRACER) && defined(CONFIG_FRAME_POINTER) #define HAVE_FUNCTION_GRAPH_FP_TEST #endif +#define HAVE_FUNCTION_GRAPH_RET_ADDR_PTR #define ARCH_SUPPORTS_FTRACE_OPS 1 #ifndef __ASSEMBLY__ diff --git a/arch/riscv/kernel/ftrace.c b/arch/riscv/kernel/ftrace.c index 5bbe1afd9463..48b5353691c3 100644 --- a/arch/riscv/kernel/ftrace.c +++ b/arch/riscv/kernel/ftrace.c @@ -106,6 +106,23 @@ int __init ftrace_dyn_arch_init(void) } #endif +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS +int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, + unsigned long addr) +{ + unsigned int call[2]; + int ret; + + make_call(rec->ip, old_addr, call); + ret = ftrace_check_current_call(rec->ip, call); + + if (ret) + return ret; + + return __ftrace_modify_call(rec->ip, addr, true); +} +#endif + #ifdef CONFIG_FUNCTION_GRAPH_TRACER /* * Most of this function is copied from arm64. diff --git a/arch/riscv/kernel/mcount-dyn.S b/arch/riscv/kernel/mcount-dyn.S index 6bbc3f88fcb3..35a6ed76cb8b 100644 --- a/arch/riscv/kernel/mcount-dyn.S +++ b/arch/riscv/kernel/mcount-dyn.S @@ -115,3 +115,125 @@ ftrace_call: RESTORE_ABI_STATE ret ENDPROC(ftrace_caller) + +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS + .macro SAVE_ALL + addi sp, sp, -(PT_SIZE_ON_STACK+16) + sd s0, (PT_SIZE_ON_STACK)(sp) + sd ra, (PT_SIZE_ON_STACK+8)(sp) + addi s0, sp, (PT_SIZE_ON_STACK+16) + + sd x1, PT_RA(sp) + sd x2, PT_SP(sp) + sd x3, PT_GP(sp) + sd x4, PT_TP(sp) + sd x5, PT_T0(sp) + sd x6, PT_T1(sp) + sd x7, PT_T2(sp) + sd x8, PT_S0(sp) + sd x9, PT_S1(sp) + sd x10, PT_A0(sp) + sd x11, PT_A1(sp) + sd x12, PT_A2(sp) + sd x13, PT_A3(sp) + sd x14, PT_A4(sp) + sd x15, PT_A5(sp) + sd x16, PT_A6(sp) + sd x17, PT_A7(sp) + sd x18, PT_S2(sp) + sd x19, PT_S3(sp) + sd x20, PT_S4(sp) + sd x21, PT_S5(sp) + sd x22, PT_S6(sp) + sd x23, PT_S7(sp) + sd x24, PT_S8(sp) + sd x25, PT_S9(sp) + sd x26, PT_S10(sp) + sd x27, PT_S11(sp) + sd x28, PT_T3(sp) + sd x29, PT_T4(sp) + sd x30, PT_T5(sp) + sd x31, PT_T6(sp) + .endm + + .macro RESTORE_ALL + ld x1, PT_RA(sp) + ld x2, PT_SP(sp) + ld x3, PT_GP(sp) + ld x4, PT_TP(sp) + ld x5, PT_T0(sp) + ld x6, PT_T1(sp) + ld x7, PT_T2(sp) + ld x8, PT_S0(sp) + ld x9, PT_S1(sp) + ld x10, PT_A0(sp) + ld x11, PT_A1(sp) + ld x12, PT_A2(sp) + ld x13, PT_A3(sp) + ld x14, PT_A4(sp) + ld x15, PT_A5(sp) + ld x16, PT_A6(sp) + ld x17, PT_A7(sp) + ld x18, PT_S2(sp) + ld x19, PT_S3(sp) + ld x20, PT_S4(sp) + ld x21, PT_S5(sp) + ld x22, PT_S6(sp) + ld x23, PT_S7(sp) + ld x24, PT_S8(sp) + ld x25, PT_S9(sp) + ld x26, PT_S10(sp) + ld x27, PT_S11(sp) + ld x28, PT_T3(sp) + ld x29, PT_T4(sp) + ld x30, PT_T5(sp) + ld x31, PT_T6(sp) + + ld s0, (PT_SIZE_ON_STACK)(sp) + ld ra, (PT_SIZE_ON_STACK+8)(sp) + addi sp, sp, (PT_SIZE_ON_STACK+16) + .endm + + .macro RESTORE_GRAPH_REG_ARGS + ld a0, PT_T0(sp) + ld a1, PT_T1(sp) +#ifdef HAVE_FUNCTION_GRAPH_FP_TEST + ld a2, PT_T2(sp) +#endif + .endm + +/* + * Most of the contents are the same as ftrace_caller. + */ +ENTRY(ftrace_regs_caller) + /* + * a3: the address of all registers in the stack + */ + ld a1, -8(s0) + addi a0, ra, -MCOUNT_INSN_SIZE + la t5, function_trace_op + ld a2, 0(t5) + addi a3, sp, -(PT_SIZE_ON_STACK+16) + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + addi t0, s0, -8 + mv t1, a0 +#ifdef HAVE_FUNCTION_GRAPH_FP_TEST + ld t2, -16(s0) +#endif +#endif + SAVE_ALL + +ftrace_regs_call: + .global ftrace_regs_call + call ftrace_stub + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + RESTORE_GRAPH_REG_ARGS + call ftrace_graph_caller +#endif + + RESTORE_ALL + ret +ENDPROC(ftrace_regs_caller) +#endif /* CONFIG_DYNAMIC_FTRACE_WITH_REGS */ -- cgit v1.2.3 From b785ec129bd9498ecd6e221e45820b00c6ed4adf Mon Sep 17 00:00:00 2001 From: Alan Kao Date: Tue, 13 Feb 2018 13:13:21 +0800 Subject: riscv/ftrace: Add HAVE_FUNCTION_GRAPH_RET_ADDR_PTR support In walk_stackframe, the pc now receives the address from calling ftrace_graph_ret_addr instead of manual calculation. Note that the original calculation, pc = frame->ra - 4 is buggy when the instruction at the return address happened to be a compressed inst. But since it is not a critical part of ftrace, it is ignored for now to ease the review process. Cc: Greentime Hu Signed-off-by: Alan Kao Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/ftrace.c | 2 +- arch/riscv/kernel/stacktrace.c | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) (limited to 'arch/riscv/kernel/ftrace.c') diff --git a/arch/riscv/kernel/ftrace.c b/arch/riscv/kernel/ftrace.c index 48b5353691c3..1157b6b52d25 100644 --- a/arch/riscv/kernel/ftrace.c +++ b/arch/riscv/kernel/ftrace.c @@ -151,7 +151,7 @@ void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, return; err = ftrace_push_return_trace(old, self_addr, &trace.depth, - frame_pointer, NULL); + frame_pointer, parent); if (err == -EBUSY) return; *parent = return_hooker; diff --git a/arch/riscv/kernel/stacktrace.c b/arch/riscv/kernel/stacktrace.c index 559aae781154..a4b1d94371a0 100644 --- a/arch/riscv/kernel/stacktrace.c +++ b/arch/riscv/kernel/stacktrace.c @@ -18,6 +18,7 @@ #include #include #include +#include #ifdef CONFIG_FRAME_POINTER @@ -63,7 +64,12 @@ static void notrace walk_stackframe(struct task_struct *task, frame = (struct stackframe *)fp - 1; sp = fp; fp = frame->fp; +#ifdef HAVE_FUNCTION_GRAPH_RET_ADDR_PTR + pc = ftrace_graph_ret_addr(current, NULL, frame->ra, + (unsigned long *)(fp - 8)); +#else pc = frame->ra - 0x4; +#endif } } -- cgit v1.2.3