diff options
49 files changed, 2494 insertions, 1644 deletions
diff --git a/Documentation/trace/kprobetrace.rst b/Documentation/trace/kprobetrace.rst index 47e765c2f2c3..235ce2ab131a 100644 --- a/Documentation/trace/kprobetrace.rst +++ b/Documentation/trace/kprobetrace.rst | |||
@@ -20,6 +20,9 @@ current_tracer. Instead of that, add probe points via | |||
20 | /sys/kernel/debug/tracing/kprobe_events, and enable it via | 20 | /sys/kernel/debug/tracing/kprobe_events, and enable it via |
21 | /sys/kernel/debug/tracing/events/kprobes/<EVENT>/enable. | 21 | /sys/kernel/debug/tracing/events/kprobes/<EVENT>/enable. |
22 | 22 | ||
23 | You can also use /sys/kernel/debug/tracing/dynamic_events instead of | ||
24 | kprobe_events. That interface will provide unified access to other | ||
25 | dynamic events too. | ||
23 | 26 | ||
24 | Synopsis of kprobe_events | 27 | Synopsis of kprobe_events |
25 | ------------------------- | 28 | ------------------------- |
diff --git a/Documentation/trace/uprobetracer.rst b/Documentation/trace/uprobetracer.rst index d0822811527a..4c3bfde2ba47 100644 --- a/Documentation/trace/uprobetracer.rst +++ b/Documentation/trace/uprobetracer.rst | |||
@@ -18,6 +18,10 @@ current_tracer. Instead of that, add probe points via | |||
18 | However unlike kprobe-event tracer, the uprobe event interface expects the | 18 | However unlike kprobe-event tracer, the uprobe event interface expects the |
19 | user to calculate the offset of the probepoint in the object. | 19 | user to calculate the offset of the probepoint in the object. |
20 | 20 | ||
21 | You can also use /sys/kernel/debug/tracing/dynamic_events instead of | ||
22 | uprobe_events. That interface will provide unified access to other | ||
23 | dynamic events too. | ||
24 | |||
21 | Synopsis of uprobe_tracer | 25 | Synopsis of uprobe_tracer |
22 | ------------------------- | 26 | ------------------------- |
23 | :: | 27 | :: |
diff --git a/arch/arm64/kernel/ftrace.c b/arch/arm64/kernel/ftrace.c index c1f30f854fb3..8e4431a8821f 100644 --- a/arch/arm64/kernel/ftrace.c +++ b/arch/arm64/kernel/ftrace.c | |||
@@ -193,6 +193,7 @@ int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, | |||
193 | 193 | ||
194 | void arch_ftrace_update_code(int command) | 194 | void arch_ftrace_update_code(int command) |
195 | { | 195 | { |
196 | command |= FTRACE_MAY_SLEEP; | ||
196 | ftrace_modify_all_code(command); | 197 | ftrace_modify_all_code(command); |
197 | } | 198 | } |
198 | 199 | ||
diff --git a/arch/arm64/kernel/perf_callchain.c b/arch/arm64/kernel/perf_callchain.c index 94754f07f67a..a34c26afacb0 100644 --- a/arch/arm64/kernel/perf_callchain.c +++ b/arch/arm64/kernel/perf_callchain.c | |||
@@ -168,7 +168,7 @@ void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, | |||
168 | frame.fp = regs->regs[29]; | 168 | frame.fp = regs->regs[29]; |
169 | frame.pc = regs->pc; | 169 | frame.pc = regs->pc; |
170 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | 170 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
171 | frame.graph = current->curr_ret_stack; | 171 | frame.graph = 0; |
172 | #endif | 172 | #endif |
173 | 173 | ||
174 | walk_stackframe(current, &frame, callchain_trace, entry); | 174 | walk_stackframe(current, &frame, callchain_trace, entry); |
diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index e0a443730e04..a0f985a6ac50 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c | |||
@@ -461,7 +461,7 @@ unsigned long get_wchan(struct task_struct *p) | |||
461 | frame.fp = thread_saved_fp(p); | 461 | frame.fp = thread_saved_fp(p); |
462 | frame.pc = thread_saved_pc(p); | 462 | frame.pc = thread_saved_pc(p); |
463 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | 463 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
464 | frame.graph = p->curr_ret_stack; | 464 | frame.graph = 0; |
465 | #endif | 465 | #endif |
466 | do { | 466 | do { |
467 | if (unwind_frame(p, &frame)) | 467 | if (unwind_frame(p, &frame)) |
diff --git a/arch/arm64/kernel/return_address.c b/arch/arm64/kernel/return_address.c index 933adbc0f654..53c40196b607 100644 --- a/arch/arm64/kernel/return_address.c +++ b/arch/arm64/kernel/return_address.c | |||
@@ -44,7 +44,7 @@ void *return_address(unsigned int level) | |||
44 | frame.fp = (unsigned long)__builtin_frame_address(0); | 44 | frame.fp = (unsigned long)__builtin_frame_address(0); |
45 | frame.pc = (unsigned long)return_address; /* dummy */ | 45 | frame.pc = (unsigned long)return_address; /* dummy */ |
46 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | 46 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
47 | frame.graph = current->curr_ret_stack; | 47 | frame.graph = 0; |
48 | #endif | 48 | #endif |
49 | 49 | ||
50 | walk_stackframe(current, &frame, save_return_addr, &data); | 50 | walk_stackframe(current, &frame, save_return_addr, &data); |
diff --git a/arch/arm64/kernel/stacktrace.c b/arch/arm64/kernel/stacktrace.c index 4989f7ea1e59..1a29f2695ff2 100644 --- a/arch/arm64/kernel/stacktrace.c +++ b/arch/arm64/kernel/stacktrace.c | |||
@@ -59,18 +59,17 @@ int notrace unwind_frame(struct task_struct *tsk, struct stackframe *frame) | |||
59 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | 59 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
60 | if (tsk->ret_stack && | 60 | if (tsk->ret_stack && |
61 | (frame->pc == (unsigned long)return_to_handler)) { | 61 | (frame->pc == (unsigned long)return_to_handler)) { |
62 | if (WARN_ON_ONCE(frame->graph == -1)) | 62 | struct ftrace_ret_stack *ret_stack; |
63 | return -EINVAL; | ||
64 | if (frame->graph < -1) | ||
65 | frame->graph += FTRACE_NOTRACE_DEPTH; | ||
66 | |||
67 | /* | 63 | /* |
68 | * This is a case where function graph tracer has | 64 | * This is a case where function graph tracer has |
69 | * modified a return address (LR) in a stack frame | 65 | * modified a return address (LR) in a stack frame |
70 | * to hook a function return. | 66 | * to hook a function return. |
71 | * So replace it to an original value. | 67 | * So replace it to an original value. |
72 | */ | 68 | */ |
73 | frame->pc = tsk->ret_stack[frame->graph--].ret; | 69 | ret_stack = ftrace_graph_get_ret_stack(tsk, frame->graph++); |
70 | if (WARN_ON_ONCE(!ret_stack)) | ||
71 | return -EINVAL; | ||
72 | frame->pc = ret_stack->ret; | ||
74 | } | 73 | } |
75 | #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ | 74 | #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ |
76 | 75 | ||
@@ -137,7 +136,7 @@ void save_stack_trace_regs(struct pt_regs *regs, struct stack_trace *trace) | |||
137 | frame.fp = regs->regs[29]; | 136 | frame.fp = regs->regs[29]; |
138 | frame.pc = regs->pc; | 137 | frame.pc = regs->pc; |
139 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | 138 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
140 | frame.graph = current->curr_ret_stack; | 139 | frame.graph = 0; |
141 | #endif | 140 | #endif |
142 | 141 | ||
143 | walk_stackframe(current, &frame, save_trace, &data); | 142 | walk_stackframe(current, &frame, save_trace, &data); |
@@ -168,7 +167,7 @@ static noinline void __save_stack_trace(struct task_struct *tsk, | |||
168 | frame.pc = (unsigned long)__save_stack_trace; | 167 | frame.pc = (unsigned long)__save_stack_trace; |
169 | } | 168 | } |
170 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | 169 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
171 | frame.graph = tsk->curr_ret_stack; | 170 | frame.graph = 0; |
172 | #endif | 171 | #endif |
173 | 172 | ||
174 | walk_stackframe(tsk, &frame, save_trace, &data); | 173 | walk_stackframe(tsk, &frame, save_trace, &data); |
diff --git a/arch/arm64/kernel/time.c b/arch/arm64/kernel/time.c index f258636273c9..a777ae90044d 100644 --- a/arch/arm64/kernel/time.c +++ b/arch/arm64/kernel/time.c | |||
@@ -52,7 +52,7 @@ unsigned long profile_pc(struct pt_regs *regs) | |||
52 | frame.fp = regs->regs[29]; | 52 | frame.fp = regs->regs[29]; |
53 | frame.pc = regs->pc; | 53 | frame.pc = regs->pc; |
54 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | 54 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
55 | frame.graph = current->curr_ret_stack; | 55 | frame.graph = 0; |
56 | #endif | 56 | #endif |
57 | do { | 57 | do { |
58 | int ret = unwind_frame(NULL, &frame); | 58 | int ret = unwind_frame(NULL, &frame); |
diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c index cdc71cf70aad..4e2fb877f8d5 100644 --- a/arch/arm64/kernel/traps.c +++ b/arch/arm64/kernel/traps.c | |||
@@ -123,7 +123,7 @@ void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk) | |||
123 | frame.pc = thread_saved_pc(tsk); | 123 | frame.pc = thread_saved_pc(tsk); |
124 | } | 124 | } |
125 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | 125 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
126 | frame.graph = tsk->curr_ret_stack; | 126 | frame.graph = 0; |
127 | #endif | 127 | #endif |
128 | 128 | ||
129 | skip = !!regs; | 129 | skip = !!regs; |
diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c index 96f34730010f..ce393df243aa 100644 --- a/arch/powerpc/kernel/process.c +++ b/arch/powerpc/kernel/process.c | |||
@@ -2061,9 +2061,10 @@ void show_stack(struct task_struct *tsk, unsigned long *stack) | |||
2061 | int count = 0; | 2061 | int count = 0; |
2062 | int firstframe = 1; | 2062 | int firstframe = 1; |
2063 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | 2063 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
2064 | int curr_frame = current->curr_ret_stack; | 2064 | struct ftrace_ret_stack *ret_stack; |
2065 | extern void return_to_handler(void); | 2065 | extern void return_to_handler(void); |
2066 | unsigned long rth = (unsigned long)return_to_handler; | 2066 | unsigned long rth = (unsigned long)return_to_handler; |
2067 | int curr_frame = 0; | ||
2067 | #endif | 2068 | #endif |
2068 | 2069 | ||
2069 | sp = (unsigned long) stack; | 2070 | sp = (unsigned long) stack; |
@@ -2089,9 +2090,13 @@ void show_stack(struct task_struct *tsk, unsigned long *stack) | |||
2089 | printk("["REG"] ["REG"] %pS", sp, ip, (void *)ip); | 2090 | printk("["REG"] ["REG"] %pS", sp, ip, (void *)ip); |
2090 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | 2091 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
2091 | if ((ip == rth) && curr_frame >= 0) { | 2092 | if ((ip == rth) && curr_frame >= 0) { |
2092 | pr_cont(" (%pS)", | 2093 | ret_stack = ftrace_graph_get_ret_stack(current, |
2093 | (void *)current->ret_stack[curr_frame].ret); | 2094 | curr_frame++); |
2094 | curr_frame--; | 2095 | if (ret_stack) |
2096 | pr_cont(" (%pS)", | ||
2097 | (void *)ret_stack->ret); | ||
2098 | else | ||
2099 | curr_frame = -1; | ||
2095 | } | 2100 | } |
2096 | #endif | 2101 | #endif |
2097 | if (firstframe) | 2102 | if (firstframe) |
diff --git a/arch/sh/kernel/dumpstack.c b/arch/sh/kernel/dumpstack.c index 93c6c0e691ee..9f1c9c11d62d 100644 --- a/arch/sh/kernel/dumpstack.c +++ b/arch/sh/kernel/dumpstack.c | |||
@@ -56,17 +56,20 @@ print_ftrace_graph_addr(unsigned long addr, void *data, | |||
56 | struct thread_info *tinfo, int *graph) | 56 | struct thread_info *tinfo, int *graph) |
57 | { | 57 | { |
58 | struct task_struct *task = tinfo->task; | 58 | struct task_struct *task = tinfo->task; |
59 | struct ftrace_ret_stack *ret_stack; | ||
59 | unsigned long ret_addr; | 60 | unsigned long ret_addr; |
60 | int index = task->curr_ret_stack; | ||
61 | 61 | ||
62 | if (addr != (unsigned long)return_to_handler) | 62 | if (addr != (unsigned long)return_to_handler) |
63 | return; | 63 | return; |
64 | 64 | ||
65 | if (!task->ret_stack || index < *graph) | 65 | if (!task->ret_stack) |
66 | return; | 66 | return; |
67 | 67 | ||
68 | index -= *graph; | 68 | ret_stack = ftrace_graph_get_ret_stack(task, *graph); |
69 | ret_addr = task->ret_stack[index].ret; | 69 | if (!ret_stack) |
70 | return; | ||
71 | |||
72 | ret_addr = ret_stack->ret; | ||
70 | 73 | ||
71 | ops->address(data, ret_addr, 1); | 74 | ops->address(data, ret_addr, 1); |
72 | 75 | ||
diff --git a/arch/sh/kernel/dwarf.c b/arch/sh/kernel/dwarf.c index 9e1d26c8a0c4..c5b426506d16 100644 --- a/arch/sh/kernel/dwarf.c +++ b/arch/sh/kernel/dwarf.c | |||
@@ -605,17 +605,18 @@ struct dwarf_frame *dwarf_unwind_stack(unsigned long pc, | |||
605 | * expected to find the real return address. | 605 | * expected to find the real return address. |
606 | */ | 606 | */ |
607 | if (pc == (unsigned long)&return_to_handler) { | 607 | if (pc == (unsigned long)&return_to_handler) { |
608 | int index = current->curr_ret_stack; | 608 | struct ftrace_ret_stack *ret_stack; |
609 | 609 | ||
610 | ret_stack = ftrace_graph_get_ret_stack(current, 0); | ||
611 | if (ret_stack) | ||
612 | pc = ret_stack->ret; | ||
610 | /* | 613 | /* |
611 | * We currently have no way of tracking how many | 614 | * We currently have no way of tracking how many |
612 | * return_to_handler()'s we've seen. If there is more | 615 | * return_to_handler()'s we've seen. If there is more |
613 | * than one patched return address on our stack, | 616 | * than one patched return address on our stack, |
614 | * complain loudly. | 617 | * complain loudly. |
615 | */ | 618 | */ |
616 | WARN_ON(index > 0); | 619 | WARN_ON(ftrace_graph_get_ret_stack(current, 1); |
617 | |||
618 | pc = current->ret_stack[index].ret; | ||
619 | } | 620 | } |
620 | #endif | 621 | #endif |
621 | 622 | ||
diff --git a/arch/sparc/kernel/perf_event.c b/arch/sparc/kernel/perf_event.c index 47c871394ccb..6de7c684c29f 100644 --- a/arch/sparc/kernel/perf_event.c +++ b/arch/sparc/kernel/perf_event.c | |||
@@ -1767,9 +1767,11 @@ void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, | |||
1767 | perf_callchain_store(entry, pc); | 1767 | perf_callchain_store(entry, pc); |
1768 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | 1768 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
1769 | if ((pc + 8UL) == (unsigned long) &return_to_handler) { | 1769 | if ((pc + 8UL) == (unsigned long) &return_to_handler) { |
1770 | int index = current->curr_ret_stack; | 1770 | struct ftrace_ret_stack *ret_stack; |
1771 | if (current->ret_stack && index >= graph) { | 1771 | ret_stack = ftrace_graph_get_ret_stack(current, |
1772 | pc = current->ret_stack[index - graph].ret; | 1772 | graph); |
1773 | if (ret_stack) { | ||
1774 | pc = ret_stack->ret; | ||
1773 | perf_callchain_store(entry, pc); | 1775 | perf_callchain_store(entry, pc); |
1774 | graph++; | 1776 | graph++; |
1775 | } | 1777 | } |
diff --git a/arch/sparc/kernel/stacktrace.c b/arch/sparc/kernel/stacktrace.c index be4c14cccc05..dd654e651500 100644 --- a/arch/sparc/kernel/stacktrace.c +++ b/arch/sparc/kernel/stacktrace.c | |||
@@ -57,9 +57,11 @@ static void __save_stack_trace(struct thread_info *tp, | |||
57 | trace->entries[trace->nr_entries++] = pc; | 57 | trace->entries[trace->nr_entries++] = pc; |
58 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | 58 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
59 | if ((pc + 8UL) == (unsigned long) &return_to_handler) { | 59 | if ((pc + 8UL) == (unsigned long) &return_to_handler) { |
60 | int index = t->curr_ret_stack; | 60 | struct ftrace_ret_stack *ret_stack; |
61 | if (t->ret_stack && index >= graph) { | 61 | ret_stack = ftrace_graph_get_ret_stack(t, |
62 | pc = t->ret_stack[index - graph].ret; | 62 | graph); |
63 | if (ret_stack) { | ||
64 | pc = ret_stack->ret; | ||
63 | if (trace->nr_entries < | 65 | if (trace->nr_entries < |
64 | trace->max_entries) | 66 | trace->max_entries) |
65 | trace->entries[trace->nr_entries++] = pc; | 67 | trace->entries[trace->nr_entries++] = pc; |
diff --git a/arch/sparc/kernel/traps_64.c b/arch/sparc/kernel/traps_64.c index aa624ed79db1..0cd02a64a451 100644 --- a/arch/sparc/kernel/traps_64.c +++ b/arch/sparc/kernel/traps_64.c | |||
@@ -2502,9 +2502,10 @@ void show_stack(struct task_struct *tsk, unsigned long *_ksp) | |||
2502 | printk(" [%016lx] %pS\n", pc, (void *) pc); | 2502 | printk(" [%016lx] %pS\n", pc, (void *) pc); |
2503 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | 2503 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
2504 | if ((pc + 8UL) == (unsigned long) &return_to_handler) { | 2504 | if ((pc + 8UL) == (unsigned long) &return_to_handler) { |
2505 | int index = tsk->curr_ret_stack; | 2505 | struct ftrace_ret_stack *ret_stack; |
2506 | if (tsk->ret_stack && index >= graph) { | 2506 | ret_stack = ftrace_graph_get_ret_stack(tsk, graph); |
2507 | pc = tsk->ret_stack[index - graph].ret; | 2507 | if (ret_stack) { |
2508 | pc = ret_stack->ret; | ||
2508 | printk(" [%016lx] %pS\n", pc, (void *) pc); | 2509 | printk(" [%016lx] %pS\n", pc, (void *) pc); |
2509 | graph++; | 2510 | graph++; |
2510 | } | 2511 | } |
diff --git a/arch/x86/kernel/ftrace.c b/arch/x86/kernel/ftrace.c index 7ee8067cbf45..8257a59704ae 100644 --- a/arch/x86/kernel/ftrace.c +++ b/arch/x86/kernel/ftrace.c | |||
@@ -733,18 +733,20 @@ union ftrace_op_code_union { | |||
733 | } __attribute__((packed)); | 733 | } __attribute__((packed)); |
734 | }; | 734 | }; |
735 | 735 | ||
736 | #define RET_SIZE 1 | ||
737 | |||
736 | static unsigned long | 738 | static unsigned long |
737 | create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size) | 739 | create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size) |
738 | { | 740 | { |
739 | unsigned const char *jmp; | ||
740 | unsigned long start_offset; | 741 | unsigned long start_offset; |
741 | unsigned long end_offset; | 742 | unsigned long end_offset; |
742 | unsigned long op_offset; | 743 | unsigned long op_offset; |
743 | unsigned long offset; | 744 | unsigned long offset; |
744 | unsigned long size; | 745 | unsigned long size; |
745 | unsigned long ip; | 746 | unsigned long retq; |
746 | unsigned long *ptr; | 747 | unsigned long *ptr; |
747 | void *trampoline; | 748 | void *trampoline; |
749 | void *ip; | ||
748 | /* 48 8b 15 <offset> is movq <offset>(%rip), %rdx */ | 750 | /* 48 8b 15 <offset> is movq <offset>(%rip), %rdx */ |
749 | unsigned const char op_ref[] = { 0x48, 0x8b, 0x15 }; | 751 | unsigned const char op_ref[] = { 0x48, 0x8b, 0x15 }; |
750 | union ftrace_op_code_union op_ptr; | 752 | union ftrace_op_code_union op_ptr; |
@@ -764,27 +766,27 @@ create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size) | |||
764 | 766 | ||
765 | /* | 767 | /* |
766 | * Allocate enough size to store the ftrace_caller code, | 768 | * Allocate enough size to store the ftrace_caller code, |
767 | * the jmp to ftrace_epilogue, as well as the address of | 769 | * the iret , as well as the address of the ftrace_ops this |
768 | * the ftrace_ops this trampoline is used for. | 770 | * trampoline is used for. |
769 | */ | 771 | */ |
770 | trampoline = alloc_tramp(size + MCOUNT_INSN_SIZE + sizeof(void *)); | 772 | trampoline = alloc_tramp(size + RET_SIZE + sizeof(void *)); |
771 | if (!trampoline) | 773 | if (!trampoline) |
772 | return 0; | 774 | return 0; |
773 | 775 | ||
774 | *tramp_size = size + MCOUNT_INSN_SIZE + sizeof(void *); | 776 | *tramp_size = size + RET_SIZE + sizeof(void *); |
775 | 777 | ||
776 | /* Copy ftrace_caller onto the trampoline memory */ | 778 | /* Copy ftrace_caller onto the trampoline memory */ |
777 | ret = probe_kernel_read(trampoline, (void *)start_offset, size); | 779 | ret = probe_kernel_read(trampoline, (void *)start_offset, size); |
778 | if (WARN_ON(ret < 0)) { | 780 | if (WARN_ON(ret < 0)) |
779 | tramp_free(trampoline, *tramp_size); | 781 | goto fail; |
780 | return 0; | ||
781 | } | ||
782 | 782 | ||
783 | ip = (unsigned long)trampoline + size; | 783 | ip = trampoline + size; |
784 | 784 | ||
785 | /* The trampoline ends with a jmp to ftrace_epilogue */ | 785 | /* The trampoline ends with ret(q) */ |
786 | jmp = ftrace_jmp_replace(ip, (unsigned long)ftrace_epilogue); | 786 | retq = (unsigned long)ftrace_stub; |
787 | memcpy(trampoline + size, jmp, MCOUNT_INSN_SIZE); | 787 | ret = probe_kernel_read(ip, (void *)retq, RET_SIZE); |
788 | if (WARN_ON(ret < 0)) | ||
789 | goto fail; | ||
788 | 790 | ||
789 | /* | 791 | /* |
790 | * The address of the ftrace_ops that is used for this trampoline | 792 | * The address of the ftrace_ops that is used for this trampoline |
@@ -794,17 +796,15 @@ create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size) | |||
794 | * the global function_trace_op variable. | 796 | * the global function_trace_op variable. |
795 | */ | 797 | */ |
796 | 798 | ||
797 | ptr = (unsigned long *)(trampoline + size + MCOUNT_INSN_SIZE); | 799 | ptr = (unsigned long *)(trampoline + size + RET_SIZE); |
798 | *ptr = (unsigned long)ops; | 800 | *ptr = (unsigned long)ops; |
799 | 801 | ||
800 | op_offset -= start_offset; | 802 | op_offset -= start_offset; |
801 | memcpy(&op_ptr, trampoline + op_offset, OP_REF_SIZE); | 803 | memcpy(&op_ptr, trampoline + op_offset, OP_REF_SIZE); |
802 | 804 | ||
803 | /* Are we pointing to the reference? */ | 805 | /* Are we pointing to the reference? */ |
804 | if (WARN_ON(memcmp(op_ptr.op, op_ref, 3) != 0)) { | 806 | if (WARN_ON(memcmp(op_ptr.op, op_ref, 3) != 0)) |
805 | tramp_free(trampoline, *tramp_size); | 807 | goto fail; |
806 | return 0; | ||
807 | } | ||
808 | 808 | ||
809 | /* Load the contents of ptr into the callback parameter */ | 809 | /* Load the contents of ptr into the callback parameter */ |
810 | offset = (unsigned long)ptr; | 810 | offset = (unsigned long)ptr; |
@@ -819,6 +819,9 @@ create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size) | |||
819 | ops->flags |= FTRACE_OPS_FL_ALLOC_TRAMP; | 819 | ops->flags |= FTRACE_OPS_FL_ALLOC_TRAMP; |
820 | 820 | ||
821 | return (unsigned long)trampoline; | 821 | return (unsigned long)trampoline; |
822 | fail: | ||
823 | tramp_free(trampoline, *tramp_size); | ||
824 | return 0; | ||
822 | } | 825 | } |
823 | 826 | ||
824 | static unsigned long calc_trampoline_call_offset(bool save_regs) | 827 | static unsigned long calc_trampoline_call_offset(bool save_regs) |
diff --git a/arch/x86/kernel/ftrace_64.S b/arch/x86/kernel/ftrace_64.S index 91b2cff4b79a..75f2b36b41a6 100644 --- a/arch/x86/kernel/ftrace_64.S +++ b/arch/x86/kernel/ftrace_64.S | |||
@@ -171,9 +171,6 @@ GLOBAL(ftrace_call) | |||
171 | restore_mcount_regs | 171 | restore_mcount_regs |
172 | 172 | ||
173 | /* | 173 | /* |
174 | * The copied trampoline must call ftrace_epilogue as it | ||
175 | * still may need to call the function graph tracer. | ||
176 | * | ||
177 | * The code up to this label is copied into trampolines so | 174 | * The code up to this label is copied into trampolines so |
178 | * think twice before adding any new code or changing the | 175 | * think twice before adding any new code or changing the |
179 | * layout here. | 176 | * layout here. |
@@ -185,7 +182,10 @@ GLOBAL(ftrace_graph_call) | |||
185 | jmp ftrace_stub | 182 | jmp ftrace_stub |
186 | #endif | 183 | #endif |
187 | 184 | ||
188 | /* This is weak to keep gas from relaxing the jumps */ | 185 | /* |
186 | * This is weak to keep gas from relaxing the jumps. | ||
187 | * It is also used to copy the retq for trampolines. | ||
188 | */ | ||
189 | WEAK(ftrace_stub) | 189 | WEAK(ftrace_stub) |
190 | retq | 190 | retq |
191 | ENDPROC(ftrace_caller) | 191 | ENDPROC(ftrace_caller) |
diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index 5c990e891d6a..730876187344 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h | |||
@@ -389,6 +389,7 @@ enum { | |||
389 | FTRACE_UPDATE_TRACE_FUNC = (1 << 2), | 389 | FTRACE_UPDATE_TRACE_FUNC = (1 << 2), |
390 | FTRACE_START_FUNC_RET = (1 << 3), | 390 | FTRACE_START_FUNC_RET = (1 << 3), |
391 | FTRACE_STOP_FUNC_RET = (1 << 4), | 391 | FTRACE_STOP_FUNC_RET = (1 << 4), |
392 | FTRACE_MAY_SLEEP = (1 << 5), | ||
392 | }; | 393 | }; |
393 | 394 | ||
394 | /* | 395 | /* |
@@ -752,6 +753,11 @@ typedef int (*trace_func_graph_ent_t)(struct ftrace_graph_ent *); /* entry */ | |||
752 | 753 | ||
753 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | 754 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
754 | 755 | ||
756 | struct fgraph_ops { | ||
757 | trace_func_graph_ent_t entryfunc; | ||
758 | trace_func_graph_ret_t retfunc; | ||
759 | }; | ||
760 | |||
755 | /* | 761 | /* |
756 | * Stack of return addresses for functions | 762 | * Stack of return addresses for functions |
757 | * of a thread. | 763 | * of a thread. |
@@ -783,6 +789,9 @@ extern int | |||
783 | function_graph_enter(unsigned long ret, unsigned long func, | 789 | function_graph_enter(unsigned long ret, unsigned long func, |
784 | unsigned long frame_pointer, unsigned long *retp); | 790 | unsigned long frame_pointer, unsigned long *retp); |
785 | 791 | ||
792 | struct ftrace_ret_stack * | ||
793 | ftrace_graph_get_ret_stack(struct task_struct *task, int idx); | ||
794 | |||
786 | unsigned long ftrace_graph_ret_addr(struct task_struct *task, int *idx, | 795 | unsigned long ftrace_graph_ret_addr(struct task_struct *task, int *idx, |
787 | unsigned long ret, unsigned long *retp); | 796 | unsigned long ret, unsigned long *retp); |
788 | 797 | ||
@@ -793,11 +802,11 @@ unsigned long ftrace_graph_ret_addr(struct task_struct *task, int *idx, | |||
793 | */ | 802 | */ |
794 | #define __notrace_funcgraph notrace | 803 | #define __notrace_funcgraph notrace |
795 | 804 | ||
796 | #define FTRACE_NOTRACE_DEPTH 65536 | ||
797 | #define FTRACE_RETFUNC_DEPTH 50 | 805 | #define FTRACE_RETFUNC_DEPTH 50 |
798 | #define FTRACE_RETSTACK_ALLOC_SIZE 32 | 806 | #define FTRACE_RETSTACK_ALLOC_SIZE 32 |
799 | extern int register_ftrace_graph(trace_func_graph_ret_t retfunc, | 807 | |
800 | trace_func_graph_ent_t entryfunc); | 808 | extern int register_ftrace_graph(struct fgraph_ops *ops); |
809 | extern void unregister_ftrace_graph(struct fgraph_ops *ops); | ||
801 | 810 | ||
802 | extern bool ftrace_graph_is_dead(void); | 811 | extern bool ftrace_graph_is_dead(void); |
803 | extern void ftrace_graph_stop(void); | 812 | extern void ftrace_graph_stop(void); |
@@ -806,17 +815,10 @@ extern void ftrace_graph_stop(void); | |||
806 | extern trace_func_graph_ret_t ftrace_graph_return; | 815 | extern trace_func_graph_ret_t ftrace_graph_return; |
807 | extern trace_func_graph_ent_t ftrace_graph_entry; | 816 | extern trace_func_graph_ent_t ftrace_graph_entry; |
808 | 817 | ||
809 | extern void unregister_ftrace_graph(void); | ||
810 | |||
811 | extern void ftrace_graph_init_task(struct task_struct *t); | 818 | extern void ftrace_graph_init_task(struct task_struct *t); |
812 | extern void ftrace_graph_exit_task(struct task_struct *t); | 819 | extern void ftrace_graph_exit_task(struct task_struct *t); |
813 | extern void ftrace_graph_init_idle_task(struct task_struct *t, int cpu); | 820 | extern void ftrace_graph_init_idle_task(struct task_struct *t, int cpu); |
814 | 821 | ||
815 | static inline int task_curr_ret_stack(struct task_struct *t) | ||
816 | { | ||
817 | return t->curr_ret_stack; | ||
818 | } | ||
819 | |||
820 | static inline void pause_graph_tracing(void) | 822 | static inline void pause_graph_tracing(void) |
821 | { | 823 | { |
822 | atomic_inc(¤t->tracing_graph_pause); | 824 | atomic_inc(¤t->tracing_graph_pause); |
@@ -834,17 +836,9 @@ static inline void ftrace_graph_init_task(struct task_struct *t) { } | |||
834 | static inline void ftrace_graph_exit_task(struct task_struct *t) { } | 836 | static inline void ftrace_graph_exit_task(struct task_struct *t) { } |
835 | static inline void ftrace_graph_init_idle_task(struct task_struct *t, int cpu) { } | 837 | static inline void ftrace_graph_init_idle_task(struct task_struct *t, int cpu) { } |
836 | 838 | ||
837 | static inline int register_ftrace_graph(trace_func_graph_ret_t retfunc, | 839 | /* Define as macros as fgraph_ops may not be defined */ |
838 | trace_func_graph_ent_t entryfunc) | 840 | #define register_ftrace_graph(ops) ({ -1; }) |
839 | { | 841 | #define unregister_ftrace_graph(ops) do { } while (0) |
840 | return -1; | ||
841 | } | ||
842 | static inline void unregister_ftrace_graph(void) { } | ||
843 | |||
844 | static inline int task_curr_ret_stack(struct task_struct *tsk) | ||
845 | { | ||
846 | return -1; | ||
847 | } | ||
848 | 842 | ||
849 | static inline unsigned long | 843 | static inline unsigned long |
850 | ftrace_graph_ret_addr(struct task_struct *task, int *idx, unsigned long ret, | 844 | ftrace_graph_ret_addr(struct task_struct *task, int *idx, unsigned long ret, |
diff --git a/include/linux/ring_buffer.h b/include/linux/ring_buffer.h index 0940fda59872..5b9ae62272bb 100644 --- a/include/linux/ring_buffer.h +++ b/include/linux/ring_buffer.h | |||
@@ -97,7 +97,7 @@ __ring_buffer_alloc(unsigned long size, unsigned flags, struct lock_class_key *k | |||
97 | __ring_buffer_alloc((size), (flags), &__key); \ | 97 | __ring_buffer_alloc((size), (flags), &__key); \ |
98 | }) | 98 | }) |
99 | 99 | ||
100 | int ring_buffer_wait(struct ring_buffer *buffer, int cpu, bool full); | 100 | int ring_buffer_wait(struct ring_buffer *buffer, int cpu, int full); |
101 | __poll_t ring_buffer_poll_wait(struct ring_buffer *buffer, int cpu, | 101 | __poll_t ring_buffer_poll_wait(struct ring_buffer *buffer, int cpu, |
102 | struct file *filp, poll_table *poll_table); | 102 | struct file *filp, poll_table *poll_table); |
103 | 103 | ||
@@ -189,6 +189,8 @@ bool ring_buffer_time_stamp_abs(struct ring_buffer *buffer); | |||
189 | 189 | ||
190 | size_t ring_buffer_page_len(void *page); | 190 | size_t ring_buffer_page_len(void *page); |
191 | 191 | ||
192 | size_t ring_buffer_nr_pages(struct ring_buffer *buffer, int cpu); | ||
193 | size_t ring_buffer_nr_dirty_pages(struct ring_buffer *buffer, int cpu); | ||
192 | 194 | ||
193 | void *ring_buffer_alloc_read_page(struct ring_buffer *buffer, int cpu); | 195 | void *ring_buffer_alloc_read_page(struct ring_buffer *buffer, int cpu); |
194 | void ring_buffer_free_read_page(struct ring_buffer *buffer, int cpu, void *data); | 196 | void ring_buffer_free_read_page(struct ring_buffer *buffer, int cpu, void *data); |
diff --git a/include/linux/string.h b/include/linux/string.h index 27d0482e5e05..7927b875f80c 100644 --- a/include/linux/string.h +++ b/include/linux/string.h | |||
@@ -456,4 +456,24 @@ static inline void memcpy_and_pad(void *dest, size_t dest_len, | |||
456 | memcpy(dest, src, dest_len); | 456 | memcpy(dest, src, dest_len); |
457 | } | 457 | } |
458 | 458 | ||
459 | /** | ||
460 | * str_has_prefix - Test if a string has a given prefix | ||
461 | * @str: The string to test | ||
462 | * @prefix: The string to see if @str starts with | ||
463 | * | ||
464 | * A common way to test a prefix of a string is to do: | ||
465 | * strncmp(str, prefix, sizeof(prefix) - 1) | ||
466 | * | ||
467 | * But this can lead to bugs due to typos, or if prefix is a pointer | ||
468 | * and not a constant. Instead use str_has_prefix(). | ||
469 | * | ||
470 | * Returns: 0 if @str does not start with @prefix | ||
471 | strlen(@prefix) if @str does start with @prefix | ||
472 | */ | ||
473 | static __always_inline size_t str_has_prefix(const char *str, const char *prefix) | ||
474 | { | ||
475 | size_t len = strlen(prefix); | ||
476 | return strncmp(str, prefix, len) == 0 ? len : 0; | ||
477 | } | ||
478 | |||
459 | #endif /* _LINUX_STRING_H_ */ | 479 | #endif /* _LINUX_STRING_H_ */ |
diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig index 5e3de28c7677..fa8b1fe824f3 100644 --- a/kernel/trace/Kconfig +++ b/kernel/trace/Kconfig | |||
@@ -461,6 +461,7 @@ config KPROBE_EVENTS | |||
461 | bool "Enable kprobes-based dynamic events" | 461 | bool "Enable kprobes-based dynamic events" |
462 | select TRACING | 462 | select TRACING |
463 | select PROBE_EVENTS | 463 | select PROBE_EVENTS |
464 | select DYNAMIC_EVENTS | ||
464 | default y | 465 | default y |
465 | help | 466 | help |
466 | This allows the user to add tracing events (similar to tracepoints) | 467 | This allows the user to add tracing events (similar to tracepoints) |
@@ -500,6 +501,7 @@ config UPROBE_EVENTS | |||
500 | depends on PERF_EVENTS | 501 | depends on PERF_EVENTS |
501 | select UPROBES | 502 | select UPROBES |
502 | select PROBE_EVENTS | 503 | select PROBE_EVENTS |
504 | select DYNAMIC_EVENTS | ||
503 | select TRACING | 505 | select TRACING |
504 | default y | 506 | default y |
505 | help | 507 | help |
@@ -518,6 +520,9 @@ config BPF_EVENTS | |||
518 | help | 520 | help |
519 | This allows the user to attach BPF programs to kprobe events. | 521 | This allows the user to attach BPF programs to kprobe events. |
520 | 522 | ||
523 | config DYNAMIC_EVENTS | ||
524 | def_bool n | ||
525 | |||
521 | config PROBE_EVENTS | 526 | config PROBE_EVENTS |
522 | def_bool n | 527 | def_bool n |
523 | 528 | ||
@@ -630,6 +635,7 @@ config HIST_TRIGGERS | |||
630 | depends on ARCH_HAVE_NMI_SAFE_CMPXCHG | 635 | depends on ARCH_HAVE_NMI_SAFE_CMPXCHG |
631 | select TRACING_MAP | 636 | select TRACING_MAP |
632 | select TRACING | 637 | select TRACING |
638 | select DYNAMIC_EVENTS | ||
633 | default n | 639 | default n |
634 | help | 640 | help |
635 | Hist triggers allow one or more arbitrary trace event fields | 641 | Hist triggers allow one or more arbitrary trace event fields |
diff --git a/kernel/trace/Makefile b/kernel/trace/Makefile index f81dadbc7c4a..c2b2148bb1d2 100644 --- a/kernel/trace/Makefile +++ b/kernel/trace/Makefile | |||
@@ -57,6 +57,7 @@ obj-$(CONFIG_MMIOTRACE) += trace_mmiotrace.o | |||
57 | obj-$(CONFIG_FUNCTION_GRAPH_TRACER) += trace_functions_graph.o | 57 | obj-$(CONFIG_FUNCTION_GRAPH_TRACER) += trace_functions_graph.o |
58 | obj-$(CONFIG_TRACE_BRANCH_PROFILING) += trace_branch.o | 58 | obj-$(CONFIG_TRACE_BRANCH_PROFILING) += trace_branch.o |
59 | obj-$(CONFIG_BLK_DEV_IO_TRACE) += blktrace.o | 59 | obj-$(CONFIG_BLK_DEV_IO_TRACE) += blktrace.o |
60 | obj-$(CONFIG_FUNCTION_GRAPH_TRACER) += fgraph.o | ||
60 | ifeq ($(CONFIG_BLOCK),y) | 61 | ifeq ($(CONFIG_BLOCK),y) |
61 | obj-$(CONFIG_EVENT_TRACING) += blktrace.o | 62 | obj-$(CONFIG_EVENT_TRACING) += blktrace.o |
62 | endif | 63 | endif |
@@ -78,6 +79,7 @@ endif | |||
78 | ifeq ($(CONFIG_TRACING),y) | 79 | ifeq ($(CONFIG_TRACING),y) |
79 | obj-$(CONFIG_KGDB_KDB) += trace_kdb.o | 80 | obj-$(CONFIG_KGDB_KDB) += trace_kdb.o |
80 | endif | 81 | endif |
82 | obj-$(CONFIG_DYNAMIC_EVENTS) += trace_dynevent.o | ||
81 | obj-$(CONFIG_PROBE_EVENTS) += trace_probe.o | 83 | obj-$(CONFIG_PROBE_EVENTS) += trace_probe.o |
82 | obj-$(CONFIG_UPROBE_EVENTS) += trace_uprobe.o | 84 | obj-$(CONFIG_UPROBE_EVENTS) += trace_uprobe.o |
83 | 85 | ||
diff --git a/kernel/trace/fgraph.c b/kernel/trace/fgraph.c new file mode 100644 index 000000000000..8dfd5021b933 --- /dev/null +++ b/kernel/trace/fgraph.c | |||
@@ -0,0 +1,626 @@ | |||
1 | // SPDX-License-Identifier: GPL-2.0 | ||
2 | /* | ||
3 | * Infrastructure to took into function calls and returns. | ||
4 | * Copyright (c) 2008-2009 Frederic Weisbecker <fweisbec@gmail.com> | ||
5 | * Mostly borrowed from function tracer which | ||
6 | * is Copyright (c) Steven Rostedt <srostedt@redhat.com> | ||
7 | * | ||
8 | * Highly modified by Steven Rostedt (VMware). | ||
9 | */ | ||
10 | #include <linux/suspend.h> | ||
11 | #include <linux/ftrace.h> | ||
12 | #include <linux/slab.h> | ||
13 | |||
14 | #include <trace/events/sched.h> | ||
15 | |||
16 | #include "ftrace_internal.h" | ||
17 | |||
18 | #ifdef CONFIG_DYNAMIC_FTRACE | ||
19 | #define ASSIGN_OPS_HASH(opsname, val) \ | ||
20 | .func_hash = val, \ | ||
21 | .local_hash.regex_lock = __MUTEX_INITIALIZER(opsname.local_hash.regex_lock), | ||
22 | #else | ||
23 | #define ASSIGN_OPS_HASH(opsname, val) | ||
24 | #endif | ||
25 | |||
26 | static bool kill_ftrace_graph; | ||
27 | int ftrace_graph_active; | ||
28 | |||
29 | /* Both enabled by default (can be cleared by function_graph tracer flags */ | ||
30 | static bool fgraph_sleep_time = true; | ||
31 | |||
32 | /** | ||
33 | * ftrace_graph_is_dead - returns true if ftrace_graph_stop() was called | ||
34 | * | ||
35 | * ftrace_graph_stop() is called when a severe error is detected in | ||
36 | * the function graph tracing. This function is called by the critical | ||
37 | * paths of function graph to keep those paths from doing any more harm. | ||
38 | */ | ||
39 | bool ftrace_graph_is_dead(void) | ||
40 | { | ||
41 | return kill_ftrace_graph; | ||
42 | } | ||
43 | |||
44 | /** | ||
45 | * ftrace_graph_stop - set to permanently disable function graph tracincg | ||
46 | * | ||
47 | * In case of an error int function graph tracing, this is called | ||
48 | * to try to keep function graph tracing from causing any more harm. | ||
49 | * Usually this is pretty severe and this is called to try to at least | ||
50 | * get a warning out to the user. | ||
51 | */ | ||
52 | void ftrace_graph_stop(void) | ||
53 | { | ||
54 | kill_ftrace_graph = true; | ||
55 | } | ||
56 | |||
57 | /* Add a function return address to the trace stack on thread info.*/ | ||
58 | static int | ||
59 | ftrace_push_return_trace(unsigned long ret, unsigned long func, | ||
60 | unsigned long frame_pointer, unsigned long *retp) | ||
61 | { | ||
62 | unsigned long long calltime; | ||
63 | int index; | ||
64 | |||
65 | if (unlikely(ftrace_graph_is_dead())) | ||
66 | return -EBUSY; | ||
67 | |||
68 | if (!current->ret_stack) | ||
69 | return -EBUSY; | ||
70 | |||
71 | /* | ||
72 | * We must make sure the ret_stack is tested before we read | ||
73 | * anything else. | ||
74 | */ | ||
75 | smp_rmb(); | ||
76 | |||
77 | /* The return trace stack is full */ | ||
78 | if (current->curr_ret_stack == FTRACE_RETFUNC_DEPTH - 1) { | ||
79 | atomic_inc(¤t->trace_overrun); | ||
80 | return -EBUSY; | ||
81 | } | ||
82 | |||
83 | calltime = trace_clock_local(); | ||
84 | |||
85 | index = ++current->curr_ret_stack; | ||
86 | barrier(); | ||
87 | current->ret_stack[index].ret = ret; | ||
88 | current->ret_stack[index].func = func; | ||
89 | current->ret_stack[index].calltime = calltime; | ||
90 | #ifdef HAVE_FUNCTION_GRAPH_FP_TEST | ||
91 | current->ret_stack[index].fp = frame_pointer; | ||
92 | #endif | ||
93 | #ifdef HAVE_FUNCTION_GRAPH_RET_ADDR_PTR | ||
94 | current->ret_stack[index].retp = retp; | ||
95 | #endif | ||
96 | return 0; | ||
97 | } | ||
98 | |||
99 | int function_graph_enter(unsigned long ret, unsigned long func, | ||
100 | unsigned long frame_pointer, unsigned long *retp) | ||
101 | { | ||
102 | struct ftrace_graph_ent trace; | ||
103 | |||
104 | trace.func = func; | ||
105 | trace.depth = ++current->curr_ret_depth; | ||
106 | |||
107 | if (ftrace_push_return_trace(ret, func, frame_pointer, retp)) | ||
108 | goto out; | ||
109 | |||
110 | /* Only trace if the calling function expects to */ | ||
111 | if (!ftrace_graph_entry(&trace)) | ||
112 | goto out_ret; | ||
113 | |||
114 | return 0; | ||
115 | out_ret: | ||
116 | current->curr_ret_stack--; | ||
117 | out: | ||
118 | current->curr_ret_depth--; | ||
119 | return -EBUSY; | ||
120 | } | ||
121 | |||
122 | /* Retrieve a function return address to the trace stack on thread info.*/ | ||
123 | static void | ||
124 | ftrace_pop_return_trace(struct ftrace_graph_ret *trace, unsigned long *ret, | ||
125 | unsigned long frame_pointer) | ||
126 | { | ||
127 | int index; | ||
128 | |||
129 | index = current->curr_ret_stack; | ||
130 | |||
131 | if (unlikely(index < 0 || index >= FTRACE_RETFUNC_DEPTH)) { | ||
132 | ftrace_graph_stop(); | ||
133 | WARN_ON(1); | ||
134 | /* Might as well panic, otherwise we have no where to go */ | ||
135 | *ret = (unsigned long)panic; | ||
136 | return; | ||
137 | } | ||
138 | |||
139 | #ifdef HAVE_FUNCTION_GRAPH_FP_TEST | ||
140 | /* | ||
141 | * The arch may choose to record the frame pointer used | ||
142 | * and check it here to make sure that it is what we expect it | ||
143 | * to be. If gcc does not set the place holder of the return | ||
144 | * address in the frame pointer, and does a copy instead, then | ||
145 | * the function graph trace will fail. This test detects this | ||
146 | * case. | ||
147 | * | ||
148 | * Currently, x86_32 with optimize for size (-Os) makes the latest | ||
149 | * gcc do the above. | ||
150 | * | ||
151 | * Note, -mfentry does not use frame pointers, and this test | ||
152 | * is not needed if CC_USING_FENTRY is set. | ||
153 | */ | ||
154 | if (unlikely(current->ret_stack[index].fp != frame_pointer)) { | ||
155 | ftrace_graph_stop(); | ||
156 | WARN(1, "Bad frame pointer: expected %lx, received %lx\n" | ||
157 | " from func %ps return to %lx\n", | ||
158 | current->ret_stack[index].fp, | ||
159 | frame_pointer, | ||
160 | (void *)current->ret_stack[index].func, | ||
161 | current->ret_stack[index].ret); | ||
162 | *ret = (unsigned long)panic; | ||
163 | return; | ||
164 | } | ||
165 | #endif | ||
166 | |||
167 | *ret = current->ret_stack[index].ret; | ||
168 | trace->func = current->ret_stack[index].func; | ||
169 | trace->calltime = current->ret_stack[index].calltime; | ||
170 | trace->overrun = atomic_read(¤t->trace_overrun); | ||
171 | trace->depth = current->curr_ret_depth--; | ||
172 | /* | ||
173 | * We still want to trace interrupts coming in if | ||
174 | * max_depth is set to 1. Make sure the decrement is | ||
175 | * seen before ftrace_graph_return. | ||
176 | */ | ||
177 | barrier(); | ||
178 | } | ||
179 | |||
180 | /* | ||
181 | * Hibernation protection. | ||
182 | * The state of the current task is too much unstable during | ||
183 | * suspend/restore to disk. We want to protect against that. | ||
184 | */ | ||
185 | static int | ||
186 | ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state, | ||
187 | void *unused) | ||
188 | { | ||
189 | switch (state) { | ||
190 | case PM_HIBERNATION_PREPARE: | ||
191 | pause_graph_tracing(); | ||
192 | break; | ||
193 | |||
194 | case PM_POST_HIBERNATION: | ||
195 | unpause_graph_tracing(); | ||
196 | break; | ||
197 | } | ||
198 | return NOTIFY_DONE; | ||
199 | } | ||
200 | |||
201 | static struct notifier_block ftrace_suspend_notifier = { | ||
202 | .notifier_call = ftrace_suspend_notifier_call, | ||
203 | }; | ||
204 | |||
205 | /* | ||
206 | * Send the trace to the ring-buffer. | ||
207 | * @return the original return address. | ||
208 | */ | ||
209 | unsigned long ftrace_return_to_handler(unsigned long frame_pointer) | ||
210 | { | ||
211 | struct ftrace_graph_ret trace; | ||
212 | unsigned long ret; | ||
213 | |||
214 | ftrace_pop_return_trace(&trace, &ret, frame_pointer); | ||
215 | trace.rettime = trace_clock_local(); | ||
216 | ftrace_graph_return(&trace); | ||
217 | /* | ||
218 | * The ftrace_graph_return() may still access the current | ||
219 | * ret_stack structure, we need to make sure the update of | ||
220 | * curr_ret_stack is after that. | ||
221 | */ | ||
222 | barrier(); | ||
223 | current->curr_ret_stack--; | ||
224 | |||
225 | if (unlikely(!ret)) { | ||
226 | ftrace_graph_stop(); | ||
227 | WARN_ON(1); | ||
228 | /* Might as well panic. What else to do? */ | ||
229 | ret = (unsigned long)panic; | ||
230 | } | ||
231 | |||
232 | return ret; | ||
233 | } | ||
234 | |||
235 | /** | ||
236 | * ftrace_graph_get_ret_stack - return the entry of the shadow stack | ||
237 | * @task: The task to read the shadow stack from | ||
238 | * @idx: Index down the shadow stack | ||
239 | * | ||
240 | * Return the ret_struct on the shadow stack of the @task at the | ||
241 | * call graph at @idx starting with zero. If @idx is zero, it | ||
242 | * will return the last saved ret_stack entry. If it is greater than | ||
243 | * zero, it will return the corresponding ret_stack for the depth | ||
244 | * of saved return addresses. | ||
245 | */ | ||
246 | struct ftrace_ret_stack * | ||
247 | ftrace_graph_get_ret_stack(struct task_struct *task, int idx) | ||
248 | { | ||
249 | idx = task->curr_ret_stack - idx; | ||
250 | |||
251 | if (idx >= 0 && idx <= task->curr_ret_stack) | ||
252 | return &task->ret_stack[idx]; | ||
253 | |||
254 | return NULL; | ||
255 | } | ||
256 | |||
257 | /** | ||
258 | * ftrace_graph_ret_addr - convert a potentially modified stack return address | ||
259 | * to its original value | ||
260 | * | ||
261 | * This function can be called by stack unwinding code to convert a found stack | ||
262 | * return address ('ret') to its original value, in case the function graph | ||
263 | * tracer has modified it to be 'return_to_handler'. If the address hasn't | ||
264 | * been modified, the unchanged value of 'ret' is returned. | ||
265 | * | ||
266 | * 'idx' is a state variable which should be initialized by the caller to zero | ||
267 | * before the first call. | ||
268 | * | ||
269 | * 'retp' is a pointer to the return address on the stack. It's ignored if | ||
270 | * the arch doesn't have HAVE_FUNCTION_GRAPH_RET_ADDR_PTR defined. | ||
271 | */ | ||
272 | #ifdef HAVE_FUNCTION_GRAPH_RET_ADDR_PTR | ||
273 | unsigned long ftrace_graph_ret_addr(struct task_struct *task, int *idx, | ||
274 | unsigned long ret, unsigned long *retp) | ||
275 | { | ||
276 | int index = task->curr_ret_stack; | ||
277 | int i; | ||
278 | |||
279 | if (ret != (unsigned long)return_to_handler) | ||
280 | return ret; | ||
281 | |||
282 | if (index < 0) | ||
283 | return ret; | ||
284 | |||
285 | for (i = 0; i <= index; i++) | ||
286 | if (task->ret_stack[i].retp == retp) | ||
287 | return task->ret_stack[i].ret; | ||
288 | |||
289 | return ret; | ||
290 | } | ||
291 | #else /* !HAVE_FUNCTION_GRAPH_RET_ADDR_PTR */ | ||
292 | unsigned long ftrace_graph_ret_addr(struct task_struct *task, int *idx, | ||
293 | unsigned long ret, unsigned long *retp) | ||
294 | { | ||
295 | int task_idx; | ||
296 | |||
297 | if (ret != (unsigned long)return_to_handler) | ||
298 | return ret; | ||
299 | |||
300 | task_idx = task->curr_ret_stack; | ||
301 | |||
302 | if (!task->ret_stack || task_idx < *idx) | ||
303 | return ret; | ||
304 | |||
305 | task_idx -= *idx; | ||
306 | (*idx)++; | ||
307 | |||
308 | return task->ret_stack[task_idx].ret; | ||
309 | } | ||
310 | #endif /* HAVE_FUNCTION_GRAPH_RET_ADDR_PTR */ | ||
311 | |||
312 | static struct ftrace_ops graph_ops = { | ||
313 | .func = ftrace_stub, | ||
314 | .flags = FTRACE_OPS_FL_RECURSION_SAFE | | ||
315 | FTRACE_OPS_FL_INITIALIZED | | ||
316 | FTRACE_OPS_FL_PID | | ||
317 | FTRACE_OPS_FL_STUB, | ||
318 | #ifdef FTRACE_GRAPH_TRAMP_ADDR | ||
319 | .trampoline = FTRACE_GRAPH_TRAMP_ADDR, | ||
320 | /* trampoline_size is only needed for dynamically allocated tramps */ | ||
321 | #endif | ||
322 | ASSIGN_OPS_HASH(graph_ops, &global_ops.local_hash) | ||
323 | }; | ||
324 | |||
325 | void ftrace_graph_sleep_time_control(bool enable) | ||
326 | { | ||
327 | fgraph_sleep_time = enable; | ||
328 | } | ||
329 | |||
330 | int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace) | ||
331 | { | ||
332 | return 0; | ||
333 | } | ||
334 | |||
335 | /* The callbacks that hook a function */ | ||
336 | trace_func_graph_ret_t ftrace_graph_return = | ||
337 | (trace_func_graph_ret_t)ftrace_stub; | ||
338 | trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub; | ||
339 | static trace_func_graph_ent_t __ftrace_graph_entry = ftrace_graph_entry_stub; | ||
340 | |||
341 | /* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */ | ||
342 | static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list) | ||
343 | { | ||
344 | int i; | ||
345 | int ret = 0; | ||
346 | int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE; | ||
347 | struct task_struct *g, *t; | ||
348 | |||
349 | for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) { | ||
350 | ret_stack_list[i] = | ||
351 | kmalloc_array(FTRACE_RETFUNC_DEPTH, | ||
352 | sizeof(struct ftrace_ret_stack), | ||
353 | GFP_KERNEL); | ||
354 | if (!ret_stack_list[i]) { | ||
355 | start = 0; | ||
356 | end = i; | ||
357 | ret = -ENOMEM; | ||
358 | goto free; | ||
359 | } | ||
360 | } | ||
361 | |||
362 | read_lock(&tasklist_lock); | ||
363 | do_each_thread(g, t) { | ||
364 | if (start == end) { | ||
365 | ret = -EAGAIN; | ||
366 | goto unlock; | ||
367 | } | ||
368 | |||
369 | if (t->ret_stack == NULL) { | ||
370 | atomic_set(&t->tracing_graph_pause, 0); | ||
371 | atomic_set(&t->trace_overrun, 0); | ||
372 | t->curr_ret_stack = -1; | ||
373 | t->curr_ret_depth = -1; | ||
374 | /* Make sure the tasks see the -1 first: */ | ||
375 | smp_wmb(); | ||
376 | t->ret_stack = ret_stack_list[start++]; | ||
377 | } | ||
378 | } while_each_thread(g, t); | ||
379 | |||
380 | unlock: | ||
381 | read_unlock(&tasklist_lock); | ||
382 | free: | ||
383 | for (i = start; i < end; i++) | ||
384 | kfree(ret_stack_list[i]); | ||
385 | return ret; | ||
386 | } | ||
387 | |||
388 | static void | ||
389 | ftrace_graph_probe_sched_switch(void *ignore, bool preempt, | ||
390 | struct task_struct *prev, struct task_struct *next) | ||
391 | { | ||
392 | unsigned long long timestamp; | ||
393 | int index; | ||
394 | |||
395 | /* | ||
396 | * Does the user want to count the time a function was asleep. | ||
397 | * If so, do not update the time stamps. | ||
398 | */ | ||
399 | if (fgraph_sleep_time) | ||
400 | return; | ||
401 | |||
402 | timestamp = trace_clock_local(); | ||
403 | |||
404 | prev->ftrace_timestamp = timestamp; | ||
405 | |||
406 | /* only process tasks that we timestamped */ | ||
407 | if (!next->ftrace_timestamp) | ||
408 | return; | ||
409 | |||
410 | /* | ||
411 | * Update all the counters in next to make up for the | ||
412 | * time next was sleeping. | ||
413 | */ | ||
414 | timestamp -= next->ftrace_timestamp; | ||
415 | |||
416 | for (index = next->curr_ret_stack; index >= 0; index--) | ||
417 | next->ret_stack[index].calltime += timestamp; | ||
418 | } | ||
419 | |||
420 | static int ftrace_graph_entry_test(struct ftrace_graph_ent *trace) | ||
421 | { | ||
422 | if (!ftrace_ops_test(&global_ops, trace->func, NULL)) | ||
423 | return 0; | ||
424 | return __ftrace_graph_entry(trace); | ||
425 | } | ||
426 | |||
427 | /* | ||
428 | * The function graph tracer should only trace the functions defined | ||
429 | * by set_ftrace_filter and set_ftrace_notrace. If another function | ||
430 | * tracer ops is registered, the graph tracer requires testing the | ||
431 | * function against the global ops, and not just trace any function | ||
432 | * that any ftrace_ops registered. | ||
433 | */ | ||
434 | void update_function_graph_func(void) | ||
435 | { | ||
436 | struct ftrace_ops *op; | ||
437 | bool do_test = false; | ||
438 | |||
439 | /* | ||
440 | * The graph and global ops share the same set of functions | ||
441 | * to test. If any other ops is on the list, then | ||
442 | * the graph tracing needs to test if its the function | ||
443 | * it should call. | ||
444 | */ | ||
445 | do_for_each_ftrace_op(op, ftrace_ops_list) { | ||
446 | if (op != &global_ops && op != &graph_ops && | ||
447 | op != &ftrace_list_end) { | ||
448 | do_test = true; | ||
449 | /* in double loop, break out with goto */ | ||
450 | goto out; | ||
451 | } | ||
452 | } while_for_each_ftrace_op(op); | ||
453 | out: | ||
454 | if (do_test) | ||
455 | ftrace_graph_entry = ftrace_graph_entry_test; | ||
456 | else | ||
457 | ftrace_graph_entry = __ftrace_graph_entry; | ||
458 | } | ||
459 | |||
460 | static DEFINE_PER_CPU(struct ftrace_ret_stack *, idle_ret_stack); | ||
461 | |||
462 | static void | ||
463 | graph_init_task(struct task_struct *t, struct ftrace_ret_stack *ret_stack) | ||
464 | { | ||
465 | atomic_set(&t->tracing_graph_pause, 0); | ||
466 | atomic_set(&t->trace_overrun, 0); | ||
467 | t->ftrace_timestamp = 0; | ||
468 | /* make curr_ret_stack visible before we add the ret_stack */ | ||
469 | smp_wmb(); | ||
470 | t->ret_stack = ret_stack; | ||
471 | } | ||
472 | |||
473 | /* | ||
474 | * Allocate a return stack for the idle task. May be the first | ||
475 | * time through, or it may be done by CPU hotplug online. | ||
476 | */ | ||
477 | void ftrace_graph_init_idle_task(struct task_struct *t, int cpu) | ||
478 | { | ||
479 | t->curr_ret_stack = -1; | ||
480 | t->curr_ret_depth = -1; | ||
481 | /* | ||
482 | * The idle task has no parent, it either has its own | ||
483 | * stack or no stack at all. | ||
484 | */ | ||
485 | if (t->ret_stack) | ||
486 | WARN_ON(t->ret_stack != per_cpu(idle_ret_stack, cpu)); | ||
487 | |||
488 | if (ftrace_graph_active) { | ||
489 | struct ftrace_ret_stack *ret_stack; | ||
490 | |||
491 | ret_stack = per_cpu(idle_ret_stack, cpu); | ||
492 | if (!ret_stack) { | ||
493 | ret_stack = | ||
494 | kmalloc_array(FTRACE_RETFUNC_DEPTH, | ||
495 | sizeof(struct ftrace_ret_stack), | ||
496 | GFP_KERNEL); | ||
497 | if (!ret_stack) | ||
498 | return; | ||
499 | per_cpu(idle_ret_stack, cpu) = ret_stack; | ||
500 | } | ||
501 | graph_init_task(t, ret_stack); | ||
502 | } | ||
503 | } | ||
504 | |||
505 | /* Allocate a return stack for newly created task */ | ||
506 | void ftrace_graph_init_task(struct task_struct *t) | ||
507 | { | ||
508 | /* Make sure we do not use the parent ret_stack */ | ||
509 | t->ret_stack = NULL; | ||
510 | t->curr_ret_stack = -1; | ||
511 | t->curr_ret_depth = -1; | ||
512 | |||
513 | if (ftrace_graph_active) { | ||
514 | struct ftrace_ret_stack *ret_stack; | ||
515 | |||
516 | ret_stack = kmalloc_array(FTRACE_RETFUNC_DEPTH, | ||
517 | sizeof(struct ftrace_ret_stack), | ||
518 | GFP_KERNEL); | ||
519 | if (!ret_stack) | ||
520 | return; | ||
521 | graph_init_task(t, ret_stack); | ||
522 | } | ||
523 | } | ||
524 | |||
525 | void ftrace_graph_exit_task(struct task_struct *t) | ||
526 | { | ||
527 | struct ftrace_ret_stack *ret_stack = t->ret_stack; | ||
528 | |||
529 | t->ret_stack = NULL; | ||
530 | /* NULL must become visible to IRQs before we free it: */ | ||
531 | barrier(); | ||
532 | |||
533 | kfree(ret_stack); | ||
534 | } | ||
535 | |||
536 | /* Allocate a return stack for each task */ | ||
537 | static int start_graph_tracing(void) | ||
538 | { | ||
539 | struct ftrace_ret_stack **ret_stack_list; | ||
540 | int ret, cpu; | ||
541 | |||
542 | ret_stack_list = kmalloc_array(FTRACE_RETSTACK_ALLOC_SIZE, | ||
543 | sizeof(struct ftrace_ret_stack *), | ||
544 | GFP_KERNEL); | ||
545 | |||
546 | if (!ret_stack_list) | ||
547 | return -ENOMEM; | ||
548 | |||
549 | /* The cpu_boot init_task->ret_stack will never be freed */ | ||
550 | for_each_online_cpu(cpu) { | ||
551 | if (!idle_task(cpu)->ret_stack) | ||
552 | ftrace_graph_init_idle_task(idle_task(cpu), cpu); | ||
553 | } | ||
554 | |||
555 | do { | ||
556 | ret = alloc_retstack_tasklist(ret_stack_list); | ||
557 | } while (ret == -EAGAIN); | ||
558 | |||
559 | if (!ret) { | ||
560 | ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL); | ||
561 | if (ret) | ||
562 | pr_info("ftrace_graph: Couldn't activate tracepoint" | ||
563 | " probe to kernel_sched_switch\n"); | ||
564 | } | ||
565 | |||
566 | kfree(ret_stack_list); | ||
567 | return ret; | ||
568 | } | ||
569 | |||
570 | int register_ftrace_graph(struct fgraph_ops *gops) | ||
571 | { | ||
572 | int ret = 0; | ||
573 | |||
574 | mutex_lock(&ftrace_lock); | ||
575 | |||
576 | /* we currently allow only one tracer registered at a time */ | ||
577 | if (ftrace_graph_active) { | ||
578 | ret = -EBUSY; | ||
579 | goto out; | ||
580 | } | ||
581 | |||
582 | register_pm_notifier(&ftrace_suspend_notifier); | ||
583 | |||
584 | ftrace_graph_active++; | ||
585 | ret = start_graph_tracing(); | ||
586 | if (ret) { | ||
587 | ftrace_graph_active--; | ||
588 | goto out; | ||
589 | } | ||
590 | |||
591 | ftrace_graph_return = gops->retfunc; | ||
592 | |||
593 | /* | ||
594 | * Update the indirect function to the entryfunc, and the | ||
595 | * function that gets called to the entry_test first. Then | ||
596 | * call the update fgraph entry function to determine if | ||
597 | * the entryfunc should be called directly or not. | ||
598 | */ | ||
599 | __ftrace_graph_entry = gops->entryfunc; | ||
600 | ftrace_graph_entry = ftrace_graph_entry_test; | ||
601 | update_function_graph_func(); | ||
602 | |||
603 | ret = ftrace_startup(&graph_ops, FTRACE_START_FUNC_RET); | ||
604 | out: | ||
605 | mutex_unlock(&ftrace_lock); | ||
606 | return ret; | ||
607 | } | ||
608 | |||
609 | void unregister_ftrace_graph(struct fgraph_ops *gops) | ||
610 | { | ||
611 | mutex_lock(&ftrace_lock); | ||
612 | |||
613 | if (unlikely(!ftrace_graph_active)) | ||
614 | goto out; | ||
615 | |||
616 | ftrace_graph_active--; | ||
617 | ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub; | ||
618 | ftrace_graph_entry = ftrace_graph_entry_stub; | ||
619 | __ftrace_graph_entry = ftrace_graph_entry_stub; | ||
620 | ftrace_shutdown(&graph_ops, FTRACE_STOP_FUNC_RET); | ||
621 | unregister_pm_notifier(&ftrace_suspend_notifier); | ||
622 | unregister_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL); | ||
623 | |||
624 | out: | ||
625 | mutex_unlock(&ftrace_lock); | ||
626 | } | ||
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index f0ff24173a0b..aac7847c0214 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c | |||
@@ -19,7 +19,6 @@ | |||
19 | #include <linux/sched/task.h> | 19 | #include <linux/sched/task.h> |
20 | #include <linux/kallsyms.h> | 20 | #include <linux/kallsyms.h> |
21 | #include <linux/seq_file.h> | 21 | #include <linux/seq_file.h> |
22 | #include <linux/suspend.h> | ||
23 | #include <linux/tracefs.h> | 22 | #include <linux/tracefs.h> |
24 | #include <linux/hardirq.h> | 23 | #include <linux/hardirq.h> |
25 | #include <linux/kthread.h> | 24 | #include <linux/kthread.h> |
@@ -40,6 +39,7 @@ | |||
40 | #include <asm/sections.h> | 39 | #include <asm/sections.h> |
41 | #include <asm/setup.h> | 40 | #include <asm/setup.h> |
42 | 41 | ||
42 | #include "ftrace_internal.h" | ||
43 | #include "trace_output.h" | 43 | #include "trace_output.h" |
44 | #include "trace_stat.h" | 44 | #include "trace_stat.h" |
45 | 45 | ||
@@ -77,7 +77,12 @@ | |||
77 | #define ASSIGN_OPS_HASH(opsname, val) | 77 | #define ASSIGN_OPS_HASH(opsname, val) |
78 | #endif | 78 | #endif |
79 | 79 | ||
80 | static struct ftrace_ops ftrace_list_end __read_mostly = { | 80 | enum { |
81 | FTRACE_MODIFY_ENABLE_FL = (1 << 0), | ||
82 | FTRACE_MODIFY_MAY_SLEEP_FL = (1 << 1), | ||
83 | }; | ||
84 | |||
85 | struct ftrace_ops ftrace_list_end __read_mostly = { | ||
81 | .func = ftrace_stub, | 86 | .func = ftrace_stub, |
82 | .flags = FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_STUB, | 87 | .flags = FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_STUB, |
83 | INIT_OPS_HASH(ftrace_list_end) | 88 | INIT_OPS_HASH(ftrace_list_end) |
@@ -112,11 +117,11 @@ static void ftrace_update_trampoline(struct ftrace_ops *ops); | |||
112 | */ | 117 | */ |
113 | static int ftrace_disabled __read_mostly; | 118 | static int ftrace_disabled __read_mostly; |
114 | 119 | ||
115 | static DEFINE_MUTEX(ftrace_lock); | 120 | DEFINE_MUTEX(ftrace_lock); |
116 | 121 | ||
117 | static struct ftrace_ops __rcu *ftrace_ops_list __read_mostly = &ftrace_list_end; | 122 | struct ftrace_ops __rcu *ftrace_ops_list __read_mostly = &ftrace_list_end; |
118 | ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub; | 123 | ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub; |
119 | static struct ftrace_ops global_ops; | 124 | struct ftrace_ops global_ops; |
120 | 125 | ||
121 | #if ARCH_SUPPORTS_FTRACE_OPS | 126 | #if ARCH_SUPPORTS_FTRACE_OPS |
122 | static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip, | 127 | static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip, |
@@ -127,26 +132,6 @@ static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip); | |||
127 | #define ftrace_ops_list_func ((ftrace_func_t)ftrace_ops_no_ops) | 132 | #define ftrace_ops_list_func ((ftrace_func_t)ftrace_ops_no_ops) |
128 | #endif | 133 | #endif |
129 | 134 | ||
130 | /* | ||
131 | * Traverse the ftrace_global_list, invoking all entries. The reason that we | ||
132 | * can use rcu_dereference_raw_notrace() is that elements removed from this list | ||
133 | * are simply leaked, so there is no need to interact with a grace-period | ||
134 | * mechanism. The rcu_dereference_raw_notrace() calls are needed to handle | ||
135 | * concurrent insertions into the ftrace_global_list. | ||
136 | * | ||
137 | * Silly Alpha and silly pointer-speculation compiler optimizations! | ||
138 | */ | ||
139 | #define do_for_each_ftrace_op(op, list) \ | ||
140 | op = rcu_dereference_raw_notrace(list); \ | ||
141 | do | ||
142 | |||
143 | /* | ||
144 | * Optimized for just a single item in the list (as that is the normal case). | ||
145 | */ | ||
146 | #define while_for_each_ftrace_op(op) \ | ||
147 | while (likely(op = rcu_dereference_raw_notrace((op)->next)) && \ | ||
148 | unlikely((op) != &ftrace_list_end)) | ||
149 | |||
150 | static inline void ftrace_ops_init(struct ftrace_ops *ops) | 135 | static inline void ftrace_ops_init(struct ftrace_ops *ops) |
151 | { | 136 | { |
152 | #ifdef CONFIG_DYNAMIC_FTRACE | 137 | #ifdef CONFIG_DYNAMIC_FTRACE |
@@ -186,18 +171,6 @@ static void ftrace_sync_ipi(void *data) | |||
186 | smp_rmb(); | 171 | smp_rmb(); |
187 | } | 172 | } |
188 | 173 | ||
189 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | ||
190 | static void update_function_graph_func(void); | ||
191 | |||
192 | /* Both enabled by default (can be cleared by function_graph tracer flags */ | ||
193 | static bool fgraph_sleep_time = true; | ||
194 | static bool fgraph_graph_time = true; | ||
195 | |||
196 | #else | ||
197 | static inline void update_function_graph_func(void) { } | ||
198 | #endif | ||
199 | |||
200 | |||
201 | static ftrace_func_t ftrace_ops_get_list_func(struct ftrace_ops *ops) | 174 | static ftrace_func_t ftrace_ops_get_list_func(struct ftrace_ops *ops) |
202 | { | 175 | { |
203 | /* | 176 | /* |
@@ -334,7 +307,7 @@ static int remove_ftrace_ops(struct ftrace_ops __rcu **list, | |||
334 | 307 | ||
335 | static void ftrace_update_trampoline(struct ftrace_ops *ops); | 308 | static void ftrace_update_trampoline(struct ftrace_ops *ops); |
336 | 309 | ||
337 | static int __register_ftrace_function(struct ftrace_ops *ops) | 310 | int __register_ftrace_function(struct ftrace_ops *ops) |
338 | { | 311 | { |
339 | if (ops->flags & FTRACE_OPS_FL_DELETED) | 312 | if (ops->flags & FTRACE_OPS_FL_DELETED) |
340 | return -EINVAL; | 313 | return -EINVAL; |
@@ -375,7 +348,7 @@ static int __register_ftrace_function(struct ftrace_ops *ops) | |||
375 | return 0; | 348 | return 0; |
376 | } | 349 | } |
377 | 350 | ||
378 | static int __unregister_ftrace_function(struct ftrace_ops *ops) | 351 | int __unregister_ftrace_function(struct ftrace_ops *ops) |
379 | { | 352 | { |
380 | int ret; | 353 | int ret; |
381 | 354 | ||
@@ -815,9 +788,16 @@ function_profile_call(unsigned long ip, unsigned long parent_ip, | |||
815 | } | 788 | } |
816 | 789 | ||
817 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | 790 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
791 | static bool fgraph_graph_time = true; | ||
792 | |||
793 | void ftrace_graph_graph_time_control(bool enable) | ||
794 | { | ||
795 | fgraph_graph_time = enable; | ||
796 | } | ||
797 | |||
818 | static int profile_graph_entry(struct ftrace_graph_ent *trace) | 798 | static int profile_graph_entry(struct ftrace_graph_ent *trace) |
819 | { | 799 | { |
820 | int index = current->curr_ret_stack; | 800 | struct ftrace_ret_stack *ret_stack; |
821 | 801 | ||
822 | function_profile_call(trace->func, 0, NULL, NULL); | 802 | function_profile_call(trace->func, 0, NULL, NULL); |
823 | 803 | ||
@@ -825,14 +805,16 @@ static int profile_graph_entry(struct ftrace_graph_ent *trace) | |||
825 | if (!current->ret_stack) | 805 | if (!current->ret_stack) |
826 | return 0; | 806 | return 0; |
827 | 807 | ||
828 | if (index >= 0 && index < FTRACE_RETFUNC_DEPTH) | 808 | ret_stack = ftrace_graph_get_ret_stack(current, 0); |
829 | current->ret_stack[index].subtime = 0; | 809 | if (ret_stack) |
810 | ret_stack->subtime = 0; | ||
830 | 811 | ||
831 | return 1; | 812 | return 1; |
832 | } | 813 | } |
833 | 814 | ||
834 | static void profile_graph_return(struct ftrace_graph_ret *trace) | 815 | static void profile_graph_return(struct ftrace_graph_ret *trace) |
835 | { | 816 | { |
817 | struct ftrace_ret_stack *ret_stack; | ||
836 | struct ftrace_profile_stat *stat; | 818 | struct ftrace_profile_stat *stat; |
837 | unsigned long long calltime; | 819 | unsigned long long calltime; |
838 | struct ftrace_profile *rec; | 820 | struct ftrace_profile *rec; |
@@ -850,16 +832,15 @@ static void profile_graph_return(struct ftrace_graph_ret *trace) | |||
850 | calltime = trace->rettime - trace->calltime; | 832 | calltime = trace->rettime - trace->calltime; |
851 | 833 | ||
852 | if (!fgraph_graph_time) { | 834 | if (!fgraph_graph_time) { |
853 | int index; | ||
854 | |||
855 | index = current->curr_ret_stack; | ||
856 | 835 | ||
857 | /* Append this call time to the parent time to subtract */ | 836 | /* Append this call time to the parent time to subtract */ |
858 | if (index) | 837 | ret_stack = ftrace_graph_get_ret_stack(current, 1); |
859 | current->ret_stack[index - 1].subtime += calltime; | 838 | if (ret_stack) |
839 | ret_stack->subtime += calltime; | ||
860 | 840 | ||
861 | if (current->ret_stack[index].subtime < calltime) | 841 | ret_stack = ftrace_graph_get_ret_stack(current, 0); |
862 | calltime -= current->ret_stack[index].subtime; | 842 | if (ret_stack && ret_stack->subtime < calltime) |
843 | calltime -= ret_stack->subtime; | ||
863 | else | 844 | else |
864 | calltime = 0; | 845 | calltime = 0; |
865 | } | 846 | } |
@@ -874,15 +855,19 @@ static void profile_graph_return(struct ftrace_graph_ret *trace) | |||
874 | local_irq_restore(flags); | 855 | local_irq_restore(flags); |
875 | } | 856 | } |
876 | 857 | ||
858 | static struct fgraph_ops fprofiler_ops = { | ||
859 | .entryfunc = &profile_graph_entry, | ||
860 | .retfunc = &profile_graph_return, | ||
861 | }; | ||
862 | |||
877 | static int register_ftrace_profiler(void) | 863 | static int register_ftrace_profiler(void) |
878 | { | 864 | { |
879 | return register_ftrace_graph(&profile_graph_return, | 865 | return register_ftrace_graph(&fprofiler_ops); |
880 | &profile_graph_entry); | ||
881 | } | 866 | } |
882 | 867 | ||
883 | static void unregister_ftrace_profiler(void) | 868 | static void unregister_ftrace_profiler(void) |
884 | { | 869 | { |
885 | unregister_ftrace_graph(); | 870 | unregister_ftrace_graph(&fprofiler_ops); |
886 | } | 871 | } |
887 | #else | 872 | #else |
888 | static struct ftrace_ops ftrace_profile_ops __read_mostly = { | 873 | static struct ftrace_ops ftrace_profile_ops __read_mostly = { |
@@ -1021,12 +1006,6 @@ static __init void ftrace_profile_tracefs(struct dentry *d_tracer) | |||
1021 | } | 1006 | } |
1022 | #endif /* CONFIG_FUNCTION_PROFILER */ | 1007 | #endif /* CONFIG_FUNCTION_PROFILER */ |
1023 | 1008 | ||
1024 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | ||
1025 | static int ftrace_graph_active; | ||
1026 | #else | ||
1027 | # define ftrace_graph_active 0 | ||
1028 | #endif | ||
1029 | |||
1030 | #ifdef CONFIG_DYNAMIC_FTRACE | 1009 | #ifdef CONFIG_DYNAMIC_FTRACE |
1031 | 1010 | ||
1032 | static struct ftrace_ops *removed_ops; | 1011 | static struct ftrace_ops *removed_ops; |
@@ -1067,7 +1046,7 @@ static const struct ftrace_hash empty_hash = { | |||
1067 | }; | 1046 | }; |
1068 | #define EMPTY_HASH ((struct ftrace_hash *)&empty_hash) | 1047 | #define EMPTY_HASH ((struct ftrace_hash *)&empty_hash) |
1069 | 1048 | ||
1070 | static struct ftrace_ops global_ops = { | 1049 | struct ftrace_ops global_ops = { |
1071 | .func = ftrace_stub, | 1050 | .func = ftrace_stub, |
1072 | .local_hash.notrace_hash = EMPTY_HASH, | 1051 | .local_hash.notrace_hash = EMPTY_HASH, |
1073 | .local_hash.filter_hash = EMPTY_HASH, | 1052 | .local_hash.filter_hash = EMPTY_HASH, |
@@ -1503,7 +1482,7 @@ static bool hash_contains_ip(unsigned long ip, | |||
1503 | * This needs to be called with preemption disabled as | 1482 | * This needs to be called with preemption disabled as |
1504 | * the hashes are freed with call_rcu(). | 1483 | * the hashes are freed with call_rcu(). |
1505 | */ | 1484 | */ |
1506 | static int | 1485 | int |
1507 | ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs) | 1486 | ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs) |
1508 | { | 1487 | { |
1509 | struct ftrace_ops_hash hash; | 1488 | struct ftrace_ops_hash hash; |
@@ -2415,10 +2394,12 @@ __ftrace_replace_code(struct dyn_ftrace *rec, int enable) | |||
2415 | return -1; /* unknow ftrace bug */ | 2394 | return -1; /* unknow ftrace bug */ |
2416 | } | 2395 | } |
2417 | 2396 | ||
2418 | void __weak ftrace_replace_code(int enable) | 2397 | void __weak ftrace_replace_code(int mod_flags) |
2419 | { | 2398 | { |
2420 | struct dyn_ftrace *rec; | 2399 | struct dyn_ftrace *rec; |
2421 | struct ftrace_page *pg; | 2400 | struct ftrace_page *pg; |
2401 | int enable = mod_flags & FTRACE_MODIFY_ENABLE_FL; | ||
2402 | int schedulable = mod_flags & FTRACE_MODIFY_MAY_SLEEP_FL; | ||
2422 | int failed; | 2403 | int failed; |
2423 | 2404 | ||
2424 | if (unlikely(ftrace_disabled)) | 2405 | if (unlikely(ftrace_disabled)) |
@@ -2435,6 +2416,8 @@ void __weak ftrace_replace_code(int enable) | |||
2435 | /* Stop processing */ | 2416 | /* Stop processing */ |
2436 | return; | 2417 | return; |
2437 | } | 2418 | } |
2419 | if (schedulable) | ||
2420 | cond_resched(); | ||
2438 | } while_for_each_ftrace_rec(); | 2421 | } while_for_each_ftrace_rec(); |
2439 | } | 2422 | } |
2440 | 2423 | ||
@@ -2548,8 +2531,12 @@ int __weak ftrace_arch_code_modify_post_process(void) | |||
2548 | void ftrace_modify_all_code(int command) | 2531 | void ftrace_modify_all_code(int command) |
2549 | { | 2532 | { |
2550 | int update = command & FTRACE_UPDATE_TRACE_FUNC; | 2533 | int update = command & FTRACE_UPDATE_TRACE_FUNC; |
2534 | int mod_flags = 0; | ||
2551 | int err = 0; | 2535 | int err = 0; |
2552 | 2536 | ||
2537 | if (command & FTRACE_MAY_SLEEP) | ||
2538 | mod_flags = FTRACE_MODIFY_MAY_SLEEP_FL; | ||
2539 | |||
2553 | /* | 2540 | /* |
2554 | * If the ftrace_caller calls a ftrace_ops func directly, | 2541 | * If the ftrace_caller calls a ftrace_ops func directly, |
2555 | * we need to make sure that it only traces functions it | 2542 | * we need to make sure that it only traces functions it |
@@ -2567,9 +2554,9 @@ void ftrace_modify_all_code(int command) | |||
2567 | } | 2554 | } |
2568 | 2555 | ||
2569 | if (command & FTRACE_UPDATE_CALLS) | 2556 | if (command & FTRACE_UPDATE_CALLS) |
2570 | ftrace_replace_code(1); | 2557 | ftrace_replace_code(mod_flags | FTRACE_MODIFY_ENABLE_FL); |
2571 | else if (command & FTRACE_DISABLE_CALLS) | 2558 | else if (command & FTRACE_DISABLE_CALLS) |
2572 | ftrace_replace_code(0); | 2559 | ftrace_replace_code(mod_flags); |
2573 | 2560 | ||
2574 | if (update && ftrace_trace_function != ftrace_ops_list_func) { | 2561 | if (update && ftrace_trace_function != ftrace_ops_list_func) { |
2575 | function_trace_op = set_function_trace_op; | 2562 | function_trace_op = set_function_trace_op; |
@@ -2682,7 +2669,7 @@ static void ftrace_startup_all(int command) | |||
2682 | update_all_ops = false; | 2669 | update_all_ops = false; |
2683 | } | 2670 | } |
2684 | 2671 | ||
2685 | static int ftrace_startup(struct ftrace_ops *ops, int command) | 2672 | int ftrace_startup(struct ftrace_ops *ops, int command) |
2686 | { | 2673 | { |
2687 | int ret; | 2674 | int ret; |
2688 | 2675 | ||
@@ -2724,7 +2711,7 @@ static int ftrace_startup(struct ftrace_ops *ops, int command) | |||
2724 | return 0; | 2711 | return 0; |
2725 | } | 2712 | } |
2726 | 2713 | ||
2727 | static int ftrace_shutdown(struct ftrace_ops *ops, int command) | 2714 | int ftrace_shutdown(struct ftrace_ops *ops, int command) |
2728 | { | 2715 | { |
2729 | int ret; | 2716 | int ret; |
2730 | 2717 | ||
@@ -6178,7 +6165,7 @@ void ftrace_init_trace_array(struct trace_array *tr) | |||
6178 | } | 6165 | } |
6179 | #else | 6166 | #else |
6180 | 6167 | ||
6181 | static struct ftrace_ops global_ops = { | 6168 | struct ftrace_ops global_ops = { |
6182 | .func = ftrace_stub, | 6169 | .func = ftrace_stub, |
6183 | .flags = FTRACE_OPS_FL_RECURSION_SAFE | | 6170 | .flags = FTRACE_OPS_FL_RECURSION_SAFE | |
6184 | FTRACE_OPS_FL_INITIALIZED | | 6171 | FTRACE_OPS_FL_INITIALIZED | |
@@ -6195,31 +6182,10 @@ core_initcall(ftrace_nodyn_init); | |||
6195 | static inline int ftrace_init_dyn_tracefs(struct dentry *d_tracer) { return 0; } | 6182 | static inline int ftrace_init_dyn_tracefs(struct dentry *d_tracer) { return 0; } |
6196 | static inline void ftrace_startup_enable(int command) { } | 6183 | static inline void ftrace_startup_enable(int command) { } |
6197 | static inline void ftrace_startup_all(int command) { } | 6184 | static inline void ftrace_startup_all(int command) { } |
6198 | /* Keep as macros so we do not need to define the commands */ | ||
6199 | # define ftrace_startup(ops, command) \ | ||
6200 | ({ \ | ||
6201 | int ___ret = __register_ftrace_function(ops); \ | ||
6202 | if (!___ret) \ | ||
6203 | (ops)->flags |= FTRACE_OPS_FL_ENABLED; \ | ||
6204 | ___ret; \ | ||
6205 | }) | ||
6206 | # define ftrace_shutdown(ops, command) \ | ||
6207 | ({ \ | ||
6208 | int ___ret = __unregister_ftrace_function(ops); \ | ||
6209 | if (!___ret) \ | ||
6210 | (ops)->flags &= ~FTRACE_OPS_FL_ENABLED; \ | ||
6211 | ___ret; \ | ||
6212 | }) | ||
6213 | 6185 | ||
6214 | # define ftrace_startup_sysctl() do { } while (0) | 6186 | # define ftrace_startup_sysctl() do { } while (0) |
6215 | # define ftrace_shutdown_sysctl() do { } while (0) | 6187 | # define ftrace_shutdown_sysctl() do { } while (0) |
6216 | 6188 | ||
6217 | static inline int | ||
6218 | ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs) | ||
6219 | { | ||
6220 | return 1; | ||
6221 | } | ||
6222 | |||
6223 | static void ftrace_update_trampoline(struct ftrace_ops *ops) | 6189 | static void ftrace_update_trampoline(struct ftrace_ops *ops) |
6224 | { | 6190 | { |
6225 | } | 6191 | } |
@@ -6746,353 +6712,3 @@ ftrace_enable_sysctl(struct ctl_table *table, int write, | |||
6746 | mutex_unlock(&ftrace_lock); | 6712 | mutex_unlock(&ftrace_lock); |
6747 | return ret; | 6713 | return ret; |
6748 | } | 6714 | } |
6749 | |||
6750 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | ||
6751 | |||
6752 | static struct ftrace_ops graph_ops = { | ||
6753 | .func = ftrace_stub, | ||
6754 | .flags = FTRACE_OPS_FL_RECURSION_SAFE | | ||
6755 | FTRACE_OPS_FL_INITIALIZED | | ||
6756 | FTRACE_OPS_FL_PID | | ||
6757 | FTRACE_OPS_FL_STUB, | ||
6758 | #ifdef FTRACE_GRAPH_TRAMP_ADDR | ||
6759 | .trampoline = FTRACE_GRAPH_TRAMP_ADDR, | ||
6760 | /* trampoline_size is only needed for dynamically allocated tramps */ | ||
6761 | #endif | ||
6762 | ASSIGN_OPS_HASH(graph_ops, &global_ops.local_hash) | ||
6763 | }; | ||
6764 | |||
6765 | void ftrace_graph_sleep_time_control(bool enable) | ||
6766 | { | ||
6767 | fgraph_sleep_time = enable; | ||
6768 | } | ||
6769 | |||
6770 | void ftrace_graph_graph_time_control(bool enable) | ||
6771 | { | ||
6772 | fgraph_graph_time = enable; | ||
6773 | } | ||
6774 | |||
6775 | int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace) | ||
6776 | { | ||
6777 | return 0; | ||
6778 | } | ||
6779 | |||
6780 | /* The callbacks that hook a function */ | ||
6781 | trace_func_graph_ret_t ftrace_graph_return = | ||
6782 | (trace_func_graph_ret_t)ftrace_stub; | ||
6783 | trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub; | ||
6784 | static trace_func_graph_ent_t __ftrace_graph_entry = ftrace_graph_entry_stub; | ||
6785 | |||
6786 | /* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */ | ||
6787 | static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list) | ||
6788 | { | ||
6789 | int i; | ||
6790 | int ret = 0; | ||
6791 | int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE; | ||
6792 | struct task_struct *g, *t; | ||
6793 | |||
6794 | for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) { | ||
6795 | ret_stack_list[i] = | ||
6796 | kmalloc_array(FTRACE_RETFUNC_DEPTH, | ||
6797 | sizeof(struct ftrace_ret_stack), | ||
6798 | GFP_KERNEL); | ||
6799 | if (!ret_stack_list[i]) { | ||
6800 | start = 0; | ||
6801 | end = i; | ||
6802 | ret = -ENOMEM; | ||
6803 | goto free; | ||
6804 | } | ||
6805 | } | ||
6806 | |||
6807 | read_lock(&tasklist_lock); | ||
6808 | do_each_thread(g, t) { | ||
6809 | if (start == end) { | ||
6810 | ret = -EAGAIN; | ||
6811 | goto unlock; | ||
6812 | } | ||
6813 | |||
6814 | if (t->ret_stack == NULL) { | ||
6815 | atomic_set(&t->tracing_graph_pause, 0); | ||
6816 | atomic_set(&t->trace_overrun, 0); | ||
6817 | t->curr_ret_stack = -1; | ||
6818 | t->curr_ret_depth = -1; | ||
6819 | /* Make sure the tasks see the -1 first: */ | ||
6820 | smp_wmb(); | ||
6821 | t->ret_stack = ret_stack_list[start++]; | ||
6822 | } | ||
6823 | } while_each_thread(g, t); | ||
6824 | |||
6825 | unlock: | ||
6826 | read_unlock(&tasklist_lock); | ||
6827 | free: | ||
6828 | for (i = start; i < end; i++) | ||
6829 | kfree(ret_stack_list[i]); | ||
6830 | return ret; | ||
6831 | } | ||
6832 | |||
6833 | static void | ||
6834 | ftrace_graph_probe_sched_switch(void *ignore, bool preempt, | ||
6835 | struct task_struct *prev, struct task_struct *next) | ||
6836 | { | ||
6837 | unsigned long long timestamp; | ||
6838 | int index; | ||
6839 | |||
6840 | /* | ||
6841 | * Does the user want to count the time a function was asleep. | ||
6842 | * If so, do not update the time stamps. | ||
6843 | */ | ||
6844 | if (fgraph_sleep_time) | ||
6845 | return; | ||
6846 | |||
6847 | timestamp = trace_clock_local(); | ||
6848 | |||
6849 | prev->ftrace_timestamp = timestamp; | ||
6850 | |||
6851 | /* only process tasks that we timestamped */ | ||
6852 | if (!next->ftrace_timestamp) | ||
6853 | return; | ||
6854 | |||
6855 | /* | ||
6856 | * Update all the counters in next to make up for the | ||
6857 | * time next was sleeping. | ||
6858 | */ | ||
6859 | timestamp -= next->ftrace_timestamp; | ||
6860 | |||
6861 | for (index = next->curr_ret_stack; index >= 0; index--) | ||
6862 | next->ret_stack[index].calltime += timestamp; | ||
6863 | } | ||
6864 | |||
6865 | /* Allocate a return stack for each task */ | ||
6866 | static int start_graph_tracing(void) | ||
6867 | { | ||
6868 | struct ftrace_ret_stack **ret_stack_list; | ||
6869 | int ret, cpu; | ||
6870 | |||
6871 | ret_stack_list = kmalloc_array(FTRACE_RETSTACK_ALLOC_SIZE, | ||
6872 | sizeof(struct ftrace_ret_stack *), | ||
6873 | GFP_KERNEL); | ||
6874 | |||
6875 | if (!ret_stack_list) | ||
6876 | return -ENOMEM; | ||
6877 | |||
6878 | /* The cpu_boot init_task->ret_stack will never be freed */ | ||
6879 | for_each_online_cpu(cpu) { | ||
6880 | if (!idle_task(cpu)->ret_stack) | ||
6881 | ftrace_graph_init_idle_task(idle_task(cpu), cpu); | ||
6882 | } | ||
6883 | |||
6884 | do { | ||
6885 | ret = alloc_retstack_tasklist(ret_stack_list); | ||
6886 | } while (ret == -EAGAIN); | ||
6887 | |||
6888 | if (!ret) { | ||
6889 | ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL); | ||
6890 | if (ret) | ||
6891 | pr_info("ftrace_graph: Couldn't activate tracepoint" | ||
6892 | " probe to kernel_sched_switch\n"); | ||
6893 | } | ||
6894 | |||
6895 | kfree(ret_stack_list); | ||
6896 | return ret; | ||
6897 | } | ||
6898 | |||
6899 | /* | ||
6900 | * Hibernation protection. | ||
6901 | * The state of the current task is too much unstable during | ||
6902 | * suspend/restore to disk. We want to protect against that. | ||
6903 | */ | ||
6904 | static int | ||
6905 | ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state, | ||
6906 | void *unused) | ||
6907 | { | ||
6908 | switch (state) { | ||
6909 | case PM_HIBERNATION_PREPARE: | ||
6910 | pause_graph_tracing(); | ||
6911 | break; | ||
6912 | |||
6913 | case PM_POST_HIBERNATION: | ||
6914 | unpause_graph_tracing(); | ||
6915 | break; | ||
6916 | } | ||
6917 | return NOTIFY_DONE; | ||
6918 | } | ||
6919 | |||
6920 | static int ftrace_graph_entry_test(struct ftrace_graph_ent *trace) | ||
6921 | { | ||
6922 | if (!ftrace_ops_test(&global_ops, trace->func, NULL)) | ||
6923 | return 0; | ||
6924 | return __ftrace_graph_entry(trace); | ||
6925 | } | ||
6926 | |||
6927 | /* | ||
6928 | * The function graph tracer should only trace the functions defined | ||
6929 | * by set_ftrace_filter and set_ftrace_notrace. If another function | ||
6930 | * tracer ops is registered, the graph tracer requires testing the | ||
6931 | * function against the global ops, and not just trace any function | ||
6932 | * that any ftrace_ops registered. | ||
6933 | */ | ||
6934 | static void update_function_graph_func(void) | ||
6935 | { | ||
6936 | struct ftrace_ops *op; | ||
6937 | bool do_test = false; | ||
6938 | |||
6939 | /* | ||
6940 | * The graph and global ops share the same set of functions | ||
6941 | * to test. If any other ops is on the list, then | ||
6942 | * the graph tracing needs to test if its the function | ||
6943 | * it should call. | ||
6944 | */ | ||
6945 | do_for_each_ftrace_op(op, ftrace_ops_list) { | ||
6946 | if (op != &global_ops && op != &graph_ops && | ||
6947 | op != &ftrace_list_end) { | ||
6948 | do_test = true; | ||
6949 | /* in double loop, break out with goto */ | ||
6950 | goto out; | ||
6951 | } | ||
6952 | } while_for_each_ftrace_op(op); | ||
6953 | out: | ||
6954 | if (do_test) | ||
6955 | ftrace_graph_entry = ftrace_graph_entry_test; | ||
6956 | else | ||
6957 | ftrace_graph_entry = __ftrace_graph_entry; | ||
6958 | } | ||
6959 | |||
6960 | static struct notifier_block ftrace_suspend_notifier = { | ||
6961 | .notifier_call = ftrace_suspend_notifier_call, | ||
6962 | }; | ||
6963 | |||
6964 | int register_ftrace_graph(trace_func_graph_ret_t retfunc, | ||
6965 | trace_func_graph_ent_t entryfunc) | ||
6966 | { | ||
6967 | int ret = 0; | ||
6968 | |||
6969 | mutex_lock(&ftrace_lock); | ||
6970 | |||
6971 | /* we currently allow only one tracer registered at a time */ | ||
6972 | if (ftrace_graph_active) { | ||
6973 | ret = -EBUSY; | ||
6974 | goto out; | ||
6975 | } | ||
6976 | |||
6977 | register_pm_notifier(&ftrace_suspend_notifier); | ||
6978 | |||
6979 | ftrace_graph_active++; | ||
6980 | ret = start_graph_tracing(); | ||
6981 | if (ret) { | ||
6982 | ftrace_graph_active--; | ||
6983 | goto out; | ||
6984 | } | ||
6985 | |||
6986 | ftrace_graph_return = retfunc; | ||
6987 | |||
6988 | /* | ||
6989 | * Update the indirect function to the entryfunc, and the | ||
6990 | * function that gets called to the entry_test first. Then | ||
6991 | * call the update fgraph entry function to determine if | ||
6992 | * the entryfunc should be called directly or not. | ||
6993 | */ | ||
6994 | __ftrace_graph_entry = entryfunc; | ||
6995 | ftrace_graph_entry = ftrace_graph_entry_test; | ||
6996 | update_function_graph_func(); | ||
6997 | |||
6998 | ret = ftrace_startup(&graph_ops, FTRACE_START_FUNC_RET); | ||
6999 | out: | ||
7000 | mutex_unlock(&ftrace_lock); | ||
7001 | return ret; | ||
7002 | } | ||
7003 | |||
7004 | void unregister_ftrace_graph(void) | ||
7005 | { | ||
7006 | mutex_lock(&ftrace_lock); | ||
7007 | |||
7008 | if (unlikely(!ftrace_graph_active)) | ||
7009 | goto out; | ||
7010 | |||
7011 | ftrace_graph_active--; | ||
7012 | ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub; | ||
7013 | ftrace_graph_entry = ftrace_graph_entry_stub; | ||
7014 | __ftrace_graph_entry = ftrace_graph_entry_stub; | ||
7015 | ftrace_shutdown(&graph_ops, FTRACE_STOP_FUNC_RET); | ||
7016 | unregister_pm_notifier(&ftrace_suspend_notifier); | ||
7017 | unregister_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL); | ||
7018 | |||
7019 | out: | ||
7020 | mutex_unlock(&ftrace_lock); | ||
7021 | } | ||
7022 | |||
7023 | static DEFINE_PER_CPU(struct ftrace_ret_stack *, idle_ret_stack); | ||
7024 | |||
7025 | static void | ||
7026 | graph_init_task(struct task_struct *t, struct ftrace_ret_stack *ret_stack) | ||
7027 | { | ||
7028 | atomic_set(&t->tracing_graph_pause, 0); | ||
7029 | atomic_set(&t->trace_overrun, 0); | ||
7030 | t->ftrace_timestamp = 0; | ||
7031 | /* make curr_ret_stack visible before we add the ret_stack */ | ||
7032 | smp_wmb(); | ||
7033 | t->ret_stack = ret_stack; | ||
7034 | } | ||
7035 | |||
7036 | /* | ||
7037 | * Allocate a return stack for the idle task. May be the first | ||
7038 | * time through, or it may be done by CPU hotplug online. | ||
7039 | */ | ||
7040 | void ftrace_graph_init_idle_task(struct task_struct *t, int cpu) | ||
7041 | { | ||
7042 | t->curr_ret_stack = -1; | ||
7043 | t->curr_ret_depth = -1; | ||
7044 | /* | ||
7045 | * The idle task has no parent, it either has its own | ||
7046 | * stack or no stack at all. | ||
7047 | */ | ||
7048 | if (t->ret_stack) | ||
7049 | WARN_ON(t->ret_stack != per_cpu(idle_ret_stack, cpu)); | ||
7050 | |||
7051 | if (ftrace_graph_active) { | ||
7052 | struct ftrace_ret_stack *ret_stack; | ||
7053 | |||
7054 | ret_stack = per_cpu(idle_ret_stack, cpu); | ||
7055 | if (!ret_stack) { | ||
7056 | ret_stack = | ||
7057 | kmalloc_array(FTRACE_RETFUNC_DEPTH, | ||
7058 | sizeof(struct ftrace_ret_stack), | ||
7059 | GFP_KERNEL); | ||
7060 | if (!ret_stack) | ||
7061 | return; | ||
7062 | per_cpu(idle_ret_stack, cpu) = ret_stack; | ||
7063 | } | ||
7064 | graph_init_task(t, ret_stack); | ||
7065 | } | ||
7066 | } | ||
7067 | |||
7068 | /* Allocate a return stack for newly created task */ | ||
7069 | void ftrace_graph_init_task(struct task_struct *t) | ||
7070 | { | ||
7071 | /* Make sure we do not use the parent ret_stack */ | ||
7072 | t->ret_stack = NULL; | ||
7073 | t->curr_ret_stack = -1; | ||
7074 | t->curr_ret_depth = -1; | ||
7075 | |||
7076 | if (ftrace_graph_active) { | ||
7077 | struct ftrace_ret_stack *ret_stack; | ||
7078 | |||
7079 | ret_stack = kmalloc_array(FTRACE_RETFUNC_DEPTH, | ||
7080 | sizeof(struct ftrace_ret_stack), | ||
7081 | GFP_KERNEL); | ||
7082 | if (!ret_stack) | ||
7083 | return; | ||
7084 | graph_init_task(t, ret_stack); | ||
7085 | } | ||
7086 | } | ||
7087 | |||
7088 | void ftrace_graph_exit_task(struct task_struct *t) | ||
7089 | { | ||
7090 | struct ftrace_ret_stack *ret_stack = t->ret_stack; | ||
7091 | |||
7092 | t->ret_stack = NULL; | ||
7093 | /* NULL must become visible to IRQs before we free it: */ | ||
7094 | barrier(); | ||
7095 | |||
7096 | kfree(ret_stack); | ||
7097 | } | ||
7098 | #endif | ||
diff --git a/kernel/trace/ftrace_internal.h b/kernel/trace/ftrace_internal.h new file mode 100644 index 000000000000..0515a2096f90 --- /dev/null +++ b/kernel/trace/ftrace_internal.h | |||
@@ -0,0 +1,75 @@ | |||
1 | /* SPDX-License-Identifier: GPL-2.0 */ | ||
2 | #ifndef _LINUX_KERNEL_FTRACE_INTERNAL_H | ||
3 | #define _LINUX_KERNEL_FTRACE_INTERNAL_H | ||
4 | |||
5 | #ifdef CONFIG_FUNCTION_TRACER | ||
6 | |||
7 | /* | ||
8 | * Traverse the ftrace_global_list, invoking all entries. The reason that we | ||
9 | * can use rcu_dereference_raw_notrace() is that elements removed from this list | ||
10 | * are simply leaked, so there is no need to interact with a grace-period | ||
11 | * mechanism. The rcu_dereference_raw_notrace() calls are needed to handle | ||
12 | * concurrent insertions into the ftrace_global_list. | ||
13 | * | ||
14 | * Silly Alpha and silly pointer-speculation compiler optimizations! | ||
15 | */ | ||
16 | #define do_for_each_ftrace_op(op, list) \ | ||
17 | op = rcu_dereference_raw_notrace(list); \ | ||
18 | do | ||
19 | |||
20 | /* | ||
21 | * Optimized for just a single item in the list (as that is the normal case). | ||
22 | */ | ||
23 | #define while_for_each_ftrace_op(op) \ | ||
24 | while (likely(op = rcu_dereference_raw_notrace((op)->next)) && \ | ||
25 | unlikely((op) != &ftrace_list_end)) | ||
26 | |||
27 | extern struct ftrace_ops __rcu *ftrace_ops_list; | ||
28 | extern struct ftrace_ops ftrace_list_end; | ||
29 | extern struct mutex ftrace_lock; | ||
30 | extern struct ftrace_ops global_ops; | ||
31 | |||
32 | #ifdef CONFIG_DYNAMIC_FTRACE | ||
33 | |||
34 | int ftrace_startup(struct ftrace_ops *ops, int command); | ||
35 | int ftrace_shutdown(struct ftrace_ops *ops, int command); | ||
36 | int ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs); | ||
37 | |||
38 | #else /* !CONFIG_DYNAMIC_FTRACE */ | ||
39 | |||
40 | int __register_ftrace_function(struct ftrace_ops *ops); | ||
41 | int __unregister_ftrace_function(struct ftrace_ops *ops); | ||
42 | /* Keep as macros so we do not need to define the commands */ | ||
43 | # define ftrace_startup(ops, command) \ | ||
44 | ({ \ | ||
45 | int ___ret = __register_ftrace_function(ops); \ | ||
46 | if (!___ret) \ | ||
47 | (ops)->flags |= FTRACE_OPS_FL_ENABLED; \ | ||
48 | ___ret; \ | ||
49 | }) | ||
50 | # define ftrace_shutdown(ops, command) \ | ||
51 | ({ \ | ||
52 | int ___ret = __unregister_ftrace_function(ops); \ | ||
53 | if (!___ret) \ | ||
54 | (ops)->flags &= ~FTRACE_OPS_FL_ENABLED; \ | ||
55 | ___ret; \ | ||
56 | }) | ||
57 | static inline int | ||
58 | ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs) | ||
59 | { | ||
60 | return 1; | ||
61 | } | ||
62 | #endif /* CONFIG_DYNAMIC_FTRACE */ | ||
63 | |||
64 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | ||
65 | extern int ftrace_graph_active; | ||
66 | void update_function_graph_func(void); | ||
67 | #else /* !CONFIG_FUNCTION_GRAPH_TRACER */ | ||
68 | # define ftrace_graph_active 0 | ||
69 | static inline void update_function_graph_func(void) { } | ||
70 | #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ | ||
71 | |||
72 | #else /* !CONFIG_FUNCTION_TRACER */ | ||
73 | #endif /* CONFIG_FUNCTION_TRACER */ | ||
74 | |||
75 | #endif | ||
diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 4f3247a53259..06e864a334bb 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c | |||
@@ -487,6 +487,10 @@ struct ring_buffer_per_cpu { | |||
487 | local_t dropped_events; | 487 | local_t dropped_events; |
488 | local_t committing; | 488 | local_t committing; |
489 | local_t commits; | 489 | local_t commits; |
490 | local_t pages_touched; | ||
491 | local_t pages_read; | ||
492 | long last_pages_touch; | ||
493 | size_t shortest_full; | ||
490 | unsigned long read; | 494 | unsigned long read; |
491 | unsigned long read_bytes; | 495 | unsigned long read_bytes; |
492 | u64 write_stamp; | 496 | u64 write_stamp; |
@@ -529,6 +533,41 @@ struct ring_buffer_iter { | |||
529 | u64 read_stamp; | 533 | u64 read_stamp; |
530 | }; | 534 | }; |
531 | 535 | ||
536 | /** | ||
537 | * ring_buffer_nr_pages - get the number of buffer pages in the ring buffer | ||
538 | * @buffer: The ring_buffer to get the number of pages from | ||
539 | * @cpu: The cpu of the ring_buffer to get the number of pages from | ||
540 | * | ||
541 | * Returns the number of pages used by a per_cpu buffer of the ring buffer. | ||
542 | */ | ||
543 | size_t ring_buffer_nr_pages(struct ring_buffer *buffer, int cpu) | ||
544 | { | ||
545 | return buffer->buffers[cpu]->nr_pages; | ||
546 | } | ||
547 | |||
548 | /** | ||
549 | * ring_buffer_nr_pages_dirty - get the number of used pages in the ring buffer | ||
550 | * @buffer: The ring_buffer to get the number of pages from | ||
551 | * @cpu: The cpu of the ring_buffer to get the number of pages from | ||
552 | * | ||
553 | * Returns the number of pages that have content in the ring buffer. | ||
554 | */ | ||
555 | size_t ring_buffer_nr_dirty_pages(struct ring_buffer *buffer, int cpu) | ||
556 | { | ||
557 | size_t read; | ||
558 | size_t cnt; | ||
559 | |||
560 | read = local_read(&buffer->buffers[cpu]->pages_read); | ||
561 | cnt = local_read(&buffer->buffers[cpu]->pages_touched); | ||
562 | /* The reader can read an empty page, but not more than that */ | ||
563 | if (cnt < read) { | ||
564 | WARN_ON_ONCE(read > cnt + 1); | ||
565 | return 0; | ||
566 | } | ||
567 | |||
568 | return cnt - read; | ||
569 | } | ||
570 | |||
532 | /* | 571 | /* |
533 | * rb_wake_up_waiters - wake up tasks waiting for ring buffer input | 572 | * rb_wake_up_waiters - wake up tasks waiting for ring buffer input |
534 | * | 573 | * |
@@ -556,7 +595,7 @@ static void rb_wake_up_waiters(struct irq_work *work) | |||
556 | * as data is added to any of the @buffer's cpu buffers. Otherwise | 595 | * as data is added to any of the @buffer's cpu buffers. Otherwise |
557 | * it will wait for data to be added to a specific cpu buffer. | 596 | * it will wait for data to be added to a specific cpu buffer. |
558 | */ | 597 | */ |
559 | int ring_buffer_wait(struct ring_buffer *buffer, int cpu, bool full) | 598 | int ring_buffer_wait(struct ring_buffer *buffer, int cpu, int full) |
560 | { | 599 | { |
561 | struct ring_buffer_per_cpu *uninitialized_var(cpu_buffer); | 600 | struct ring_buffer_per_cpu *uninitialized_var(cpu_buffer); |
562 | DEFINE_WAIT(wait); | 601 | DEFINE_WAIT(wait); |
@@ -571,7 +610,7 @@ int ring_buffer_wait(struct ring_buffer *buffer, int cpu, bool full) | |||
571 | if (cpu == RING_BUFFER_ALL_CPUS) { | 610 | if (cpu == RING_BUFFER_ALL_CPUS) { |
572 | work = &buffer->irq_work; | 611 | work = &buffer->irq_work; |
573 | /* Full only makes sense on per cpu reads */ | 612 | /* Full only makes sense on per cpu reads */ |
574 | full = false; | 613 | full = 0; |
575 | } else { | 614 | } else { |
576 | if (!cpumask_test_cpu(cpu, buffer->cpumask)) | 615 | if (!cpumask_test_cpu(cpu, buffer->cpumask)) |
577 | return -ENODEV; | 616 | return -ENODEV; |
@@ -623,15 +662,22 @@ int ring_buffer_wait(struct ring_buffer *buffer, int cpu, bool full) | |||
623 | !ring_buffer_empty_cpu(buffer, cpu)) { | 662 | !ring_buffer_empty_cpu(buffer, cpu)) { |
624 | unsigned long flags; | 663 | unsigned long flags; |
625 | bool pagebusy; | 664 | bool pagebusy; |
665 | size_t nr_pages; | ||
666 | size_t dirty; | ||
626 | 667 | ||
627 | if (!full) | 668 | if (!full) |
628 | break; | 669 | break; |
629 | 670 | ||
630 | raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); | 671 | raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); |
631 | pagebusy = cpu_buffer->reader_page == cpu_buffer->commit_page; | 672 | pagebusy = cpu_buffer->reader_page == cpu_buffer->commit_page; |
673 | nr_pages = cpu_buffer->nr_pages; | ||
674 | dirty = ring_buffer_nr_dirty_pages(buffer, cpu); | ||
675 | if (!cpu_buffer->shortest_full || | ||
676 | cpu_buffer->shortest_full < full) | ||
677 | cpu_buffer->shortest_full = full; | ||
632 | raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); | 678 | raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); |
633 | 679 | if (!pagebusy && | |
634 | if (!pagebusy) | 680 | (!nr_pages || (dirty * 100) > full * nr_pages)) |
635 | break; | 681 | break; |
636 | } | 682 | } |
637 | 683 | ||
@@ -1054,6 +1100,7 @@ static void rb_tail_page_update(struct ring_buffer_per_cpu *cpu_buffer, | |||
1054 | old_write = local_add_return(RB_WRITE_INTCNT, &next_page->write); | 1100 | old_write = local_add_return(RB_WRITE_INTCNT, &next_page->write); |
1055 | old_entries = local_add_return(RB_WRITE_INTCNT, &next_page->entries); | 1101 | old_entries = local_add_return(RB_WRITE_INTCNT, &next_page->entries); |
1056 | 1102 | ||
1103 | local_inc(&cpu_buffer->pages_touched); | ||
1057 | /* | 1104 | /* |
1058 | * Just make sure we have seen our old_write and synchronize | 1105 | * Just make sure we have seen our old_write and synchronize |
1059 | * with any interrupts that come in. | 1106 | * with any interrupts that come in. |
@@ -2586,7 +2633,9 @@ static void rb_commit(struct ring_buffer_per_cpu *cpu_buffer, | |||
2586 | static __always_inline void | 2633 | static __always_inline void |
2587 | rb_wakeups(struct ring_buffer *buffer, struct ring_buffer_per_cpu *cpu_buffer) | 2634 | rb_wakeups(struct ring_buffer *buffer, struct ring_buffer_per_cpu *cpu_buffer) |
2588 | { | 2635 | { |
2589 | bool pagebusy; | 2636 | size_t nr_pages; |
2637 | size_t dirty; | ||
2638 | size_t full; | ||
2590 | 2639 | ||
2591 | if (buffer->irq_work.waiters_pending) { | 2640 | if (buffer->irq_work.waiters_pending) { |
2592 | buffer->irq_work.waiters_pending = false; | 2641 | buffer->irq_work.waiters_pending = false; |
@@ -2600,14 +2649,27 @@ rb_wakeups(struct ring_buffer *buffer, struct ring_buffer_per_cpu *cpu_buffer) | |||
2600 | irq_work_queue(&cpu_buffer->irq_work.work); | 2649 | irq_work_queue(&cpu_buffer->irq_work.work); |
2601 | } | 2650 | } |
2602 | 2651 | ||
2603 | pagebusy = cpu_buffer->reader_page == cpu_buffer->commit_page; | 2652 | if (cpu_buffer->last_pages_touch == local_read(&cpu_buffer->pages_touched)) |
2653 | return; | ||
2604 | 2654 | ||
2605 | if (!pagebusy && cpu_buffer->irq_work.full_waiters_pending) { | 2655 | if (cpu_buffer->reader_page == cpu_buffer->commit_page) |
2606 | cpu_buffer->irq_work.wakeup_full = true; | 2656 | return; |
2607 | cpu_buffer->irq_work.full_waiters_pending = false; | 2657 | |
2608 | /* irq_work_queue() supplies it's own memory barriers */ | 2658 | if (!cpu_buffer->irq_work.full_waiters_pending) |
2609 | irq_work_queue(&cpu_buffer->irq_work.work); | 2659 | return; |
2610 | } | 2660 | |
2661 | cpu_buffer->last_pages_touch = local_read(&cpu_buffer->pages_touched); | ||
2662 | |||
2663 | full = cpu_buffer->shortest_full; | ||
2664 | nr_pages = cpu_buffer->nr_pages; | ||
2665 | dirty = ring_buffer_nr_dirty_pages(buffer, cpu_buffer->cpu); | ||
2666 | if (full && nr_pages && (dirty * 100) <= full * nr_pages) | ||
2667 | return; | ||
2668 | |||
2669 | cpu_buffer->irq_work.wakeup_full = true; | ||
2670 | cpu_buffer->irq_work.full_waiters_pending = false; | ||
2671 | /* irq_work_queue() supplies it's own memory barriers */ | ||
2672 | irq_work_queue(&cpu_buffer->irq_work.work); | ||
2611 | } | 2673 | } |
2612 | 2674 | ||
2613 | /* | 2675 | /* |
@@ -3732,13 +3794,15 @@ rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer) | |||
3732 | goto spin; | 3794 | goto spin; |
3733 | 3795 | ||
3734 | /* | 3796 | /* |
3735 | * Yeah! We succeeded in replacing the page. | 3797 | * Yay! We succeeded in replacing the page. |
3736 | * | 3798 | * |
3737 | * Now make the new head point back to the reader page. | 3799 | * Now make the new head point back to the reader page. |
3738 | */ | 3800 | */ |
3739 | rb_list_head(reader->list.next)->prev = &cpu_buffer->reader_page->list; | 3801 | rb_list_head(reader->list.next)->prev = &cpu_buffer->reader_page->list; |
3740 | rb_inc_page(cpu_buffer, &cpu_buffer->head_page); | 3802 | rb_inc_page(cpu_buffer, &cpu_buffer->head_page); |
3741 | 3803 | ||
3804 | local_inc(&cpu_buffer->pages_read); | ||
3805 | |||
3742 | /* Finally update the reader page to the new head */ | 3806 | /* Finally update the reader page to the new head */ |
3743 | cpu_buffer->reader_page = reader; | 3807 | cpu_buffer->reader_page = reader; |
3744 | cpu_buffer->reader_page->read = 0; | 3808 | cpu_buffer->reader_page->read = 0; |
@@ -4334,6 +4398,10 @@ rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer) | |||
4334 | local_set(&cpu_buffer->entries, 0); | 4398 | local_set(&cpu_buffer->entries, 0); |
4335 | local_set(&cpu_buffer->committing, 0); | 4399 | local_set(&cpu_buffer->committing, 0); |
4336 | local_set(&cpu_buffer->commits, 0); | 4400 | local_set(&cpu_buffer->commits, 0); |
4401 | local_set(&cpu_buffer->pages_touched, 0); | ||
4402 | local_set(&cpu_buffer->pages_read, 0); | ||
4403 | cpu_buffer->last_pages_touch = 0; | ||
4404 | cpu_buffer->shortest_full = 0; | ||
4337 | cpu_buffer->read = 0; | 4405 | cpu_buffer->read = 0; |
4338 | cpu_buffer->read_bytes = 0; | 4406 | cpu_buffer->read_bytes = 0; |
4339 | 4407 | ||
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 51612b4a603f..c521b7347482 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c | |||
@@ -1431,7 +1431,7 @@ update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu) | |||
1431 | } | 1431 | } |
1432 | #endif /* CONFIG_TRACER_MAX_TRACE */ | 1432 | #endif /* CONFIG_TRACER_MAX_TRACE */ |
1433 | 1433 | ||
1434 | static int wait_on_pipe(struct trace_iterator *iter, bool full) | 1434 | static int wait_on_pipe(struct trace_iterator *iter, int full) |
1435 | { | 1435 | { |
1436 | /* Iterators are static, they should be filled or empty */ | 1436 | /* Iterators are static, they should be filled or empty */ |
1437 | if (trace_buffer_iter(iter, iter->cpu_file)) | 1437 | if (trace_buffer_iter(iter, iter->cpu_file)) |
@@ -2452,7 +2452,7 @@ static inline void ftrace_exports_disable(void) | |||
2452 | static_branch_disable(&ftrace_exports_enabled); | 2452 | static_branch_disable(&ftrace_exports_enabled); |
2453 | } | 2453 | } |
2454 | 2454 | ||
2455 | void ftrace_exports(struct ring_buffer_event *event) | 2455 | static void ftrace_exports(struct ring_buffer_event *event) |
2456 | { | 2456 | { |
2457 | struct trace_export *export; | 2457 | struct trace_export *export; |
2458 | 2458 | ||
@@ -4408,13 +4408,15 @@ static int trace_set_options(struct trace_array *tr, char *option) | |||
4408 | int neg = 0; | 4408 | int neg = 0; |
4409 | int ret; | 4409 | int ret; |
4410 | size_t orig_len = strlen(option); | 4410 | size_t orig_len = strlen(option); |
4411 | int len; | ||
4411 | 4412 | ||
4412 | cmp = strstrip(option); | 4413 | cmp = strstrip(option); |
4413 | 4414 | ||
4414 | if (strncmp(cmp, "no", 2) == 0) { | 4415 | len = str_has_prefix(cmp, "no"); |
4416 | if (len) | ||
4415 | neg = 1; | 4417 | neg = 1; |
4416 | cmp += 2; | 4418 | |
4417 | } | 4419 | cmp += len; |
4418 | 4420 | ||
4419 | mutex_lock(&trace_types_lock); | 4421 | mutex_lock(&trace_types_lock); |
4420 | 4422 | ||
@@ -4604,6 +4606,10 @@ static const char readme_msg[] = | |||
4604 | "\t\t\t traces\n" | 4606 | "\t\t\t traces\n" |
4605 | #endif | 4607 | #endif |
4606 | #endif /* CONFIG_STACK_TRACER */ | 4608 | #endif /* CONFIG_STACK_TRACER */ |
4609 | #ifdef CONFIG_DYNAMIC_EVENTS | ||
4610 | " dynamic_events\t\t- Add/remove/show the generic dynamic events\n" | ||
4611 | "\t\t\t Write into this file to define/undefine new trace events.\n" | ||
4612 | #endif | ||
4607 | #ifdef CONFIG_KPROBE_EVENTS | 4613 | #ifdef CONFIG_KPROBE_EVENTS |
4608 | " kprobe_events\t\t- Add/remove/show the kernel dynamic events\n" | 4614 | " kprobe_events\t\t- Add/remove/show the kernel dynamic events\n" |
4609 | "\t\t\t Write into this file to define/undefine new trace events.\n" | 4615 | "\t\t\t Write into this file to define/undefine new trace events.\n" |
@@ -4616,6 +4622,9 @@ static const char readme_msg[] = | |||
4616 | "\t accepts: event-definitions (one definition per line)\n" | 4622 | "\t accepts: event-definitions (one definition per line)\n" |
4617 | "\t Format: p[:[<group>/]<event>] <place> [<args>]\n" | 4623 | "\t Format: p[:[<group>/]<event>] <place> [<args>]\n" |
4618 | "\t r[maxactive][:[<group>/]<event>] <place> [<args>]\n" | 4624 | "\t r[maxactive][:[<group>/]<event>] <place> [<args>]\n" |
4625 | #ifdef CONFIG_HIST_TRIGGERS | ||
4626 | "\t s:[synthetic/]<event> <field> [<field>]\n" | ||
4627 | #endif | ||
4619 | "\t -:[<group>/]<event>\n" | 4628 | "\t -:[<group>/]<event>\n" |
4620 | #ifdef CONFIG_KPROBE_EVENTS | 4629 | #ifdef CONFIG_KPROBE_EVENTS |
4621 | "\t place: [<module>:]<symbol>[+<offset>]|<memaddr>\n" | 4630 | "\t place: [<module>:]<symbol>[+<offset>]|<memaddr>\n" |
@@ -4634,6 +4643,11 @@ static const char readme_msg[] = | |||
4634 | "\t type: s8/16/32/64, u8/16/32/64, x8/16/32/64, string, symbol,\n" | 4643 | "\t type: s8/16/32/64, u8/16/32/64, x8/16/32/64, string, symbol,\n" |
4635 | "\t b<bit-width>@<bit-offset>/<container-size>,\n" | 4644 | "\t b<bit-width>@<bit-offset>/<container-size>,\n" |
4636 | "\t <type>\\[<array-size>\\]\n" | 4645 | "\t <type>\\[<array-size>\\]\n" |
4646 | #ifdef CONFIG_HIST_TRIGGERS | ||
4647 | "\t field: <stype> <name>;\n" | ||
4648 | "\t stype: u8/u16/u32/u64, s8/s16/s32/s64, pid_t,\n" | ||
4649 | "\t [unsigned] char/int/long\n" | ||
4650 | #endif | ||
4637 | #endif | 4651 | #endif |
4638 | " events/\t\t- Directory containing all trace event subsystems:\n" | 4652 | " events/\t\t- Directory containing all trace event subsystems:\n" |
4639 | " enable\t\t- Write 0/1 to enable/disable tracing of all events\n" | 4653 | " enable\t\t- Write 0/1 to enable/disable tracing of all events\n" |
@@ -5693,7 +5707,7 @@ static int tracing_wait_pipe(struct file *filp) | |||
5693 | 5707 | ||
5694 | mutex_unlock(&iter->mutex); | 5708 | mutex_unlock(&iter->mutex); |
5695 | 5709 | ||
5696 | ret = wait_on_pipe(iter, false); | 5710 | ret = wait_on_pipe(iter, 0); |
5697 | 5711 | ||
5698 | mutex_lock(&iter->mutex); | 5712 | mutex_lock(&iter->mutex); |
5699 | 5713 | ||
@@ -6751,7 +6765,7 @@ tracing_buffers_read(struct file *filp, char __user *ubuf, | |||
6751 | if ((filp->f_flags & O_NONBLOCK)) | 6765 | if ((filp->f_flags & O_NONBLOCK)) |
6752 | return -EAGAIN; | 6766 | return -EAGAIN; |
6753 | 6767 | ||
6754 | ret = wait_on_pipe(iter, false); | 6768 | ret = wait_on_pipe(iter, 0); |
6755 | if (ret) | 6769 | if (ret) |
6756 | return ret; | 6770 | return ret; |
6757 | 6771 | ||
@@ -6948,7 +6962,7 @@ tracing_buffers_splice_read(struct file *file, loff_t *ppos, | |||
6948 | if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK)) | 6962 | if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK)) |
6949 | goto out; | 6963 | goto out; |
6950 | 6964 | ||
6951 | ret = wait_on_pipe(iter, true); | 6965 | ret = wait_on_pipe(iter, iter->tr->buffer_percent); |
6952 | if (ret) | 6966 | if (ret) |
6953 | goto out; | 6967 | goto out; |
6954 | 6968 | ||
@@ -7662,6 +7676,53 @@ static const struct file_operations rb_simple_fops = { | |||
7662 | .llseek = default_llseek, | 7676 | .llseek = default_llseek, |
7663 | }; | 7677 | }; |
7664 | 7678 | ||
7679 | static ssize_t | ||
7680 | buffer_percent_read(struct file *filp, char __user *ubuf, | ||
7681 | size_t cnt, loff_t *ppos) | ||
7682 | { | ||
7683 | struct trace_array *tr = filp->private_data; | ||
7684 | char buf[64]; | ||
7685 | int r; | ||
7686 | |||
7687 | r = tr->buffer_percent; | ||
7688 | r = sprintf(buf, "%d\n", r); | ||
7689 | |||
7690 | return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); | ||
7691 | } | ||
7692 | |||
7693 | static ssize_t | ||
7694 | buffer_percent_write(struct file *filp, const char __user *ubuf, | ||
7695 | size_t cnt, loff_t *ppos) | ||
7696 | { | ||
7697 | struct trace_array *tr = filp->private_data; | ||
7698 | unsigned long val; | ||
7699 | int ret; | ||
7700 | |||
7701 | ret = kstrtoul_from_user(ubuf, cnt, 10, &val); | ||
7702 | if (ret) | ||
7703 | return ret; | ||
7704 | |||
7705 | if (val > 100) | ||
7706 | return -EINVAL; | ||
7707 | |||
7708 | if (!val) | ||
7709 | val = 1; | ||
7710 | |||
7711 | tr->buffer_percent = val; | ||
7712 | |||
7713 | (*ppos)++; | ||
7714 | |||
7715 | return cnt; | ||
7716 | } | ||
7717 | |||
7718 | static const struct file_operations buffer_percent_fops = { | ||
7719 | .open = tracing_open_generic_tr, | ||
7720 | .read = buffer_percent_read, | ||
7721 | .write = buffer_percent_write, | ||
7722 | .release = tracing_release_generic_tr, | ||
7723 | .llseek = default_llseek, | ||
7724 | }; | ||
7725 | |||
7665 | struct dentry *trace_instance_dir; | 7726 | struct dentry *trace_instance_dir; |
7666 | 7727 | ||
7667 | static void | 7728 | static void |
@@ -7970,6 +8031,11 @@ init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer) | |||
7970 | trace_create_file("timestamp_mode", 0444, d_tracer, tr, | 8031 | trace_create_file("timestamp_mode", 0444, d_tracer, tr, |
7971 | &trace_time_stamp_mode_fops); | 8032 | &trace_time_stamp_mode_fops); |
7972 | 8033 | ||
8034 | tr->buffer_percent = 50; | ||
8035 | |||
8036 | trace_create_file("buffer_percent", 0444, d_tracer, | ||
8037 | tr, &buffer_percent_fops); | ||
8038 | |||
7973 | create_trace_options_dir(tr); | 8039 | create_trace_options_dir(tr); |
7974 | 8040 | ||
7975 | #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER) | 8041 | #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER) |
diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index 447bd96ee658..08900828d282 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h | |||
@@ -247,6 +247,7 @@ struct trace_array { | |||
247 | int clock_id; | 247 | int clock_id; |
248 | int nr_topts; | 248 | int nr_topts; |
249 | bool clear_trace; | 249 | bool clear_trace; |
250 | int buffer_percent; | ||
250 | struct tracer *current_trace; | 251 | struct tracer *current_trace; |
251 | unsigned int trace_flags; | 252 | unsigned int trace_flags; |
252 | unsigned char trace_flags_index[TRACE_FLAGS_MAX_SIZE]; | 253 | unsigned char trace_flags_index[TRACE_FLAGS_MAX_SIZE]; |
@@ -534,6 +535,13 @@ enum { | |||
534 | 535 | ||
535 | TRACE_GRAPH_DEPTH_START_BIT, | 536 | TRACE_GRAPH_DEPTH_START_BIT, |
536 | TRACE_GRAPH_DEPTH_END_BIT, | 537 | TRACE_GRAPH_DEPTH_END_BIT, |
538 | |||
539 | /* | ||
540 | * To implement set_graph_notrace, if this bit is set, we ignore | ||
541 | * function graph tracing of called functions, until the return | ||
542 | * function is called to clear it. | ||
543 | */ | ||
544 | TRACE_GRAPH_NOTRACE_BIT, | ||
537 | }; | 545 | }; |
538 | 546 | ||
539 | #define trace_recursion_set(bit) do { (current)->trace_recursion |= (1<<(bit)); } while (0) | 547 | #define trace_recursion_set(bit) do { (current)->trace_recursion |= (1<<(bit)); } while (0) |
@@ -855,7 +863,12 @@ static __always_inline bool ftrace_hash_empty(struct ftrace_hash *hash) | |||
855 | #define TRACE_GRAPH_PRINT_FILL_MASK (0x3 << TRACE_GRAPH_PRINT_FILL_SHIFT) | 863 | #define TRACE_GRAPH_PRINT_FILL_MASK (0x3 << TRACE_GRAPH_PRINT_FILL_SHIFT) |
856 | 864 | ||
857 | extern void ftrace_graph_sleep_time_control(bool enable); | 865 | extern void ftrace_graph_sleep_time_control(bool enable); |
866 | |||
867 | #ifdef CONFIG_FUNCTION_PROFILER | ||
858 | extern void ftrace_graph_graph_time_control(bool enable); | 868 | extern void ftrace_graph_graph_time_control(bool enable); |
869 | #else | ||
870 | static inline void ftrace_graph_graph_time_control(bool enable) { } | ||
871 | #endif | ||
859 | 872 | ||
860 | extern enum print_line_t | 873 | extern enum print_line_t |
861 | print_graph_function_flags(struct trace_iterator *iter, u32 flags); | 874 | print_graph_function_flags(struct trace_iterator *iter, u32 flags); |
diff --git a/kernel/trace/trace_dynevent.c b/kernel/trace/trace_dynevent.c new file mode 100644 index 000000000000..dd1f43588d70 --- /dev/null +++ b/kernel/trace/trace_dynevent.c | |||
@@ -0,0 +1,217 @@ | |||
1 | // SPDX-License-Identifier: GPL-2.0 | ||
2 | /* | ||
3 | * Generic dynamic event control interface | ||
4 | * | ||
5 | * Copyright (C) 2018 Masami Hiramatsu <mhiramat@kernel.org> | ||
6 | */ | ||
7 | |||
8 | #include <linux/debugfs.h> | ||
9 | #include <linux/kernel.h> | ||
10 | #include <linux/list.h> | ||
11 | #include <linux/mm.h> | ||
12 | #include <linux/mutex.h> | ||
13 | #include <linux/tracefs.h> | ||
14 | |||
15 | #include "trace.h" | ||
16 | #include "trace_dynevent.h" | ||
17 | |||
18 | static DEFINE_MUTEX(dyn_event_ops_mutex); | ||
19 | static LIST_HEAD(dyn_event_ops_list); | ||
20 | |||
21 | int dyn_event_register(struct dyn_event_operations *ops) | ||
22 | { | ||
23 | if (!ops || !ops->create || !ops->show || !ops->is_busy || | ||
24 | !ops->free || !ops->match) | ||
25 | return -EINVAL; | ||
26 | |||
27 | INIT_LIST_HEAD(&ops->list); | ||
28 | mutex_lock(&dyn_event_ops_mutex); | ||
29 | list_add_tail(&ops->list, &dyn_event_ops_list); | ||
30 | mutex_unlock(&dyn_event_ops_mutex); | ||
31 | return 0; | ||
32 | } | ||
33 | |||
34 | int dyn_event_release(int argc, char **argv, struct dyn_event_operations *type) | ||
35 | { | ||
36 | struct dyn_event *pos, *n; | ||
37 | char *system = NULL, *event, *p; | ||
38 | int ret = -ENOENT; | ||
39 | |||
40 | if (argv[0][0] == '-') { | ||
41 | if (argv[0][1] != ':') | ||
42 | return -EINVAL; | ||
43 | event = &argv[0][2]; | ||
44 | } else { | ||
45 | event = strchr(argv[0], ':'); | ||
46 | if (!event) | ||
47 | return -EINVAL; | ||
48 | event++; | ||
49 | } | ||
50 | |||
51 | p = strchr(event, '/'); | ||
52 | if (p) { | ||
53 | system = event; | ||
54 | event = p + 1; | ||
55 | *p = '\0'; | ||
56 | } | ||
57 | if (event[0] == '\0') | ||
58 | return -EINVAL; | ||
59 | |||
60 | mutex_lock(&event_mutex); | ||
61 | for_each_dyn_event_safe(pos, n) { | ||
62 | if (type && type != pos->ops) | ||
63 | continue; | ||
64 | if (pos->ops->match(system, event, pos)) { | ||
65 | ret = pos->ops->free(pos); | ||
66 | break; | ||
67 | } | ||
68 | } | ||
69 | mutex_unlock(&event_mutex); | ||
70 | |||
71 | return ret; | ||
72 | } | ||
73 | |||
74 | static int create_dyn_event(int argc, char **argv) | ||
75 | { | ||
76 | struct dyn_event_operations *ops; | ||
77 | int ret; | ||
78 | |||
79 | if (argv[0][0] == '-' || argv[0][0] == '!') | ||
80 | return dyn_event_release(argc, argv, NULL); | ||
81 | |||
82 | mutex_lock(&dyn_event_ops_mutex); | ||
83 | list_for_each_entry(ops, &dyn_event_ops_list, list) { | ||
84 | ret = ops->create(argc, (const char **)argv); | ||
85 | if (!ret || ret != -ECANCELED) | ||
86 | break; | ||
87 | } | ||
88 | mutex_unlock(&dyn_event_ops_mutex); | ||
89 | if (ret == -ECANCELED) | ||
90 | ret = -EINVAL; | ||
91 | |||
92 | return ret; | ||
93 | } | ||
94 | |||
95 | /* Protected by event_mutex */ | ||
96 | LIST_HEAD(dyn_event_list); | ||
97 | |||
98 | void *dyn_event_seq_start(struct seq_file *m, loff_t *pos) | ||
99 | { | ||
100 | mutex_lock(&event_mutex); | ||
101 | return seq_list_start(&dyn_event_list, *pos); | ||
102 | } | ||
103 | |||
104 | void *dyn_event_seq_next(struct seq_file *m, void *v, loff_t *pos) | ||
105 | { | ||
106 | return seq_list_next(v, &dyn_event_list, pos); | ||
107 | } | ||
108 | |||
109 | void dyn_event_seq_stop(struct seq_file *m, void *v) | ||
110 | { | ||
111 | mutex_unlock(&event_mutex); | ||
112 | } | ||
113 | |||
114 | static int dyn_event_seq_show(struct seq_file *m, void *v) | ||
115 | { | ||
116 | struct dyn_event *ev = v; | ||
117 | |||
118 | if (ev && ev->ops) | ||
119 | return ev->ops->show(m, ev); | ||
120 | |||
121 | return 0; | ||
122 | } | ||
123 | |||
124 | static const struct seq_operations dyn_event_seq_op = { | ||
125 | .start = dyn_event_seq_start, | ||
126 | .next = dyn_event_seq_next, | ||
127 | .stop = dyn_event_seq_stop, | ||
128 | .show = dyn_event_seq_show | ||
129 | }; | ||
130 | |||
131 | /* | ||
132 | * dyn_events_release_all - Release all specific events | ||
133 | * @type: the dyn_event_operations * which filters releasing events | ||
134 | * | ||
135 | * This releases all events which ->ops matches @type. If @type is NULL, | ||
136 | * all events are released. | ||
137 | * Return -EBUSY if any of them are in use, and return other errors when | ||
138 | * it failed to free the given event. Except for -EBUSY, event releasing | ||
139 | * process will be aborted at that point and there may be some other | ||
140 | * releasable events on the list. | ||
141 | */ | ||
142 | int dyn_events_release_all(struct dyn_event_operations *type) | ||
143 | { | ||
144 | struct dyn_event *ev, *tmp; | ||
145 | int ret = 0; | ||
146 | |||
147 | mutex_lock(&event_mutex); | ||
148 | for_each_dyn_event(ev) { | ||
149 | if (type && ev->ops != type) | ||
150 | continue; | ||
151 | if (ev->ops->is_busy(ev)) { | ||
152 | ret = -EBUSY; | ||
153 | goto out; | ||
154 | } | ||
155 | } | ||
156 | for_each_dyn_event_safe(ev, tmp) { | ||
157 | if (type && ev->ops != type) | ||
158 | continue; | ||
159 | ret = ev->ops->free(ev); | ||
160 | if (ret) | ||
161 | break; | ||
162 | } | ||
163 | out: | ||
164 | mutex_unlock(&event_mutex); | ||
165 | |||
166 | return ret; | ||
167 | } | ||
168 | |||
169 | static int dyn_event_open(struct inode *inode, struct file *file) | ||
170 | { | ||
171 | int ret; | ||
172 | |||
173 | if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) { | ||
174 | ret = dyn_events_release_all(NULL); | ||
175 | if (ret < 0) | ||
176 | return ret; | ||
177 | } | ||
178 | |||
179 | return seq_open(file, &dyn_event_seq_op); | ||
180 | } | ||
181 | |||
182 | static ssize_t dyn_event_write(struct file *file, const char __user *buffer, | ||
183 | size_t count, loff_t *ppos) | ||
184 | { | ||
185 | return trace_parse_run_command(file, buffer, count, ppos, | ||
186 | create_dyn_event); | ||
187 | } | ||
188 | |||
189 | static const struct file_operations dynamic_events_ops = { | ||
190 | .owner = THIS_MODULE, | ||
191 | .open = dyn_event_open, | ||
192 | .read = seq_read, | ||
193 | .llseek = seq_lseek, | ||
194 | .release = seq_release, | ||
195 | .write = dyn_event_write, | ||
196 | }; | ||
197 | |||
198 | /* Make a tracefs interface for controlling dynamic events */ | ||
199 | static __init int init_dynamic_event(void) | ||
200 | { | ||
201 | struct dentry *d_tracer; | ||
202 | struct dentry *entry; | ||
203 | |||
204 | d_tracer = tracing_init_dentry(); | ||
205 | if (IS_ERR(d_tracer)) | ||
206 | return 0; | ||
207 | |||
208 | entry = tracefs_create_file("dynamic_events", 0644, d_tracer, | ||
209 | NULL, &dynamic_events_ops); | ||
210 | |||
211 | /* Event list interface */ | ||
212 | if (!entry) | ||
213 | pr_warn("Could not create tracefs 'dynamic_events' entry\n"); | ||
214 | |||
215 | return 0; | ||
216 | } | ||
217 | fs_initcall(init_dynamic_event); | ||
diff --git a/kernel/trace/trace_dynevent.h b/kernel/trace/trace_dynevent.h new file mode 100644 index 000000000000..8c334064e4d6 --- /dev/null +++ b/kernel/trace/trace_dynevent.h | |||
@@ -0,0 +1,119 @@ | |||
1 | /* SPDX-License-Identifier: GPL-2.0 */ | ||
2 | /* | ||
3 | * Common header file for generic dynamic events. | ||
4 | */ | ||
5 | |||
6 | #ifndef _TRACE_DYNEVENT_H | ||
7 | #define _TRACE_DYNEVENT_H | ||
8 | |||
9 | #include <linux/kernel.h> | ||
10 | #include <linux/list.h> | ||
11 | #include <linux/mutex.h> | ||
12 | #include <linux/seq_file.h> | ||
13 | |||
14 | #include "trace.h" | ||
15 | |||
16 | struct dyn_event; | ||
17 | |||
18 | /** | ||
19 | * struct dyn_event_operations - Methods for each type of dynamic events | ||
20 | * | ||
21 | * These methods must be set for each type, since there is no default method. | ||
22 | * Before using this for dyn_event_init(), it must be registered by | ||
23 | * dyn_event_register(). | ||
24 | * | ||
25 | * @create: Parse and create event method. This is invoked when user passes | ||
26 | * a event definition to dynamic_events interface. This must not destruct | ||
27 | * the arguments and return -ECANCELED if given arguments doesn't match its | ||
28 | * command prefix. | ||
29 | * @show: Showing method. This is invoked when user reads the event definitions | ||
30 | * via dynamic_events interface. | ||
31 | * @is_busy: Check whether given event is busy so that it can not be deleted. | ||
32 | * Return true if it is busy, otherwides false. | ||
33 | * @free: Delete the given event. Return 0 if success, otherwides error. | ||
34 | * @match: Check whether given event and system name match this event. | ||
35 | * Return true if it matches, otherwides false. | ||
36 | * | ||
37 | * Except for @create, these methods are called under holding event_mutex. | ||
38 | */ | ||
39 | struct dyn_event_operations { | ||
40 | struct list_head list; | ||
41 | int (*create)(int argc, const char *argv[]); | ||
42 | int (*show)(struct seq_file *m, struct dyn_event *ev); | ||
43 | bool (*is_busy)(struct dyn_event *ev); | ||
44 | int (*free)(struct dyn_event *ev); | ||
45 | bool (*match)(const char *system, const char *event, | ||
46 | struct dyn_event *ev); | ||
47 | }; | ||
48 | |||
49 | /* Register new dyn_event type -- must be called at first */ | ||
50 | int dyn_event_register(struct dyn_event_operations *ops); | ||
51 | |||
52 | /** | ||
53 | * struct dyn_event - Dynamic event list header | ||
54 | * | ||
55 | * The dyn_event structure encapsulates a list and a pointer to the operators | ||
56 | * for making a global list of dynamic events. | ||
57 | * User must includes this in each event structure, so that those events can | ||
58 | * be added/removed via dynamic_events interface. | ||
59 | */ | ||
60 | struct dyn_event { | ||
61 | struct list_head list; | ||
62 | struct dyn_event_operations *ops; | ||
63 | }; | ||
64 | |||
65 | extern struct list_head dyn_event_list; | ||
66 | |||
67 | static inline | ||
68 | int dyn_event_init(struct dyn_event *ev, struct dyn_event_operations *ops) | ||
69 | { | ||
70 | if (!ev || !ops) | ||
71 | return -EINVAL; | ||
72 | |||
73 | INIT_LIST_HEAD(&ev->list); | ||
74 | ev->ops = ops; | ||
75 | return 0; | ||
76 | } | ||
77 | |||
78 | static inline int dyn_event_add(struct dyn_event *ev) | ||
79 | { | ||
80 | lockdep_assert_held(&event_mutex); | ||
81 | |||
82 | if (!ev || !ev->ops) | ||
83 | return -EINVAL; | ||
84 | |||
85 | list_add_tail(&ev->list, &dyn_event_list); | ||
86 | return 0; | ||
87 | } | ||
88 | |||
89 | static inline void dyn_event_remove(struct dyn_event *ev) | ||
90 | { | ||
91 | lockdep_assert_held(&event_mutex); | ||
92 | list_del_init(&ev->list); | ||
93 | } | ||
94 | |||
95 | void *dyn_event_seq_start(struct seq_file *m, loff_t *pos); | ||
96 | void *dyn_event_seq_next(struct seq_file *m, void *v, loff_t *pos); | ||
97 | void dyn_event_seq_stop(struct seq_file *m, void *v); | ||
98 | int dyn_events_release_all(struct dyn_event_operations *type); | ||
99 | int dyn_event_release(int argc, char **argv, struct dyn_event_operations *type); | ||
100 | |||
101 | /* | ||
102 | * for_each_dyn_event - iterate over the dyn_event list | ||
103 | * @pos: the struct dyn_event * to use as a loop cursor | ||
104 | * | ||
105 | * This is just a basement of for_each macro. Wrap this for | ||
106 | * each actual event structure with ops filtering. | ||
107 | */ | ||
108 | #define for_each_dyn_event(pos) \ | ||
109 | list_for_each_entry(pos, &dyn_event_list, list) | ||
110 | |||
111 | /* | ||
112 | * for_each_dyn_event - iterate over the dyn_event list safely | ||
113 | * @pos: the struct dyn_event * to use as a loop cursor | ||
114 | * @n: the struct dyn_event * to use as temporary storage | ||
115 | */ | ||
116 | #define for_each_dyn_event_safe(pos, n) \ | ||
117 | list_for_each_entry_safe(pos, n, &dyn_event_list, list) | ||
118 | |||
119 | #endif | ||
diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c index f94be0c2827b..5b3b0c3c8a47 100644 --- a/kernel/trace/trace_events.c +++ b/kernel/trace/trace_events.c | |||
@@ -1251,7 +1251,7 @@ static int f_show(struct seq_file *m, void *v) | |||
1251 | */ | 1251 | */ |
1252 | array_descriptor = strchr(field->type, '['); | 1252 | array_descriptor = strchr(field->type, '['); |
1253 | 1253 | ||
1254 | if (!strncmp(field->type, "__data_loc", 10)) | 1254 | if (str_has_prefix(field->type, "__data_loc")) |
1255 | array_descriptor = NULL; | 1255 | array_descriptor = NULL; |
1256 | 1256 | ||
1257 | if (!array_descriptor) | 1257 | if (!array_descriptor) |
@@ -2309,7 +2309,8 @@ static void __add_event_to_tracers(struct trace_event_call *call); | |||
2309 | int trace_add_event_call(struct trace_event_call *call) | 2309 | int trace_add_event_call(struct trace_event_call *call) |
2310 | { | 2310 | { |
2311 | int ret; | 2311 | int ret; |
2312 | mutex_lock(&event_mutex); | 2312 | lockdep_assert_held(&event_mutex); |
2313 | |||
2313 | mutex_lock(&trace_types_lock); | 2314 | mutex_lock(&trace_types_lock); |
2314 | 2315 | ||
2315 | ret = __register_event(call, NULL); | 2316 | ret = __register_event(call, NULL); |
@@ -2317,7 +2318,6 @@ int trace_add_event_call(struct trace_event_call *call) | |||
2317 | __add_event_to_tracers(call); | 2318 | __add_event_to_tracers(call); |
2318 | 2319 | ||
2319 | mutex_unlock(&trace_types_lock); | 2320 | mutex_unlock(&trace_types_lock); |
2320 | mutex_unlock(&event_mutex); | ||
2321 | return ret; | 2321 | return ret; |
2322 | } | 2322 | } |
2323 | 2323 | ||
@@ -2371,13 +2371,13 @@ int trace_remove_event_call(struct trace_event_call *call) | |||
2371 | { | 2371 | { |
2372 | int ret; | 2372 | int ret; |
2373 | 2373 | ||
2374 | mutex_lock(&event_mutex); | 2374 | lockdep_assert_held(&event_mutex); |
2375 | |||
2375 | mutex_lock(&trace_types_lock); | 2376 | mutex_lock(&trace_types_lock); |
2376 | down_write(&trace_event_sem); | 2377 | down_write(&trace_event_sem); |
2377 | ret = probe_remove_event_call(call); | 2378 | ret = probe_remove_event_call(call); |
2378 | up_write(&trace_event_sem); | 2379 | up_write(&trace_event_sem); |
2379 | mutex_unlock(&trace_types_lock); | 2380 | mutex_unlock(&trace_types_lock); |
2380 | mutex_unlock(&event_mutex); | ||
2381 | 2381 | ||
2382 | return ret; | 2382 | return ret; |
2383 | } | 2383 | } |
diff --git a/kernel/trace/trace_events_hist.c b/kernel/trace/trace_events_hist.c index eb908ef2ecec..449d90cfa151 100644 --- a/kernel/trace/trace_events_hist.c +++ b/kernel/trace/trace_events_hist.c | |||
@@ -15,6 +15,7 @@ | |||
15 | 15 | ||
16 | #include "tracing_map.h" | 16 | #include "tracing_map.h" |
17 | #include "trace.h" | 17 | #include "trace.h" |
18 | #include "trace_dynevent.h" | ||
18 | 19 | ||
19 | #define SYNTH_SYSTEM "synthetic" | 20 | #define SYNTH_SYSTEM "synthetic" |
20 | #define SYNTH_FIELDS_MAX 16 | 21 | #define SYNTH_FIELDS_MAX 16 |
@@ -39,6 +40,16 @@ enum field_op_id { | |||
39 | FIELD_OP_UNARY_MINUS, | 40 | FIELD_OP_UNARY_MINUS, |
40 | }; | 41 | }; |
41 | 42 | ||
43 | /* | ||
44 | * A hist_var (histogram variable) contains variable information for | ||
45 | * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF | ||
46 | * flag set. A hist_var has a variable name e.g. ts0, and is | ||
47 | * associated with a given histogram trigger, as specified by | ||
48 | * hist_data. The hist_var idx is the unique index assigned to the | ||
49 | * variable by the hist trigger's tracing_map. The idx is what is | ||
50 | * used to set a variable's value and, by a variable reference, to | ||
51 | * retrieve it. | ||
52 | */ | ||
42 | struct hist_var { | 53 | struct hist_var { |
43 | char *name; | 54 | char *name; |
44 | struct hist_trigger_data *hist_data; | 55 | struct hist_trigger_data *hist_data; |
@@ -55,12 +66,29 @@ struct hist_field { | |||
55 | const char *type; | 66 | const char *type; |
56 | struct hist_field *operands[HIST_FIELD_OPERANDS_MAX]; | 67 | struct hist_field *operands[HIST_FIELD_OPERANDS_MAX]; |
57 | struct hist_trigger_data *hist_data; | 68 | struct hist_trigger_data *hist_data; |
69 | |||
70 | /* | ||
71 | * Variable fields contain variable-specific info in var. | ||
72 | */ | ||
58 | struct hist_var var; | 73 | struct hist_var var; |
59 | enum field_op_id operator; | 74 | enum field_op_id operator; |
60 | char *system; | 75 | char *system; |
61 | char *event_name; | 76 | char *event_name; |
77 | |||
78 | /* | ||
79 | * The name field is used for EXPR and VAR_REF fields. VAR | ||
80 | * fields contain the variable name in var.name. | ||
81 | */ | ||
62 | char *name; | 82 | char *name; |
63 | unsigned int var_idx; | 83 | |
84 | /* | ||
85 | * When a histogram trigger is hit, if it has any references | ||
86 | * to variables, the values of those variables are collected | ||
87 | * into a var_ref_vals array by resolve_var_refs(). The | ||
88 | * current value of each variable is read from the tracing_map | ||
89 | * using the hist field's hist_var.idx and entered into the | ||
90 | * var_ref_idx entry i.e. var_ref_vals[var_ref_idx]. | ||
91 | */ | ||
64 | unsigned int var_ref_idx; | 92 | unsigned int var_ref_idx; |
65 | bool read_once; | 93 | bool read_once; |
66 | }; | 94 | }; |
@@ -279,8 +307,6 @@ struct hist_trigger_data { | |||
279 | struct action_data *actions[HIST_ACTIONS_MAX]; | 307 | struct action_data *actions[HIST_ACTIONS_MAX]; |
280 | unsigned int n_actions; | 308 | unsigned int n_actions; |
281 | 309 | ||
282 | struct hist_field *synth_var_refs[SYNTH_FIELDS_MAX]; | ||
283 | unsigned int n_synth_var_refs; | ||
284 | struct field_var *field_vars[SYNTH_FIELDS_MAX]; | 310 | struct field_var *field_vars[SYNTH_FIELDS_MAX]; |
285 | unsigned int n_field_vars; | 311 | unsigned int n_field_vars; |
286 | unsigned int n_field_var_str; | 312 | unsigned int n_field_var_str; |
@@ -292,6 +318,21 @@ struct hist_trigger_data { | |||
292 | unsigned int n_max_var_str; | 318 | unsigned int n_max_var_str; |
293 | }; | 319 | }; |
294 | 320 | ||
321 | static int synth_event_create(int argc, const char **argv); | ||
322 | static int synth_event_show(struct seq_file *m, struct dyn_event *ev); | ||
323 | static int synth_event_release(struct dyn_event *ev); | ||
324 | static bool synth_event_is_busy(struct dyn_event *ev); | ||
325 | static bool synth_event_match(const char *system, const char *event, | ||
326 | struct dyn_event *ev); | ||
327 | |||
328 | static struct dyn_event_operations synth_event_ops = { | ||
329 | .create = synth_event_create, | ||
330 | .show = synth_event_show, | ||
331 | .is_busy = synth_event_is_busy, | ||
332 | .free = synth_event_release, | ||
333 | .match = synth_event_match, | ||
334 | }; | ||
335 | |||
295 | struct synth_field { | 336 | struct synth_field { |
296 | char *type; | 337 | char *type; |
297 | char *name; | 338 | char *name; |
@@ -301,7 +342,7 @@ struct synth_field { | |||
301 | }; | 342 | }; |
302 | 343 | ||
303 | struct synth_event { | 344 | struct synth_event { |
304 | struct list_head list; | 345 | struct dyn_event devent; |
305 | int ref; | 346 | int ref; |
306 | char *name; | 347 | char *name; |
307 | struct synth_field **fields; | 348 | struct synth_field **fields; |
@@ -312,6 +353,32 @@ struct synth_event { | |||
312 | struct tracepoint *tp; | 353 | struct tracepoint *tp; |
313 | }; | 354 | }; |
314 | 355 | ||
356 | static bool is_synth_event(struct dyn_event *ev) | ||
357 | { | ||
358 | return ev->ops == &synth_event_ops; | ||
359 | } | ||
360 | |||
361 | static struct synth_event *to_synth_event(struct dyn_event *ev) | ||
362 | { | ||
363 | return container_of(ev, struct synth_event, devent); | ||
364 | } | ||
365 | |||
366 | static bool synth_event_is_busy(struct dyn_event *ev) | ||
367 | { | ||
368 | struct synth_event *event = to_synth_event(ev); | ||
369 | |||
370 | return event->ref != 0; | ||
371 | } | ||
372 | |||
373 | static bool synth_event_match(const char *system, const char *event, | ||
374 | struct dyn_event *ev) | ||
375 | { | ||
376 | struct synth_event *sev = to_synth_event(ev); | ||
377 | |||
378 | return strcmp(sev->name, event) == 0 && | ||
379 | (!system || strcmp(system, SYNTH_SYSTEM) == 0); | ||
380 | } | ||
381 | |||
315 | struct action_data; | 382 | struct action_data; |
316 | 383 | ||
317 | typedef void (*action_fn_t) (struct hist_trigger_data *hist_data, | 384 | typedef void (*action_fn_t) (struct hist_trigger_data *hist_data, |
@@ -326,6 +393,14 @@ struct action_data { | |||
326 | 393 | ||
327 | union { | 394 | union { |
328 | struct { | 395 | struct { |
396 | /* | ||
397 | * When a histogram trigger is hit, the values of any | ||
398 | * references to variables, including variables being passed | ||
399 | * as parameters to synthetic events, are collected into a | ||
400 | * var_ref_vals array. This var_ref_idx is the index of the | ||
401 | * first param in the array to be passed to the synthetic | ||
402 | * event invocation. | ||
403 | */ | ||
329 | unsigned int var_ref_idx; | 404 | unsigned int var_ref_idx; |
330 | char *match_event; | 405 | char *match_event; |
331 | char *match_event_system; | 406 | char *match_event_system; |
@@ -402,9 +477,6 @@ static bool have_hist_err(void) | |||
402 | return false; | 477 | return false; |
403 | } | 478 | } |
404 | 479 | ||
405 | static LIST_HEAD(synth_event_list); | ||
406 | static DEFINE_MUTEX(synth_event_mutex); | ||
407 | |||
408 | struct synth_trace_event { | 480 | struct synth_trace_event { |
409 | struct trace_entry ent; | 481 | struct trace_entry ent; |
410 | u64 fields[]; | 482 | u64 fields[]; |
@@ -446,7 +518,7 @@ static int synth_event_define_fields(struct trace_event_call *call) | |||
446 | 518 | ||
447 | static bool synth_field_signed(char *type) | 519 | static bool synth_field_signed(char *type) |
448 | { | 520 | { |
449 | if (strncmp(type, "u", 1) == 0) | 521 | if (str_has_prefix(type, "u")) |
450 | return false; | 522 | return false; |
451 | 523 | ||
452 | return true; | 524 | return true; |
@@ -469,7 +541,7 @@ static int synth_field_string_size(char *type) | |||
469 | start = strstr(type, "char["); | 541 | start = strstr(type, "char["); |
470 | if (start == NULL) | 542 | if (start == NULL) |
471 | return -EINVAL; | 543 | return -EINVAL; |
472 | start += strlen("char["); | 544 | start += sizeof("char[") - 1; |
473 | 545 | ||
474 | end = strchr(type, ']'); | 546 | end = strchr(type, ']'); |
475 | if (!end || end < start) | 547 | if (!end || end < start) |
@@ -738,14 +810,12 @@ static void free_synth_field(struct synth_field *field) | |||
738 | kfree(field); | 810 | kfree(field); |
739 | } | 811 | } |
740 | 812 | ||
741 | static struct synth_field *parse_synth_field(int argc, char **argv, | 813 | static struct synth_field *parse_synth_field(int argc, const char **argv, |
742 | int *consumed) | 814 | int *consumed) |
743 | { | 815 | { |
744 | struct synth_field *field; | 816 | struct synth_field *field; |
745 | const char *prefix = NULL; | 817 | const char *prefix = NULL, *field_type = argv[0], *field_name, *array; |
746 | char *field_type = argv[0], *field_name; | ||
747 | int len, ret = 0; | 818 | int len, ret = 0; |
748 | char *array; | ||
749 | 819 | ||
750 | if (field_type[0] == ';') | 820 | if (field_type[0] == ';') |
751 | field_type++; | 821 | field_type++; |
@@ -762,20 +832,31 @@ static struct synth_field *parse_synth_field(int argc, char **argv, | |||
762 | *consumed = 2; | 832 | *consumed = 2; |
763 | } | 833 | } |
764 | 834 | ||
765 | len = strlen(field_name); | ||
766 | if (field_name[len - 1] == ';') | ||
767 | field_name[len - 1] = '\0'; | ||
768 | |||
769 | field = kzalloc(sizeof(*field), GFP_KERNEL); | 835 | field = kzalloc(sizeof(*field), GFP_KERNEL); |
770 | if (!field) | 836 | if (!field) |
771 | return ERR_PTR(-ENOMEM); | 837 | return ERR_PTR(-ENOMEM); |
772 | 838 | ||
773 | len = strlen(field_type) + 1; | 839 | len = strlen(field_name); |
774 | array = strchr(field_name, '['); | 840 | array = strchr(field_name, '['); |
775 | if (array) | 841 | if (array) |
842 | len -= strlen(array); | ||
843 | else if (field_name[len - 1] == ';') | ||
844 | len--; | ||
845 | |||
846 | field->name = kmemdup_nul(field_name, len, GFP_KERNEL); | ||
847 | if (!field->name) { | ||
848 | ret = -ENOMEM; | ||
849 | goto free; | ||
850 | } | ||
851 | |||
852 | if (field_type[0] == ';') | ||
853 | field_type++; | ||
854 | len = strlen(field_type) + 1; | ||
855 | if (array) | ||
776 | len += strlen(array); | 856 | len += strlen(array); |
777 | if (prefix) | 857 | if (prefix) |
778 | len += strlen(prefix); | 858 | len += strlen(prefix); |
859 | |||
779 | field->type = kzalloc(len, GFP_KERNEL); | 860 | field->type = kzalloc(len, GFP_KERNEL); |
780 | if (!field->type) { | 861 | if (!field->type) { |
781 | ret = -ENOMEM; | 862 | ret = -ENOMEM; |
@@ -786,7 +867,8 @@ static struct synth_field *parse_synth_field(int argc, char **argv, | |||
786 | strcat(field->type, field_type); | 867 | strcat(field->type, field_type); |
787 | if (array) { | 868 | if (array) { |
788 | strcat(field->type, array); | 869 | strcat(field->type, array); |
789 | *array = '\0'; | 870 | if (field->type[len - 1] == ';') |
871 | field->type[len - 1] = '\0'; | ||
790 | } | 872 | } |
791 | 873 | ||
792 | field->size = synth_field_size(field->type); | 874 | field->size = synth_field_size(field->type); |
@@ -800,11 +882,6 @@ static struct synth_field *parse_synth_field(int argc, char **argv, | |||
800 | 882 | ||
801 | field->is_signed = synth_field_signed(field->type); | 883 | field->is_signed = synth_field_signed(field->type); |
802 | 884 | ||
803 | field->name = kstrdup(field_name, GFP_KERNEL); | ||
804 | if (!field->name) { | ||
805 | ret = -ENOMEM; | ||
806 | goto free; | ||
807 | } | ||
808 | out: | 885 | out: |
809 | return field; | 886 | return field; |
810 | free: | 887 | free: |
@@ -868,9 +945,13 @@ static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals, | |||
868 | 945 | ||
869 | static struct synth_event *find_synth_event(const char *name) | 946 | static struct synth_event *find_synth_event(const char *name) |
870 | { | 947 | { |
948 | struct dyn_event *pos; | ||
871 | struct synth_event *event; | 949 | struct synth_event *event; |
872 | 950 | ||
873 | list_for_each_entry(event, &synth_event_list, list) { | 951 | for_each_dyn_event(pos) { |
952 | if (!is_synth_event(pos)) | ||
953 | continue; | ||
954 | event = to_synth_event(pos); | ||
874 | if (strcmp(event->name, name) == 0) | 955 | if (strcmp(event->name, name) == 0) |
875 | return event; | 956 | return event; |
876 | } | 957 | } |
@@ -959,7 +1040,7 @@ static void free_synth_event(struct synth_event *event) | |||
959 | kfree(event); | 1040 | kfree(event); |
960 | } | 1041 | } |
961 | 1042 | ||
962 | static struct synth_event *alloc_synth_event(char *event_name, int n_fields, | 1043 | static struct synth_event *alloc_synth_event(const char *name, int n_fields, |
963 | struct synth_field **fields) | 1044 | struct synth_field **fields) |
964 | { | 1045 | { |
965 | struct synth_event *event; | 1046 | struct synth_event *event; |
@@ -971,7 +1052,7 @@ static struct synth_event *alloc_synth_event(char *event_name, int n_fields, | |||
971 | goto out; | 1052 | goto out; |
972 | } | 1053 | } |
973 | 1054 | ||
974 | event->name = kstrdup(event_name, GFP_KERNEL); | 1055 | event->name = kstrdup(name, GFP_KERNEL); |
975 | if (!event->name) { | 1056 | if (!event->name) { |
976 | kfree(event); | 1057 | kfree(event); |
977 | event = ERR_PTR(-ENOMEM); | 1058 | event = ERR_PTR(-ENOMEM); |
@@ -985,6 +1066,8 @@ static struct synth_event *alloc_synth_event(char *event_name, int n_fields, | |||
985 | goto out; | 1066 | goto out; |
986 | } | 1067 | } |
987 | 1068 | ||
1069 | dyn_event_init(&event->devent, &synth_event_ops); | ||
1070 | |||
988 | for (i = 0; i < n_fields; i++) | 1071 | for (i = 0; i < n_fields; i++) |
989 | event->fields[i] = fields[i]; | 1072 | event->fields[i] = fields[i]; |
990 | 1073 | ||
@@ -1008,29 +1091,11 @@ struct hist_var_data { | |||
1008 | struct hist_trigger_data *hist_data; | 1091 | struct hist_trigger_data *hist_data; |
1009 | }; | 1092 | }; |
1010 | 1093 | ||
1011 | static void add_or_delete_synth_event(struct synth_event *event, int delete) | 1094 | static int __create_synth_event(int argc, const char *name, const char **argv) |
1012 | { | ||
1013 | if (delete) | ||
1014 | free_synth_event(event); | ||
1015 | else { | ||
1016 | mutex_lock(&synth_event_mutex); | ||
1017 | if (!find_synth_event(event->name)) | ||
1018 | list_add(&event->list, &synth_event_list); | ||
1019 | else | ||
1020 | free_synth_event(event); | ||
1021 | mutex_unlock(&synth_event_mutex); | ||
1022 | } | ||
1023 | } | ||
1024 | |||
1025 | static int create_synth_event(int argc, char **argv) | ||
1026 | { | 1095 | { |
1027 | struct synth_field *field, *fields[SYNTH_FIELDS_MAX]; | 1096 | struct synth_field *field, *fields[SYNTH_FIELDS_MAX]; |
1028 | struct synth_event *event = NULL; | 1097 | struct synth_event *event = NULL; |
1029 | bool delete_event = false; | ||
1030 | int i, consumed = 0, n_fields = 0, ret = 0; | 1098 | int i, consumed = 0, n_fields = 0, ret = 0; |
1031 | char *name; | ||
1032 | |||
1033 | mutex_lock(&synth_event_mutex); | ||
1034 | 1099 | ||
1035 | /* | 1100 | /* |
1036 | * Argument syntax: | 1101 | * Argument syntax: |
@@ -1038,42 +1103,19 @@ static int create_synth_event(int argc, char **argv) | |||
1038 | * - Remove synthetic event: !<event_name> field[;field] ... | 1103 | * - Remove synthetic event: !<event_name> field[;field] ... |
1039 | * where 'field' = type field_name | 1104 | * where 'field' = type field_name |
1040 | */ | 1105 | */ |
1041 | if (argc < 1) { | ||
1042 | ret = -EINVAL; | ||
1043 | goto out; | ||
1044 | } | ||
1045 | 1106 | ||
1046 | name = argv[0]; | 1107 | if (name[0] == '\0' || argc < 1) |
1047 | if (name[0] == '!') { | 1108 | return -EINVAL; |
1048 | delete_event = true; | 1109 | |
1049 | name++; | 1110 | mutex_lock(&event_mutex); |
1050 | } | ||
1051 | 1111 | ||
1052 | event = find_synth_event(name); | 1112 | event = find_synth_event(name); |
1053 | if (event) { | 1113 | if (event) { |
1054 | if (delete_event) { | ||
1055 | if (event->ref) { | ||
1056 | event = NULL; | ||
1057 | ret = -EBUSY; | ||
1058 | goto out; | ||
1059 | } | ||
1060 | list_del(&event->list); | ||
1061 | goto out; | ||
1062 | } | ||
1063 | event = NULL; | ||
1064 | ret = -EEXIST; | 1114 | ret = -EEXIST; |
1065 | goto out; | 1115 | goto out; |
1066 | } else if (delete_event) { | ||
1067 | ret = -ENOENT; | ||
1068 | goto out; | ||
1069 | } | ||
1070 | |||
1071 | if (argc < 2) { | ||
1072 | ret = -EINVAL; | ||
1073 | goto out; | ||
1074 | } | 1116 | } |
1075 | 1117 | ||
1076 | for (i = 1; i < argc - 1; i++) { | 1118 | for (i = 0; i < argc - 1; i++) { |
1077 | if (strcmp(argv[i], ";") == 0) | 1119 | if (strcmp(argv[i], ";") == 0) |
1078 | continue; | 1120 | continue; |
1079 | if (n_fields == SYNTH_FIELDS_MAX) { | 1121 | if (n_fields == SYNTH_FIELDS_MAX) { |
@@ -1101,83 +1143,91 @@ static int create_synth_event(int argc, char **argv) | |||
1101 | event = NULL; | 1143 | event = NULL; |
1102 | goto err; | 1144 | goto err; |
1103 | } | 1145 | } |
1146 | ret = register_synth_event(event); | ||
1147 | if (!ret) | ||
1148 | dyn_event_add(&event->devent); | ||
1149 | else | ||
1150 | free_synth_event(event); | ||
1104 | out: | 1151 | out: |
1105 | mutex_unlock(&synth_event_mutex); | 1152 | mutex_unlock(&event_mutex); |
1106 | |||
1107 | if (event) { | ||
1108 | if (delete_event) { | ||
1109 | ret = unregister_synth_event(event); | ||
1110 | add_or_delete_synth_event(event, !ret); | ||
1111 | } else { | ||
1112 | ret = register_synth_event(event); | ||
1113 | add_or_delete_synth_event(event, ret); | ||
1114 | } | ||
1115 | } | ||
1116 | 1153 | ||
1117 | return ret; | 1154 | return ret; |
1118 | err: | 1155 | err: |
1119 | mutex_unlock(&synth_event_mutex); | ||
1120 | |||
1121 | for (i = 0; i < n_fields; i++) | 1156 | for (i = 0; i < n_fields; i++) |
1122 | free_synth_field(fields[i]); | 1157 | free_synth_field(fields[i]); |
1123 | free_synth_event(event); | ||
1124 | 1158 | ||
1125 | return ret; | 1159 | goto out; |
1126 | } | 1160 | } |
1127 | 1161 | ||
1128 | static int release_all_synth_events(void) | 1162 | static int create_or_delete_synth_event(int argc, char **argv) |
1129 | { | 1163 | { |
1130 | struct list_head release_events; | 1164 | const char *name = argv[0]; |
1131 | struct synth_event *event, *e; | 1165 | struct synth_event *event = NULL; |
1132 | int ret = 0; | 1166 | int ret; |
1133 | |||
1134 | INIT_LIST_HEAD(&release_events); | ||
1135 | |||
1136 | mutex_lock(&synth_event_mutex); | ||
1137 | 1167 | ||
1138 | list_for_each_entry(event, &synth_event_list, list) { | 1168 | /* trace_run_command() ensures argc != 0 */ |
1139 | if (event->ref) { | 1169 | if (name[0] == '!') { |
1140 | mutex_unlock(&synth_event_mutex); | 1170 | mutex_lock(&event_mutex); |
1141 | return -EBUSY; | 1171 | event = find_synth_event(name + 1); |
1142 | } | 1172 | if (event) { |
1173 | if (event->ref) | ||
1174 | ret = -EBUSY; | ||
1175 | else { | ||
1176 | ret = unregister_synth_event(event); | ||
1177 | if (!ret) { | ||
1178 | dyn_event_remove(&event->devent); | ||
1179 | free_synth_event(event); | ||
1180 | } | ||
1181 | } | ||
1182 | } else | ||
1183 | ret = -ENOENT; | ||
1184 | mutex_unlock(&event_mutex); | ||
1185 | return ret; | ||
1143 | } | 1186 | } |
1144 | 1187 | ||
1145 | list_splice_init(&event->list, &release_events); | 1188 | ret = __create_synth_event(argc - 1, name, (const char **)argv + 1); |
1189 | return ret == -ECANCELED ? -EINVAL : ret; | ||
1190 | } | ||
1146 | 1191 | ||
1147 | mutex_unlock(&synth_event_mutex); | 1192 | static int synth_event_create(int argc, const char **argv) |
1193 | { | ||
1194 | const char *name = argv[0]; | ||
1195 | int len; | ||
1148 | 1196 | ||
1149 | list_for_each_entry_safe(event, e, &release_events, list) { | 1197 | if (name[0] != 's' || name[1] != ':') |
1150 | list_del(&event->list); | 1198 | return -ECANCELED; |
1199 | name += 2; | ||
1151 | 1200 | ||
1152 | ret = unregister_synth_event(event); | 1201 | /* This interface accepts group name prefix */ |
1153 | add_or_delete_synth_event(event, !ret); | 1202 | if (strchr(name, '/')) { |
1203 | len = sizeof(SYNTH_SYSTEM "/") - 1; | ||
1204 | if (strncmp(name, SYNTH_SYSTEM "/", len)) | ||
1205 | return -EINVAL; | ||
1206 | name += len; | ||
1154 | } | 1207 | } |
1155 | 1208 | return __create_synth_event(argc - 1, name, argv + 1); | |
1156 | return ret; | ||
1157 | } | 1209 | } |
1158 | 1210 | ||
1159 | 1211 | static int synth_event_release(struct dyn_event *ev) | |
1160 | static void *synth_events_seq_start(struct seq_file *m, loff_t *pos) | ||
1161 | { | 1212 | { |
1162 | mutex_lock(&synth_event_mutex); | 1213 | struct synth_event *event = to_synth_event(ev); |
1214 | int ret; | ||
1163 | 1215 | ||
1164 | return seq_list_start(&synth_event_list, *pos); | 1216 | if (event->ref) |
1165 | } | 1217 | return -EBUSY; |
1166 | 1218 | ||
1167 | static void *synth_events_seq_next(struct seq_file *m, void *v, loff_t *pos) | 1219 | ret = unregister_synth_event(event); |
1168 | { | 1220 | if (ret) |
1169 | return seq_list_next(v, &synth_event_list, pos); | 1221 | return ret; |
1170 | } | ||
1171 | 1222 | ||
1172 | static void synth_events_seq_stop(struct seq_file *m, void *v) | 1223 | dyn_event_remove(ev); |
1173 | { | 1224 | free_synth_event(event); |
1174 | mutex_unlock(&synth_event_mutex); | 1225 | return 0; |
1175 | } | 1226 | } |
1176 | 1227 | ||
1177 | static int synth_events_seq_show(struct seq_file *m, void *v) | 1228 | static int __synth_event_show(struct seq_file *m, struct synth_event *event) |
1178 | { | 1229 | { |
1179 | struct synth_field *field; | 1230 | struct synth_field *field; |
1180 | struct synth_event *event = v; | ||
1181 | unsigned int i; | 1231 | unsigned int i; |
1182 | 1232 | ||
1183 | seq_printf(m, "%s\t", event->name); | 1233 | seq_printf(m, "%s\t", event->name); |
@@ -1195,11 +1245,30 @@ static int synth_events_seq_show(struct seq_file *m, void *v) | |||
1195 | return 0; | 1245 | return 0; |
1196 | } | 1246 | } |
1197 | 1247 | ||
1248 | static int synth_event_show(struct seq_file *m, struct dyn_event *ev) | ||
1249 | { | ||
1250 | struct synth_event *event = to_synth_event(ev); | ||
1251 | |||
1252 | seq_printf(m, "s:%s/", event->class.system); | ||
1253 | |||
1254 | return __synth_event_show(m, event); | ||
1255 | } | ||
1256 | |||
1257 | static int synth_events_seq_show(struct seq_file *m, void *v) | ||
1258 | { | ||
1259 | struct dyn_event *ev = v; | ||
1260 | |||
1261 | if (!is_synth_event(ev)) | ||
1262 | return 0; | ||
1263 | |||
1264 | return __synth_event_show(m, to_synth_event(ev)); | ||
1265 | } | ||
1266 | |||
1198 | static const struct seq_operations synth_events_seq_op = { | 1267 | static const struct seq_operations synth_events_seq_op = { |
1199 | .start = synth_events_seq_start, | 1268 | .start = dyn_event_seq_start, |
1200 | .next = synth_events_seq_next, | 1269 | .next = dyn_event_seq_next, |
1201 | .stop = synth_events_seq_stop, | 1270 | .stop = dyn_event_seq_stop, |
1202 | .show = synth_events_seq_show | 1271 | .show = synth_events_seq_show, |
1203 | }; | 1272 | }; |
1204 | 1273 | ||
1205 | static int synth_events_open(struct inode *inode, struct file *file) | 1274 | static int synth_events_open(struct inode *inode, struct file *file) |
@@ -1207,7 +1276,7 @@ static int synth_events_open(struct inode *inode, struct file *file) | |||
1207 | int ret; | 1276 | int ret; |
1208 | 1277 | ||
1209 | if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) { | 1278 | if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) { |
1210 | ret = release_all_synth_events(); | 1279 | ret = dyn_events_release_all(&synth_event_ops); |
1211 | if (ret < 0) | 1280 | if (ret < 0) |
1212 | return ret; | 1281 | return ret; |
1213 | } | 1282 | } |
@@ -1220,7 +1289,7 @@ static ssize_t synth_events_write(struct file *file, | |||
1220 | size_t count, loff_t *ppos) | 1289 | size_t count, loff_t *ppos) |
1221 | { | 1290 | { |
1222 | return trace_parse_run_command(file, buffer, count, ppos, | 1291 | return trace_parse_run_command(file, buffer, count, ppos, |
1223 | create_synth_event); | 1292 | create_or_delete_synth_event); |
1224 | } | 1293 | } |
1225 | 1294 | ||
1226 | static const struct file_operations synth_events_fops = { | 1295 | static const struct file_operations synth_events_fops = { |
@@ -1257,82 +1326,73 @@ static u64 hist_field_cpu(struct hist_field *hist_field, | |||
1257 | return cpu; | 1326 | return cpu; |
1258 | } | 1327 | } |
1259 | 1328 | ||
1329 | /** | ||
1330 | * check_field_for_var_ref - Check if a VAR_REF field references a variable | ||
1331 | * @hist_field: The VAR_REF field to check | ||
1332 | * @var_data: The hist trigger that owns the variable | ||
1333 | * @var_idx: The trigger variable identifier | ||
1334 | * | ||
1335 | * Check the given VAR_REF field to see whether or not it references | ||
1336 | * the given variable associated with the given trigger. | ||
1337 | * | ||
1338 | * Return: The VAR_REF field if it does reference the variable, NULL if not | ||
1339 | */ | ||
1260 | static struct hist_field * | 1340 | static struct hist_field * |
1261 | check_field_for_var_ref(struct hist_field *hist_field, | 1341 | check_field_for_var_ref(struct hist_field *hist_field, |
1262 | struct hist_trigger_data *var_data, | 1342 | struct hist_trigger_data *var_data, |
1263 | unsigned int var_idx) | 1343 | unsigned int var_idx) |
1264 | { | 1344 | { |
1265 | struct hist_field *found = NULL; | 1345 | WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF)); |
1266 | |||
1267 | if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF) { | ||
1268 | if (hist_field->var.idx == var_idx && | ||
1269 | hist_field->var.hist_data == var_data) { | ||
1270 | found = hist_field; | ||
1271 | } | ||
1272 | } | ||
1273 | 1346 | ||
1274 | return found; | 1347 | if (hist_field && hist_field->var.idx == var_idx && |
1275 | } | 1348 | hist_field->var.hist_data == var_data) |
1276 | 1349 | return hist_field; | |
1277 | static struct hist_field * | ||
1278 | check_field_for_var_refs(struct hist_trigger_data *hist_data, | ||
1279 | struct hist_field *hist_field, | ||
1280 | struct hist_trigger_data *var_data, | ||
1281 | unsigned int var_idx, | ||
1282 | unsigned int level) | ||
1283 | { | ||
1284 | struct hist_field *found = NULL; | ||
1285 | unsigned int i; | ||
1286 | |||
1287 | if (level > 3) | ||
1288 | return found; | ||
1289 | |||
1290 | if (!hist_field) | ||
1291 | return found; | ||
1292 | |||
1293 | found = check_field_for_var_ref(hist_field, var_data, var_idx); | ||
1294 | if (found) | ||
1295 | return found; | ||
1296 | |||
1297 | for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) { | ||
1298 | struct hist_field *operand; | ||
1299 | |||
1300 | operand = hist_field->operands[i]; | ||
1301 | found = check_field_for_var_refs(hist_data, operand, var_data, | ||
1302 | var_idx, level + 1); | ||
1303 | if (found) | ||
1304 | return found; | ||
1305 | } | ||
1306 | 1350 | ||
1307 | return found; | 1351 | return NULL; |
1308 | } | 1352 | } |
1309 | 1353 | ||
1354 | /** | ||
1355 | * find_var_ref - Check if a trigger has a reference to a trigger variable | ||
1356 | * @hist_data: The hist trigger that might have a reference to the variable | ||
1357 | * @var_data: The hist trigger that owns the variable | ||
1358 | * @var_idx: The trigger variable identifier | ||
1359 | * | ||
1360 | * Check the list of var_refs[] on the first hist trigger to see | ||
1361 | * whether any of them are references to the variable on the second | ||
1362 | * trigger. | ||
1363 | * | ||
1364 | * Return: The VAR_REF field referencing the variable if so, NULL if not | ||
1365 | */ | ||
1310 | static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data, | 1366 | static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data, |
1311 | struct hist_trigger_data *var_data, | 1367 | struct hist_trigger_data *var_data, |
1312 | unsigned int var_idx) | 1368 | unsigned int var_idx) |
1313 | { | 1369 | { |
1314 | struct hist_field *hist_field, *found = NULL; | 1370 | struct hist_field *hist_field; |
1315 | unsigned int i; | 1371 | unsigned int i; |
1316 | 1372 | ||
1317 | for_each_hist_field(i, hist_data) { | 1373 | for (i = 0; i < hist_data->n_var_refs; i++) { |
1318 | hist_field = hist_data->fields[i]; | 1374 | hist_field = hist_data->var_refs[i]; |
1319 | found = check_field_for_var_refs(hist_data, hist_field, | 1375 | if (check_field_for_var_ref(hist_field, var_data, var_idx)) |
1320 | var_data, var_idx, 0); | 1376 | return hist_field; |
1321 | if (found) | ||
1322 | return found; | ||
1323 | } | ||
1324 | |||
1325 | for (i = 0; i < hist_data->n_synth_var_refs; i++) { | ||
1326 | hist_field = hist_data->synth_var_refs[i]; | ||
1327 | found = check_field_for_var_refs(hist_data, hist_field, | ||
1328 | var_data, var_idx, 0); | ||
1329 | if (found) | ||
1330 | return found; | ||
1331 | } | 1377 | } |
1332 | 1378 | ||
1333 | return found; | 1379 | return NULL; |
1334 | } | 1380 | } |
1335 | 1381 | ||
1382 | /** | ||
1383 | * find_any_var_ref - Check if there is a reference to a given trigger variable | ||
1384 | * @hist_data: The hist trigger | ||
1385 | * @var_idx: The trigger variable identifier | ||
1386 | * | ||
1387 | * Check to see whether the given variable is currently referenced by | ||
1388 | * any other trigger. | ||
1389 | * | ||
1390 | * The trigger the variable is defined on is explicitly excluded - the | ||
1391 | * assumption being that a self-reference doesn't prevent a trigger | ||
1392 | * from being removed. | ||
1393 | * | ||
1394 | * Return: The VAR_REF field referencing the variable if so, NULL if not | ||
1395 | */ | ||
1336 | static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data, | 1396 | static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data, |
1337 | unsigned int var_idx) | 1397 | unsigned int var_idx) |
1338 | { | 1398 | { |
@@ -1351,6 +1411,19 @@ static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data, | |||
1351 | return found; | 1411 | return found; |
1352 | } | 1412 | } |
1353 | 1413 | ||
1414 | /** | ||
1415 | * check_var_refs - Check if there is a reference to any of trigger's variables | ||
1416 | * @hist_data: The hist trigger | ||
1417 | * | ||
1418 | * A trigger can define one or more variables. If any one of them is | ||
1419 | * currently referenced by any other trigger, this function will | ||
1420 | * determine that. | ||
1421 | |||
1422 | * Typically used to determine whether or not a trigger can be removed | ||
1423 | * - if there are any references to a trigger's variables, it cannot. | ||
1424 | * | ||
1425 | * Return: True if there is a reference to any of trigger's variables | ||
1426 | */ | ||
1354 | static bool check_var_refs(struct hist_trigger_data *hist_data) | 1427 | static bool check_var_refs(struct hist_trigger_data *hist_data) |
1355 | { | 1428 | { |
1356 | struct hist_field *field; | 1429 | struct hist_field *field; |
@@ -1808,8 +1881,8 @@ static int parse_action(char *str, struct hist_trigger_attrs *attrs) | |||
1808 | if (attrs->n_actions >= HIST_ACTIONS_MAX) | 1881 | if (attrs->n_actions >= HIST_ACTIONS_MAX) |
1809 | return ret; | 1882 | return ret; |
1810 | 1883 | ||
1811 | if ((strncmp(str, "onmatch(", strlen("onmatch(")) == 0) || | 1884 | if ((str_has_prefix(str, "onmatch(")) || |
1812 | (strncmp(str, "onmax(", strlen("onmax(")) == 0)) { | 1885 | (str_has_prefix(str, "onmax("))) { |
1813 | attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL); | 1886 | attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL); |
1814 | if (!attrs->action_str[attrs->n_actions]) { | 1887 | if (!attrs->action_str[attrs->n_actions]) { |
1815 | ret = -ENOMEM; | 1888 | ret = -ENOMEM; |
@@ -1826,34 +1899,34 @@ static int parse_assignment(char *str, struct hist_trigger_attrs *attrs) | |||
1826 | { | 1899 | { |
1827 | int ret = 0; | 1900 | int ret = 0; |
1828 | 1901 | ||
1829 | if ((strncmp(str, "key=", strlen("key=")) == 0) || | 1902 | if ((str_has_prefix(str, "key=")) || |
1830 | (strncmp(str, "keys=", strlen("keys=")) == 0)) { | 1903 | (str_has_prefix(str, "keys="))) { |
1831 | attrs->keys_str = kstrdup(str, GFP_KERNEL); | 1904 | attrs->keys_str = kstrdup(str, GFP_KERNEL); |
1832 | if (!attrs->keys_str) { | 1905 | if (!attrs->keys_str) { |
1833 | ret = -ENOMEM; | 1906 | ret = -ENOMEM; |
1834 | goto out; | 1907 | goto out; |
1835 | } | 1908 | } |
1836 | } else if ((strncmp(str, "val=", strlen("val=")) == 0) || | 1909 | } else if ((str_has_prefix(str, "val=")) || |
1837 | (strncmp(str, "vals=", strlen("vals=")) == 0) || | 1910 | (str_has_prefix(str, "vals=")) || |
1838 | (strncmp(str, "values=", strlen("values=")) == 0)) { | 1911 | (str_has_prefix(str, "values="))) { |
1839 | attrs->vals_str = kstrdup(str, GFP_KERNEL); | 1912 | attrs->vals_str = kstrdup(str, GFP_KERNEL); |
1840 | if (!attrs->vals_str) { | 1913 | if (!attrs->vals_str) { |
1841 | ret = -ENOMEM; | 1914 | ret = -ENOMEM; |
1842 | goto out; | 1915 | goto out; |
1843 | } | 1916 | } |
1844 | } else if (strncmp(str, "sort=", strlen("sort=")) == 0) { | 1917 | } else if (str_has_prefix(str, "sort=")) { |
1845 | attrs->sort_key_str = kstrdup(str, GFP_KERNEL); | 1918 | attrs->sort_key_str = kstrdup(str, GFP_KERNEL); |
1846 | if (!attrs->sort_key_str) { | 1919 | if (!attrs->sort_key_str) { |
1847 | ret = -ENOMEM; | 1920 | ret = -ENOMEM; |
1848 | goto out; | 1921 | goto out; |
1849 | } | 1922 | } |
1850 | } else if (strncmp(str, "name=", strlen("name=")) == 0) { | 1923 | } else if (str_has_prefix(str, "name=")) { |
1851 | attrs->name = kstrdup(str, GFP_KERNEL); | 1924 | attrs->name = kstrdup(str, GFP_KERNEL); |
1852 | if (!attrs->name) { | 1925 | if (!attrs->name) { |
1853 | ret = -ENOMEM; | 1926 | ret = -ENOMEM; |
1854 | goto out; | 1927 | goto out; |
1855 | } | 1928 | } |
1856 | } else if (strncmp(str, "clock=", strlen("clock=")) == 0) { | 1929 | } else if (str_has_prefix(str, "clock=")) { |
1857 | strsep(&str, "="); | 1930 | strsep(&str, "="); |
1858 | if (!str) { | 1931 | if (!str) { |
1859 | ret = -EINVAL; | 1932 | ret = -EINVAL; |
@@ -1866,7 +1939,7 @@ static int parse_assignment(char *str, struct hist_trigger_attrs *attrs) | |||
1866 | ret = -ENOMEM; | 1939 | ret = -ENOMEM; |
1867 | goto out; | 1940 | goto out; |
1868 | } | 1941 | } |
1869 | } else if (strncmp(str, "size=", strlen("size=")) == 0) { | 1942 | } else if (str_has_prefix(str, "size=")) { |
1870 | int map_bits = parse_map_size(str); | 1943 | int map_bits = parse_map_size(str); |
1871 | 1944 | ||
1872 | if (map_bits < 0) { | 1945 | if (map_bits < 0) { |
@@ -2151,6 +2224,15 @@ static int contains_operator(char *str) | |||
2151 | return field_op; | 2224 | return field_op; |
2152 | } | 2225 | } |
2153 | 2226 | ||
2227 | static void __destroy_hist_field(struct hist_field *hist_field) | ||
2228 | { | ||
2229 | kfree(hist_field->var.name); | ||
2230 | kfree(hist_field->name); | ||
2231 | kfree(hist_field->type); | ||
2232 | |||
2233 | kfree(hist_field); | ||
2234 | } | ||
2235 | |||
2154 | static void destroy_hist_field(struct hist_field *hist_field, | 2236 | static void destroy_hist_field(struct hist_field *hist_field, |
2155 | unsigned int level) | 2237 | unsigned int level) |
2156 | { | 2238 | { |
@@ -2162,14 +2244,13 @@ static void destroy_hist_field(struct hist_field *hist_field, | |||
2162 | if (!hist_field) | 2244 | if (!hist_field) |
2163 | return; | 2245 | return; |
2164 | 2246 | ||
2247 | if (hist_field->flags & HIST_FIELD_FL_VAR_REF) | ||
2248 | return; /* var refs will be destroyed separately */ | ||
2249 | |||
2165 | for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) | 2250 | for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) |
2166 | destroy_hist_field(hist_field->operands[i], level + 1); | 2251 | destroy_hist_field(hist_field->operands[i], level + 1); |
2167 | 2252 | ||
2168 | kfree(hist_field->var.name); | 2253 | __destroy_hist_field(hist_field); |
2169 | kfree(hist_field->name); | ||
2170 | kfree(hist_field->type); | ||
2171 | |||
2172 | kfree(hist_field); | ||
2173 | } | 2254 | } |
2174 | 2255 | ||
2175 | static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data, | 2256 | static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data, |
@@ -2296,6 +2377,12 @@ static void destroy_hist_fields(struct hist_trigger_data *hist_data) | |||
2296 | hist_data->fields[i] = NULL; | 2377 | hist_data->fields[i] = NULL; |
2297 | } | 2378 | } |
2298 | } | 2379 | } |
2380 | |||
2381 | for (i = 0; i < hist_data->n_var_refs; i++) { | ||
2382 | WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF)); | ||
2383 | __destroy_hist_field(hist_data->var_refs[i]); | ||
2384 | hist_data->var_refs[i] = NULL; | ||
2385 | } | ||
2299 | } | 2386 | } |
2300 | 2387 | ||
2301 | static int init_var_ref(struct hist_field *ref_field, | 2388 | static int init_var_ref(struct hist_field *ref_field, |
@@ -2354,7 +2441,23 @@ static int init_var_ref(struct hist_field *ref_field, | |||
2354 | goto out; | 2441 | goto out; |
2355 | } | 2442 | } |
2356 | 2443 | ||
2357 | static struct hist_field *create_var_ref(struct hist_field *var_field, | 2444 | /** |
2445 | * create_var_ref - Create a variable reference and attach it to trigger | ||
2446 | * @hist_data: The trigger that will be referencing the variable | ||
2447 | * @var_field: The VAR field to create a reference to | ||
2448 | * @system: The optional system string | ||
2449 | * @event_name: The optional event_name string | ||
2450 | * | ||
2451 | * Given a variable hist_field, create a VAR_REF hist_field that | ||
2452 | * represents a reference to it. | ||
2453 | * | ||
2454 | * This function also adds the reference to the trigger that | ||
2455 | * now references the variable. | ||
2456 | * | ||
2457 | * Return: The VAR_REF field if successful, NULL if not | ||
2458 | */ | ||
2459 | static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data, | ||
2460 | struct hist_field *var_field, | ||
2358 | char *system, char *event_name) | 2461 | char *system, char *event_name) |
2359 | { | 2462 | { |
2360 | unsigned long flags = HIST_FIELD_FL_VAR_REF; | 2463 | unsigned long flags = HIST_FIELD_FL_VAR_REF; |
@@ -2366,6 +2469,9 @@ static struct hist_field *create_var_ref(struct hist_field *var_field, | |||
2366 | destroy_hist_field(ref_field, 0); | 2469 | destroy_hist_field(ref_field, 0); |
2367 | return NULL; | 2470 | return NULL; |
2368 | } | 2471 | } |
2472 | |||
2473 | hist_data->var_refs[hist_data->n_var_refs] = ref_field; | ||
2474 | ref_field->var_ref_idx = hist_data->n_var_refs++; | ||
2369 | } | 2475 | } |
2370 | 2476 | ||
2371 | return ref_field; | 2477 | return ref_field; |
@@ -2439,7 +2545,8 @@ static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data, | |||
2439 | 2545 | ||
2440 | var_field = find_event_var(hist_data, system, event_name, var_name); | 2546 | var_field = find_event_var(hist_data, system, event_name, var_name); |
2441 | if (var_field) | 2547 | if (var_field) |
2442 | ref_field = create_var_ref(var_field, system, event_name); | 2548 | ref_field = create_var_ref(hist_data, var_field, |
2549 | system, event_name); | ||
2443 | 2550 | ||
2444 | if (!ref_field) | 2551 | if (!ref_field) |
2445 | hist_err_event("Couldn't find variable: $", | 2552 | hist_err_event("Couldn't find variable: $", |
@@ -2557,8 +2664,6 @@ static struct hist_field *parse_atom(struct hist_trigger_data *hist_data, | |||
2557 | if (!s) { | 2664 | if (!s) { |
2558 | hist_field = parse_var_ref(hist_data, ref_system, ref_event, ref_var); | 2665 | hist_field = parse_var_ref(hist_data, ref_system, ref_event, ref_var); |
2559 | if (hist_field) { | 2666 | if (hist_field) { |
2560 | hist_data->var_refs[hist_data->n_var_refs] = hist_field; | ||
2561 | hist_field->var_ref_idx = hist_data->n_var_refs++; | ||
2562 | if (var_name) { | 2667 | if (var_name) { |
2563 | hist_field = create_alias(hist_data, hist_field, var_name); | 2668 | hist_field = create_alias(hist_data, hist_field, var_name); |
2564 | if (!hist_field) { | 2669 | if (!hist_field) { |
@@ -3332,7 +3437,6 @@ static int onmax_create(struct hist_trigger_data *hist_data, | |||
3332 | unsigned int var_ref_idx = hist_data->n_var_refs; | 3437 | unsigned int var_ref_idx = hist_data->n_var_refs; |
3333 | struct field_var *field_var; | 3438 | struct field_var *field_var; |
3334 | char *onmax_var_str, *param; | 3439 | char *onmax_var_str, *param; |
3335 | unsigned long flags; | ||
3336 | unsigned int i; | 3440 | unsigned int i; |
3337 | int ret = 0; | 3441 | int ret = 0; |
3338 | 3442 | ||
@@ -3349,18 +3453,10 @@ static int onmax_create(struct hist_trigger_data *hist_data, | |||
3349 | return -EINVAL; | 3453 | return -EINVAL; |
3350 | } | 3454 | } |
3351 | 3455 | ||
3352 | flags = HIST_FIELD_FL_VAR_REF; | 3456 | ref_field = create_var_ref(hist_data, var_field, NULL, NULL); |
3353 | ref_field = create_hist_field(hist_data, NULL, flags, NULL); | ||
3354 | if (!ref_field) | 3457 | if (!ref_field) |
3355 | return -ENOMEM; | 3458 | return -ENOMEM; |
3356 | 3459 | ||
3357 | if (init_var_ref(ref_field, var_field, NULL, NULL)) { | ||
3358 | destroy_hist_field(ref_field, 0); | ||
3359 | ret = -ENOMEM; | ||
3360 | goto out; | ||
3361 | } | ||
3362 | hist_data->var_refs[hist_data->n_var_refs] = ref_field; | ||
3363 | ref_field->var_ref_idx = hist_data->n_var_refs++; | ||
3364 | data->onmax.var = ref_field; | 3460 | data->onmax.var = ref_field; |
3365 | 3461 | ||
3366 | data->fn = onmax_save; | 3462 | data->fn = onmax_save; |
@@ -3462,7 +3558,7 @@ static struct action_data *onmax_parse(char *str) | |||
3462 | if (!onmax_fn_name || !str) | 3558 | if (!onmax_fn_name || !str) |
3463 | goto free; | 3559 | goto free; |
3464 | 3560 | ||
3465 | if (strncmp(onmax_fn_name, "save", strlen("save")) == 0) { | 3561 | if (str_has_prefix(onmax_fn_name, "save")) { |
3466 | char *params = strsep(&str, ")"); | 3562 | char *params = strsep(&str, ")"); |
3467 | 3563 | ||
3468 | if (!params) { | 3564 | if (!params) { |
@@ -3493,7 +3589,7 @@ static void onmatch_destroy(struct action_data *data) | |||
3493 | { | 3589 | { |
3494 | unsigned int i; | 3590 | unsigned int i; |
3495 | 3591 | ||
3496 | mutex_lock(&synth_event_mutex); | 3592 | lockdep_assert_held(&event_mutex); |
3497 | 3593 | ||
3498 | kfree(data->onmatch.match_event); | 3594 | kfree(data->onmatch.match_event); |
3499 | kfree(data->onmatch.match_event_system); | 3595 | kfree(data->onmatch.match_event_system); |
@@ -3506,8 +3602,6 @@ static void onmatch_destroy(struct action_data *data) | |||
3506 | data->onmatch.synth_event->ref--; | 3602 | data->onmatch.synth_event->ref--; |
3507 | 3603 | ||
3508 | kfree(data); | 3604 | kfree(data); |
3509 | |||
3510 | mutex_unlock(&synth_event_mutex); | ||
3511 | } | 3605 | } |
3512 | 3606 | ||
3513 | static void destroy_field_var(struct field_var *field_var) | 3607 | static void destroy_field_var(struct field_var *field_var) |
@@ -3539,23 +3633,6 @@ static void save_field_var(struct hist_trigger_data *hist_data, | |||
3539 | } | 3633 | } |
3540 | 3634 | ||
3541 | 3635 | ||
3542 | static void destroy_synth_var_refs(struct hist_trigger_data *hist_data) | ||
3543 | { | ||
3544 | unsigned int i; | ||
3545 | |||
3546 | for (i = 0; i < hist_data->n_synth_var_refs; i++) | ||
3547 | destroy_hist_field(hist_data->synth_var_refs[i], 0); | ||
3548 | } | ||
3549 | |||
3550 | static void save_synth_var_ref(struct hist_trigger_data *hist_data, | ||
3551 | struct hist_field *var_ref) | ||
3552 | { | ||
3553 | hist_data->synth_var_refs[hist_data->n_synth_var_refs++] = var_ref; | ||
3554 | |||
3555 | hist_data->var_refs[hist_data->n_var_refs] = var_ref; | ||
3556 | var_ref->var_ref_idx = hist_data->n_var_refs++; | ||
3557 | } | ||
3558 | |||
3559 | static int check_synth_field(struct synth_event *event, | 3636 | static int check_synth_field(struct synth_event *event, |
3560 | struct hist_field *hist_field, | 3637 | struct hist_field *hist_field, |
3561 | unsigned int field_pos) | 3638 | unsigned int field_pos) |
@@ -3658,15 +3735,14 @@ static int onmatch_create(struct hist_trigger_data *hist_data, | |||
3658 | struct synth_event *event; | 3735 | struct synth_event *event; |
3659 | int ret = 0; | 3736 | int ret = 0; |
3660 | 3737 | ||
3661 | mutex_lock(&synth_event_mutex); | 3738 | lockdep_assert_held(&event_mutex); |
3739 | |||
3662 | event = find_synth_event(data->onmatch.synth_event_name); | 3740 | event = find_synth_event(data->onmatch.synth_event_name); |
3663 | if (!event) { | 3741 | if (!event) { |
3664 | hist_err("onmatch: Couldn't find synthetic event: ", data->onmatch.synth_event_name); | 3742 | hist_err("onmatch: Couldn't find synthetic event: ", data->onmatch.synth_event_name); |
3665 | mutex_unlock(&synth_event_mutex); | ||
3666 | return -EINVAL; | 3743 | return -EINVAL; |
3667 | } | 3744 | } |
3668 | event->ref++; | 3745 | event->ref++; |
3669 | mutex_unlock(&synth_event_mutex); | ||
3670 | 3746 | ||
3671 | var_ref_idx = hist_data->n_var_refs; | 3747 | var_ref_idx = hist_data->n_var_refs; |
3672 | 3748 | ||
@@ -3708,14 +3784,14 @@ static int onmatch_create(struct hist_trigger_data *hist_data, | |||
3708 | } | 3784 | } |
3709 | 3785 | ||
3710 | if (check_synth_field(event, hist_field, field_pos) == 0) { | 3786 | if (check_synth_field(event, hist_field, field_pos) == 0) { |
3711 | var_ref = create_var_ref(hist_field, system, event_name); | 3787 | var_ref = create_var_ref(hist_data, hist_field, |
3788 | system, event_name); | ||
3712 | if (!var_ref) { | 3789 | if (!var_ref) { |
3713 | kfree(p); | 3790 | kfree(p); |
3714 | ret = -ENOMEM; | 3791 | ret = -ENOMEM; |
3715 | goto err; | 3792 | goto err; |
3716 | } | 3793 | } |
3717 | 3794 | ||
3718 | save_synth_var_ref(hist_data, var_ref); | ||
3719 | field_pos++; | 3795 | field_pos++; |
3720 | kfree(p); | 3796 | kfree(p); |
3721 | continue; | 3797 | continue; |
@@ -3740,9 +3816,7 @@ static int onmatch_create(struct hist_trigger_data *hist_data, | |||
3740 | out: | 3816 | out: |
3741 | return ret; | 3817 | return ret; |
3742 | err: | 3818 | err: |
3743 | mutex_lock(&synth_event_mutex); | ||
3744 | event->ref--; | 3819 | event->ref--; |
3745 | mutex_unlock(&synth_event_mutex); | ||
3746 | 3820 | ||
3747 | goto out; | 3821 | goto out; |
3748 | } | 3822 | } |
@@ -4268,12 +4342,13 @@ static int parse_actions(struct hist_trigger_data *hist_data) | |||
4268 | unsigned int i; | 4342 | unsigned int i; |
4269 | int ret = 0; | 4343 | int ret = 0; |
4270 | char *str; | 4344 | char *str; |
4345 | int len; | ||
4271 | 4346 | ||
4272 | for (i = 0; i < hist_data->attrs->n_actions; i++) { | 4347 | for (i = 0; i < hist_data->attrs->n_actions; i++) { |
4273 | str = hist_data->attrs->action_str[i]; | 4348 | str = hist_data->attrs->action_str[i]; |
4274 | 4349 | ||
4275 | if (strncmp(str, "onmatch(", strlen("onmatch(")) == 0) { | 4350 | if ((len = str_has_prefix(str, "onmatch("))) { |
4276 | char *action_str = str + strlen("onmatch("); | 4351 | char *action_str = str + len; |
4277 | 4352 | ||
4278 | data = onmatch_parse(tr, action_str); | 4353 | data = onmatch_parse(tr, action_str); |
4279 | if (IS_ERR(data)) { | 4354 | if (IS_ERR(data)) { |
@@ -4281,8 +4356,8 @@ static int parse_actions(struct hist_trigger_data *hist_data) | |||
4281 | break; | 4356 | break; |
4282 | } | 4357 | } |
4283 | data->fn = action_trace; | 4358 | data->fn = action_trace; |
4284 | } else if (strncmp(str, "onmax(", strlen("onmax(")) == 0) { | 4359 | } else if ((len = str_has_prefix(str, "onmax("))) { |
4285 | char *action_str = str + strlen("onmax("); | 4360 | char *action_str = str + len; |
4286 | 4361 | ||
4287 | data = onmax_parse(action_str); | 4362 | data = onmax_parse(action_str); |
4288 | if (IS_ERR(data)) { | 4363 | if (IS_ERR(data)) { |
@@ -4461,7 +4536,6 @@ static void destroy_hist_data(struct hist_trigger_data *hist_data) | |||
4461 | destroy_actions(hist_data); | 4536 | destroy_actions(hist_data); |
4462 | destroy_field_vars(hist_data); | 4537 | destroy_field_vars(hist_data); |
4463 | destroy_field_var_hists(hist_data); | 4538 | destroy_field_var_hists(hist_data); |
4464 | destroy_synth_var_refs(hist_data); | ||
4465 | 4539 | ||
4466 | kfree(hist_data); | 4540 | kfree(hist_data); |
4467 | } | 4541 | } |
@@ -5450,6 +5524,8 @@ static void hist_unreg_all(struct trace_event_file *file) | |||
5450 | struct synth_event *se; | 5524 | struct synth_event *se; |
5451 | const char *se_name; | 5525 | const char *se_name; |
5452 | 5526 | ||
5527 | lockdep_assert_held(&event_mutex); | ||
5528 | |||
5453 | if (hist_file_check_refs(file)) | 5529 | if (hist_file_check_refs(file)) |
5454 | return; | 5530 | return; |
5455 | 5531 | ||
@@ -5459,12 +5535,10 @@ static void hist_unreg_all(struct trace_event_file *file) | |||
5459 | list_del_rcu(&test->list); | 5535 | list_del_rcu(&test->list); |
5460 | trace_event_trigger_enable_disable(file, 0); | 5536 | trace_event_trigger_enable_disable(file, 0); |
5461 | 5537 | ||
5462 | mutex_lock(&synth_event_mutex); | ||
5463 | se_name = trace_event_name(file->event_call); | 5538 | se_name = trace_event_name(file->event_call); |
5464 | se = find_synth_event(se_name); | 5539 | se = find_synth_event(se_name); |
5465 | if (se) | 5540 | if (se) |
5466 | se->ref--; | 5541 | se->ref--; |
5467 | mutex_unlock(&synth_event_mutex); | ||
5468 | 5542 | ||
5469 | update_cond_flag(file); | 5543 | update_cond_flag(file); |
5470 | if (hist_data->enable_timestamps) | 5544 | if (hist_data->enable_timestamps) |
@@ -5490,6 +5564,8 @@ static int event_hist_trigger_func(struct event_command *cmd_ops, | |||
5490 | char *trigger, *p; | 5564 | char *trigger, *p; |
5491 | int ret = 0; | 5565 | int ret = 0; |
5492 | 5566 | ||
5567 | lockdep_assert_held(&event_mutex); | ||
5568 | |||
5493 | if (glob && strlen(glob)) { | 5569 | if (glob && strlen(glob)) { |
5494 | last_cmd_set(param); | 5570 | last_cmd_set(param); |
5495 | hist_err_clear(); | 5571 | hist_err_clear(); |
@@ -5516,9 +5592,9 @@ static int event_hist_trigger_func(struct event_command *cmd_ops, | |||
5516 | p++; | 5592 | p++; |
5517 | continue; | 5593 | continue; |
5518 | } | 5594 | } |
5519 | if (p >= param + strlen(param) - strlen("if") - 1) | 5595 | if (p >= param + strlen(param) - (sizeof("if") - 1) - 1) |
5520 | return -EINVAL; | 5596 | return -EINVAL; |
5521 | if (*(p + strlen("if")) != ' ' && *(p + strlen("if")) != '\t') { | 5597 | if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') { |
5522 | p++; | 5598 | p++; |
5523 | continue; | 5599 | continue; |
5524 | } | 5600 | } |
@@ -5580,14 +5656,10 @@ static int event_hist_trigger_func(struct event_command *cmd_ops, | |||
5580 | } | 5656 | } |
5581 | 5657 | ||
5582 | cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file); | 5658 | cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file); |
5583 | |||
5584 | mutex_lock(&synth_event_mutex); | ||
5585 | se_name = trace_event_name(file->event_call); | 5659 | se_name = trace_event_name(file->event_call); |
5586 | se = find_synth_event(se_name); | 5660 | se = find_synth_event(se_name); |
5587 | if (se) | 5661 | if (se) |
5588 | se->ref--; | 5662 | se->ref--; |
5589 | mutex_unlock(&synth_event_mutex); | ||
5590 | |||
5591 | ret = 0; | 5663 | ret = 0; |
5592 | goto out_free; | 5664 | goto out_free; |
5593 | } | 5665 | } |
@@ -5623,13 +5695,10 @@ enable: | |||
5623 | if (ret) | 5695 | if (ret) |
5624 | goto out_unreg; | 5696 | goto out_unreg; |
5625 | 5697 | ||
5626 | mutex_lock(&synth_event_mutex); | ||
5627 | se_name = trace_event_name(file->event_call); | 5698 | se_name = trace_event_name(file->event_call); |
5628 | se = find_synth_event(se_name); | 5699 | se = find_synth_event(se_name); |
5629 | if (se) | 5700 | if (se) |
5630 | se->ref++; | 5701 | se->ref++; |
5631 | mutex_unlock(&synth_event_mutex); | ||
5632 | |||
5633 | /* Just return zero, not the number of registered triggers */ | 5702 | /* Just return zero, not the number of registered triggers */ |
5634 | ret = 0; | 5703 | ret = 0; |
5635 | out: | 5704 | out: |
@@ -5812,6 +5881,12 @@ static __init int trace_events_hist_init(void) | |||
5812 | struct dentry *d_tracer; | 5881 | struct dentry *d_tracer; |
5813 | int err = 0; | 5882 | int err = 0; |
5814 | 5883 | ||
5884 | err = dyn_event_register(&synth_event_ops); | ||
5885 | if (err) { | ||
5886 | pr_warn("Could not register synth_event_ops\n"); | ||
5887 | return err; | ||
5888 | } | ||
5889 | |||
5815 | d_tracer = tracing_init_dentry(); | 5890 | d_tracer = tracing_init_dentry(); |
5816 | if (IS_ERR(d_tracer)) { | 5891 | if (IS_ERR(d_tracer)) { |
5817 | err = PTR_ERR(d_tracer); | 5892 | err = PTR_ERR(d_tracer); |
diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c index 086af4f5c3e8..c2af1560e856 100644 --- a/kernel/trace/trace_functions_graph.c +++ b/kernel/trace/trace_functions_graph.c | |||
@@ -16,33 +16,6 @@ | |||
16 | #include "trace.h" | 16 | #include "trace.h" |
17 | #include "trace_output.h" | 17 | #include "trace_output.h" |
18 | 18 | ||
19 | static bool kill_ftrace_graph; | ||
20 | |||
21 | /** | ||
22 | * ftrace_graph_is_dead - returns true if ftrace_graph_stop() was called | ||
23 | * | ||
24 | * ftrace_graph_stop() is called when a severe error is detected in | ||
25 | * the function graph tracing. This function is called by the critical | ||
26 | * paths of function graph to keep those paths from doing any more harm. | ||
27 | */ | ||
28 | bool ftrace_graph_is_dead(void) | ||
29 | { | ||
30 | return kill_ftrace_graph; | ||
31 | } | ||
32 | |||
33 | /** | ||
34 | * ftrace_graph_stop - set to permanently disable function graph tracincg | ||
35 | * | ||
36 | * In case of an error int function graph tracing, this is called | ||
37 | * to try to keep function graph tracing from causing any more harm. | ||
38 | * Usually this is pretty severe and this is called to try to at least | ||
39 | * get a warning out to the user. | ||
40 | */ | ||
41 | void ftrace_graph_stop(void) | ||
42 | { | ||
43 | kill_ftrace_graph = true; | ||
44 | } | ||
45 | |||
46 | /* When set, irq functions will be ignored */ | 19 | /* When set, irq functions will be ignored */ |
47 | static int ftrace_graph_skip_irqs; | 20 | static int ftrace_graph_skip_irqs; |
48 | 21 | ||
@@ -87,8 +60,12 @@ static struct tracer_opt trace_opts[] = { | |||
87 | { TRACER_OPT(funcgraph-tail, TRACE_GRAPH_PRINT_TAIL) }, | 60 | { TRACER_OPT(funcgraph-tail, TRACE_GRAPH_PRINT_TAIL) }, |
88 | /* Include sleep time (scheduled out) between entry and return */ | 61 | /* Include sleep time (scheduled out) between entry and return */ |
89 | { TRACER_OPT(sleep-time, TRACE_GRAPH_SLEEP_TIME) }, | 62 | { TRACER_OPT(sleep-time, TRACE_GRAPH_SLEEP_TIME) }, |
63 | |||
64 | #ifdef CONFIG_FUNCTION_PROFILER | ||
90 | /* Include time within nested functions */ | 65 | /* Include time within nested functions */ |
91 | { TRACER_OPT(graph-time, TRACE_GRAPH_GRAPH_TIME) }, | 66 | { TRACER_OPT(graph-time, TRACE_GRAPH_GRAPH_TIME) }, |
67 | #endif | ||
68 | |||
92 | { } /* Empty entry */ | 69 | { } /* Empty entry */ |
93 | }; | 70 | }; |
94 | 71 | ||
@@ -117,258 +94,6 @@ static void | |||
117 | print_graph_duration(struct trace_array *tr, unsigned long long duration, | 94 | print_graph_duration(struct trace_array *tr, unsigned long long duration, |
118 | struct trace_seq *s, u32 flags); | 95 | struct trace_seq *s, u32 flags); |
119 | 96 | ||
120 | /* Add a function return address to the trace stack on thread info.*/ | ||
121 | static int | ||
122 | ftrace_push_return_trace(unsigned long ret, unsigned long func, | ||
123 | unsigned long frame_pointer, unsigned long *retp) | ||
124 | { | ||
125 | unsigned long long calltime; | ||
126 | int index; | ||
127 | |||
128 | if (unlikely(ftrace_graph_is_dead())) | ||
129 | return -EBUSY; | ||
130 | |||
131 | if (!current->ret_stack) | ||
132 | return -EBUSY; | ||
133 | |||
134 | /* | ||
135 | * We must make sure the ret_stack is tested before we read | ||
136 | * anything else. | ||
137 | */ | ||
138 | smp_rmb(); | ||
139 | |||
140 | /* The return trace stack is full */ | ||
141 | if (current->curr_ret_stack == FTRACE_RETFUNC_DEPTH - 1) { | ||
142 | atomic_inc(¤t->trace_overrun); | ||
143 | return -EBUSY; | ||
144 | } | ||
145 | |||
146 | /* | ||
147 | * The curr_ret_stack is an index to ftrace return stack of | ||
148 | * current task. Its value should be in [0, FTRACE_RETFUNC_ | ||
149 | * DEPTH) when the function graph tracer is used. To support | ||
150 | * filtering out specific functions, it makes the index | ||
151 | * negative by subtracting huge value (FTRACE_NOTRACE_DEPTH) | ||
152 | * so when it sees a negative index the ftrace will ignore | ||
153 | * the record. And the index gets recovered when returning | ||
154 | * from the filtered function by adding the FTRACE_NOTRACE_ | ||
155 | * DEPTH and then it'll continue to record functions normally. | ||
156 | * | ||
157 | * The curr_ret_stack is initialized to -1 and get increased | ||
158 | * in this function. So it can be less than -1 only if it was | ||
159 | * filtered out via ftrace_graph_notrace_addr() which can be | ||
160 | * set from set_graph_notrace file in tracefs by user. | ||
161 | */ | ||
162 | if (current->curr_ret_stack < -1) | ||
163 | return -EBUSY; | ||
164 | |||
165 | calltime = trace_clock_local(); | ||
166 | |||
167 | index = ++current->curr_ret_stack; | ||
168 | if (ftrace_graph_notrace_addr(func)) | ||
169 | current->curr_ret_stack -= FTRACE_NOTRACE_DEPTH; | ||
170 | barrier(); | ||
171 | current->ret_stack[index].ret = ret; | ||
172 | current->ret_stack[index].func = func; | ||
173 | current->ret_stack[index].calltime = calltime; | ||
174 | #ifdef HAVE_FUNCTION_GRAPH_FP_TEST | ||
175 | current->ret_stack[index].fp = frame_pointer; | ||
176 | #endif | ||
177 | #ifdef HAVE_FUNCTION_GRAPH_RET_ADDR_PTR | ||
178 | current->ret_stack[index].retp = retp; | ||
179 | #endif | ||
180 | return 0; | ||
181 | } | ||
182 | |||
183 | int function_graph_enter(unsigned long ret, unsigned long func, | ||
184 | unsigned long frame_pointer, unsigned long *retp) | ||
185 | { | ||
186 | struct ftrace_graph_ent trace; | ||
187 | |||
188 | trace.func = func; | ||
189 | trace.depth = ++current->curr_ret_depth; | ||
190 | |||
191 | if (ftrace_push_return_trace(ret, func, | ||
192 | frame_pointer, retp)) | ||
193 | goto out; | ||
194 | |||
195 | /* Only trace if the calling function expects to */ | ||
196 | if (!ftrace_graph_entry(&trace)) | ||
197 | goto out_ret; | ||
198 | |||
199 | return 0; | ||
200 | out_ret: | ||
201 | current->curr_ret_stack--; | ||
202 | out: | ||
203 | current->curr_ret_depth--; | ||
204 | return -EBUSY; | ||
205 | } | ||
206 | |||
207 | /* Retrieve a function return address to the trace stack on thread info.*/ | ||
208 | static void | ||
209 | ftrace_pop_return_trace(struct ftrace_graph_ret *trace, unsigned long *ret, | ||
210 | unsigned long frame_pointer) | ||
211 | { | ||
212 | int index; | ||
213 | |||
214 | index = current->curr_ret_stack; | ||
215 | |||
216 | /* | ||
217 | * A negative index here means that it's just returned from a | ||
218 | * notrace'd function. Recover index to get an original | ||
219 | * return address. See ftrace_push_return_trace(). | ||
220 | * | ||
221 | * TODO: Need to check whether the stack gets corrupted. | ||
222 | */ | ||
223 | if (index < 0) | ||
224 | index += FTRACE_NOTRACE_DEPTH; | ||
225 | |||
226 | if (unlikely(index < 0 || index >= FTRACE_RETFUNC_DEPTH)) { | ||
227 | ftrace_graph_stop(); | ||
228 | WARN_ON(1); | ||
229 | /* Might as well panic, otherwise we have no where to go */ | ||
230 | *ret = (unsigned long)panic; | ||
231 | return; | ||
232 | } | ||
233 | |||
234 | #ifdef HAVE_FUNCTION_GRAPH_FP_TEST | ||
235 | /* | ||
236 | * The arch may choose to record the frame pointer used | ||
237 | * and check it here to make sure that it is what we expect it | ||
238 | * to be. If gcc does not set the place holder of the return | ||
239 | * address in the frame pointer, and does a copy instead, then | ||
240 | * the function graph trace will fail. This test detects this | ||
241 | * case. | ||
242 | * | ||
243 | * Currently, x86_32 with optimize for size (-Os) makes the latest | ||
244 | * gcc do the above. | ||
245 | * | ||
246 | * Note, -mfentry does not use frame pointers, and this test | ||
247 | * is not needed if CC_USING_FENTRY is set. | ||
248 | */ | ||
249 | if (unlikely(current->ret_stack[index].fp != frame_pointer)) { | ||
250 | ftrace_graph_stop(); | ||
251 | WARN(1, "Bad frame pointer: expected %lx, received %lx\n" | ||
252 | " from func %ps return to %lx\n", | ||
253 | current->ret_stack[index].fp, | ||
254 | frame_pointer, | ||
255 | (void *)current->ret_stack[index].func, | ||
256 | current->ret_stack[index].ret); | ||
257 | *ret = (unsigned long)panic; | ||
258 | return; | ||
259 | } | ||
260 | #endif | ||
261 | |||
262 | *ret = current->ret_stack[index].ret; | ||
263 | trace->func = current->ret_stack[index].func; | ||
264 | trace->calltime = current->ret_stack[index].calltime; | ||
265 | trace->overrun = atomic_read(¤t->trace_overrun); | ||
266 | trace->depth = current->curr_ret_depth--; | ||
267 | /* | ||
268 | * We still want to trace interrupts coming in if | ||
269 | * max_depth is set to 1. Make sure the decrement is | ||
270 | * seen before ftrace_graph_return. | ||
271 | */ | ||
272 | barrier(); | ||
273 | } | ||
274 | |||
275 | /* | ||
276 | * Send the trace to the ring-buffer. | ||
277 | * @return the original return address. | ||
278 | */ | ||
279 | unsigned long ftrace_return_to_handler(unsigned long frame_pointer) | ||
280 | { | ||
281 | struct ftrace_graph_ret trace; | ||
282 | unsigned long ret; | ||
283 | |||
284 | ftrace_pop_return_trace(&trace, &ret, frame_pointer); | ||
285 | trace.rettime = trace_clock_local(); | ||
286 | ftrace_graph_return(&trace); | ||
287 | /* | ||
288 | * The ftrace_graph_return() may still access the current | ||
289 | * ret_stack structure, we need to make sure the update of | ||
290 | * curr_ret_stack is after that. | ||
291 | */ | ||
292 | barrier(); | ||
293 | current->curr_ret_stack--; | ||
294 | /* | ||
295 | * The curr_ret_stack can be less than -1 only if it was | ||
296 | * filtered out and it's about to return from the function. | ||
297 | * Recover the index and continue to trace normal functions. | ||
298 | */ | ||
299 | if (current->curr_ret_stack < -1) { | ||
300 | current->curr_ret_stack += FTRACE_NOTRACE_DEPTH; | ||
301 | return ret; | ||
302 | } | ||
303 | |||
304 | if (unlikely(!ret)) { | ||
305 | ftrace_graph_stop(); | ||
306 | WARN_ON(1); | ||
307 | /* Might as well panic. What else to do? */ | ||
308 | ret = (unsigned long)panic; | ||
309 | } | ||
310 | |||
311 | return ret; | ||
312 | } | ||
313 | |||
314 | /** | ||
315 | * ftrace_graph_ret_addr - convert a potentially modified stack return address | ||
316 | * to its original value | ||
317 | * | ||
318 | * This function can be called by stack unwinding code to convert a found stack | ||
319 | * return address ('ret') to its original value, in case the function graph | ||
320 | * tracer has modified it to be 'return_to_handler'. If the address hasn't | ||
321 | * been modified, the unchanged value of 'ret' is returned. | ||
322 | * | ||
323 | * 'idx' is a state variable which should be initialized by the caller to zero | ||
324 | * before the first call. | ||
325 | * | ||
326 | * 'retp' is a pointer to the return address on the stack. It's ignored if | ||
327 | * the arch doesn't have HAVE_FUNCTION_GRAPH_RET_ADDR_PTR defined. | ||
328 | */ | ||
329 | #ifdef HAVE_FUNCTION_GRAPH_RET_ADDR_PTR | ||
330 | unsigned long ftrace_graph_ret_addr(struct task_struct *task, int *idx, | ||
331 | unsigned long ret, unsigned long *retp) | ||
332 | { | ||
333 | int index = task->curr_ret_stack; | ||
334 | int i; | ||
335 | |||
336 | if (ret != (unsigned long)return_to_handler) | ||
337 | return ret; | ||
338 | |||
339 | if (index < -1) | ||
340 | index += FTRACE_NOTRACE_DEPTH; | ||
341 | |||
342 | if (index < 0) | ||
343 | return ret; | ||
344 | |||
345 | for (i = 0; i <= index; i++) | ||
346 | if (task->ret_stack[i].retp == retp) | ||
347 | return task->ret_stack[i].ret; | ||
348 | |||
349 | return ret; | ||
350 | } | ||
351 | #else /* !HAVE_FUNCTION_GRAPH_RET_ADDR_PTR */ | ||
352 | unsigned long ftrace_graph_ret_addr(struct task_struct *task, int *idx, | ||
353 | unsigned long ret, unsigned long *retp) | ||
354 | { | ||
355 | int task_idx; | ||
356 | |||
357 | if (ret != (unsigned long)return_to_handler) | ||
358 | return ret; | ||
359 | |||
360 | task_idx = task->curr_ret_stack; | ||
361 | |||
362 | if (!task->ret_stack || task_idx < *idx) | ||
363 | return ret; | ||
364 | |||
365 | task_idx -= *idx; | ||
366 | (*idx)++; | ||
367 | |||
368 | return task->ret_stack[task_idx].ret; | ||
369 | } | ||
370 | #endif /* HAVE_FUNCTION_GRAPH_RET_ADDR_PTR */ | ||
371 | |||
372 | int __trace_graph_entry(struct trace_array *tr, | 97 | int __trace_graph_entry(struct trace_array *tr, |
373 | struct ftrace_graph_ent *trace, | 98 | struct ftrace_graph_ent *trace, |
374 | unsigned long flags, | 99 | unsigned long flags, |
@@ -409,6 +134,18 @@ int trace_graph_entry(struct ftrace_graph_ent *trace) | |||
409 | int cpu; | 134 | int cpu; |
410 | int pc; | 135 | int pc; |
411 | 136 | ||
137 | if (trace_recursion_test(TRACE_GRAPH_NOTRACE_BIT)) | ||
138 | return 0; | ||
139 | |||
140 | if (ftrace_graph_notrace_addr(trace->func)) { | ||
141 | trace_recursion_set(TRACE_GRAPH_NOTRACE_BIT); | ||
142 | /* | ||
143 | * Need to return 1 to have the return called | ||
144 | * that will clear the NOTRACE bit. | ||
145 | */ | ||
146 | return 1; | ||
147 | } | ||
148 | |||
412 | if (!ftrace_trace_task(tr)) | 149 | if (!ftrace_trace_task(tr)) |
413 | return 0; | 150 | return 0; |
414 | 151 | ||
@@ -511,6 +248,11 @@ void trace_graph_return(struct ftrace_graph_ret *trace) | |||
511 | 248 | ||
512 | ftrace_graph_addr_finish(trace); | 249 | ftrace_graph_addr_finish(trace); |
513 | 250 | ||
251 | if (trace_recursion_test(TRACE_GRAPH_NOTRACE_BIT)) { | ||
252 | trace_recursion_clear(TRACE_GRAPH_NOTRACE_BIT); | ||
253 | return; | ||
254 | } | ||
255 | |||
514 | local_irq_save(flags); | 256 | local_irq_save(flags); |
515 | cpu = raw_smp_processor_id(); | 257 | cpu = raw_smp_processor_id(); |
516 | data = per_cpu_ptr(tr->trace_buffer.data, cpu); | 258 | data = per_cpu_ptr(tr->trace_buffer.data, cpu); |
@@ -536,6 +278,11 @@ static void trace_graph_thresh_return(struct ftrace_graph_ret *trace) | |||
536 | { | 278 | { |
537 | ftrace_graph_addr_finish(trace); | 279 | ftrace_graph_addr_finish(trace); |
538 | 280 | ||
281 | if (trace_recursion_test(TRACE_GRAPH_NOTRACE_BIT)) { | ||
282 | trace_recursion_clear(TRACE_GRAPH_NOTRACE_BIT); | ||
283 | return; | ||
284 | } | ||
285 | |||
539 | if (tracing_thresh && | 286 | if (tracing_thresh && |
540 | (trace->rettime - trace->calltime < tracing_thresh)) | 287 | (trace->rettime - trace->calltime < tracing_thresh)) |
541 | return; | 288 | return; |
@@ -543,17 +290,25 @@ static void trace_graph_thresh_return(struct ftrace_graph_ret *trace) | |||
543 | trace_graph_return(trace); | 290 | trace_graph_return(trace); |
544 | } | 291 | } |
545 | 292 | ||
293 | static struct fgraph_ops funcgraph_thresh_ops = { | ||
294 | .entryfunc = &trace_graph_entry, | ||
295 | .retfunc = &trace_graph_thresh_return, | ||
296 | }; | ||
297 | |||
298 | static struct fgraph_ops funcgraph_ops = { | ||
299 | .entryfunc = &trace_graph_entry, | ||
300 | .retfunc = &trace_graph_return, | ||
301 | }; | ||
302 | |||
546 | static int graph_trace_init(struct trace_array *tr) | 303 | static int graph_trace_init(struct trace_array *tr) |
547 | { | 304 | { |
548 | int ret; | 305 | int ret; |
549 | 306 | ||
550 | set_graph_array(tr); | 307 | set_graph_array(tr); |
551 | if (tracing_thresh) | 308 | if (tracing_thresh) |
552 | ret = register_ftrace_graph(&trace_graph_thresh_return, | 309 | ret = register_ftrace_graph(&funcgraph_thresh_ops); |
553 | &trace_graph_entry); | ||
554 | else | 310 | else |
555 | ret = register_ftrace_graph(&trace_graph_return, | 311 | ret = register_ftrace_graph(&funcgraph_ops); |
556 | &trace_graph_entry); | ||
557 | if (ret) | 312 | if (ret) |
558 | return ret; | 313 | return ret; |
559 | tracing_start_cmdline_record(); | 314 | tracing_start_cmdline_record(); |
@@ -564,7 +319,10 @@ static int graph_trace_init(struct trace_array *tr) | |||
564 | static void graph_trace_reset(struct trace_array *tr) | 319 | static void graph_trace_reset(struct trace_array *tr) |
565 | { | 320 | { |
566 | tracing_stop_cmdline_record(); | 321 | tracing_stop_cmdline_record(); |
567 | unregister_ftrace_graph(); | 322 | if (tracing_thresh) |
323 | unregister_ftrace_graph(&funcgraph_thresh_ops); | ||
324 | else | ||
325 | unregister_ftrace_graph(&funcgraph_ops); | ||
568 | } | 326 | } |
569 | 327 | ||
570 | static int graph_trace_update_thresh(struct trace_array *tr) | 328 | static int graph_trace_update_thresh(struct trace_array *tr) |
@@ -874,10 +632,6 @@ print_graph_entry_leaf(struct trace_iterator *iter, | |||
874 | 632 | ||
875 | cpu_data = per_cpu_ptr(data->cpu_data, cpu); | 633 | cpu_data = per_cpu_ptr(data->cpu_data, cpu); |
876 | 634 | ||
877 | /* If a graph tracer ignored set_graph_notrace */ | ||
878 | if (call->depth < -1) | ||
879 | call->depth += FTRACE_NOTRACE_DEPTH; | ||
880 | |||
881 | /* | 635 | /* |
882 | * Comments display at + 1 to depth. Since | 636 | * Comments display at + 1 to depth. Since |
883 | * this is a leaf function, keep the comments | 637 | * this is a leaf function, keep the comments |
@@ -920,10 +674,6 @@ print_graph_entry_nested(struct trace_iterator *iter, | |||
920 | struct fgraph_cpu_data *cpu_data; | 674 | struct fgraph_cpu_data *cpu_data; |
921 | int cpu = iter->cpu; | 675 | int cpu = iter->cpu; |
922 | 676 | ||
923 | /* If a graph tracer ignored set_graph_notrace */ | ||
924 | if (call->depth < -1) | ||
925 | call->depth += FTRACE_NOTRACE_DEPTH; | ||
926 | |||
927 | cpu_data = per_cpu_ptr(data->cpu_data, cpu); | 677 | cpu_data = per_cpu_ptr(data->cpu_data, cpu); |
928 | cpu_data->depth = call->depth; | 678 | cpu_data->depth = call->depth; |
929 | 679 | ||
diff --git a/kernel/trace/trace_irqsoff.c b/kernel/trace/trace_irqsoff.c index 98ea6d28df15..d3294721f119 100644 --- a/kernel/trace/trace_irqsoff.c +++ b/kernel/trace/trace_irqsoff.c | |||
@@ -218,6 +218,11 @@ static void irqsoff_graph_return(struct ftrace_graph_ret *trace) | |||
218 | atomic_dec(&data->disabled); | 218 | atomic_dec(&data->disabled); |
219 | } | 219 | } |
220 | 220 | ||
221 | static struct fgraph_ops fgraph_ops = { | ||
222 | .entryfunc = &irqsoff_graph_entry, | ||
223 | .retfunc = &irqsoff_graph_return, | ||
224 | }; | ||
225 | |||
221 | static void irqsoff_trace_open(struct trace_iterator *iter) | 226 | static void irqsoff_trace_open(struct trace_iterator *iter) |
222 | { | 227 | { |
223 | if (is_graph(iter->tr)) | 228 | if (is_graph(iter->tr)) |
@@ -272,13 +277,6 @@ __trace_function(struct trace_array *tr, | |||
272 | #else | 277 | #else |
273 | #define __trace_function trace_function | 278 | #define __trace_function trace_function |
274 | 279 | ||
275 | #ifdef CONFIG_FUNCTION_TRACER | ||
276 | static int irqsoff_graph_entry(struct ftrace_graph_ent *trace) | ||
277 | { | ||
278 | return -1; | ||
279 | } | ||
280 | #endif | ||
281 | |||
282 | static enum print_line_t irqsoff_print_line(struct trace_iterator *iter) | 280 | static enum print_line_t irqsoff_print_line(struct trace_iterator *iter) |
283 | { | 281 | { |
284 | return TRACE_TYPE_UNHANDLED; | 282 | return TRACE_TYPE_UNHANDLED; |
@@ -288,7 +286,6 @@ static void irqsoff_trace_open(struct trace_iterator *iter) { } | |||
288 | static void irqsoff_trace_close(struct trace_iterator *iter) { } | 286 | static void irqsoff_trace_close(struct trace_iterator *iter) { } |
289 | 287 | ||
290 | #ifdef CONFIG_FUNCTION_TRACER | 288 | #ifdef CONFIG_FUNCTION_TRACER |
291 | static void irqsoff_graph_return(struct ftrace_graph_ret *trace) { } | ||
292 | static void irqsoff_print_header(struct seq_file *s) | 289 | static void irqsoff_print_header(struct seq_file *s) |
293 | { | 290 | { |
294 | trace_default_header(s); | 291 | trace_default_header(s); |
@@ -468,8 +465,7 @@ static int register_irqsoff_function(struct trace_array *tr, int graph, int set) | |||
468 | return 0; | 465 | return 0; |
469 | 466 | ||
470 | if (graph) | 467 | if (graph) |
471 | ret = register_ftrace_graph(&irqsoff_graph_return, | 468 | ret = register_ftrace_graph(&fgraph_ops); |
472 | &irqsoff_graph_entry); | ||
473 | else | 469 | else |
474 | ret = register_ftrace_function(tr->ops); | 470 | ret = register_ftrace_function(tr->ops); |
475 | 471 | ||
@@ -485,7 +481,7 @@ static void unregister_irqsoff_function(struct trace_array *tr, int graph) | |||
485 | return; | 481 | return; |
486 | 482 | ||
487 | if (graph) | 483 | if (graph) |
488 | unregister_ftrace_graph(); | 484 | unregister_ftrace_graph(&fgraph_ops); |
489 | else | 485 | else |
490 | unregister_ftrace_function(tr->ops); | 486 | unregister_ftrace_function(tr->ops); |
491 | 487 | ||
diff --git a/kernel/trace/trace_kprobe.c b/kernel/trace/trace_kprobe.c index adc153ab51c0..5c19b8c41c7e 100644 --- a/kernel/trace/trace_kprobe.c +++ b/kernel/trace/trace_kprobe.c | |||
@@ -12,6 +12,7 @@ | |||
12 | #include <linux/rculist.h> | 12 | #include <linux/rculist.h> |
13 | #include <linux/error-injection.h> | 13 | #include <linux/error-injection.h> |
14 | 14 | ||
15 | #include "trace_dynevent.h" | ||
15 | #include "trace_kprobe_selftest.h" | 16 | #include "trace_kprobe_selftest.h" |
16 | #include "trace_probe.h" | 17 | #include "trace_probe.h" |
17 | #include "trace_probe_tmpl.h" | 18 | #include "trace_probe_tmpl.h" |
@@ -19,17 +20,51 @@ | |||
19 | #define KPROBE_EVENT_SYSTEM "kprobes" | 20 | #define KPROBE_EVENT_SYSTEM "kprobes" |
20 | #define KRETPROBE_MAXACTIVE_MAX 4096 | 21 | #define KRETPROBE_MAXACTIVE_MAX 4096 |
21 | 22 | ||
23 | static int trace_kprobe_create(int argc, const char **argv); | ||
24 | static int trace_kprobe_show(struct seq_file *m, struct dyn_event *ev); | ||
25 | static int trace_kprobe_release(struct dyn_event *ev); | ||
26 | static bool trace_kprobe_is_busy(struct dyn_event *ev); | ||
27 | static bool trace_kprobe_match(const char *system, const char *event, | ||
28 | struct dyn_event *ev); | ||
29 | |||
30 | static struct dyn_event_operations trace_kprobe_ops = { | ||
31 | .create = trace_kprobe_create, | ||
32 | .show = trace_kprobe_show, | ||
33 | .is_busy = trace_kprobe_is_busy, | ||
34 | .free = trace_kprobe_release, | ||
35 | .match = trace_kprobe_match, | ||
36 | }; | ||
37 | |||
22 | /** | 38 | /** |
23 | * Kprobe event core functions | 39 | * Kprobe event core functions |
24 | */ | 40 | */ |
25 | struct trace_kprobe { | 41 | struct trace_kprobe { |
26 | struct list_head list; | 42 | struct dyn_event devent; |
27 | struct kretprobe rp; /* Use rp.kp for kprobe use */ | 43 | struct kretprobe rp; /* Use rp.kp for kprobe use */ |
28 | unsigned long __percpu *nhit; | 44 | unsigned long __percpu *nhit; |
29 | const char *symbol; /* symbol name */ | 45 | const char *symbol; /* symbol name */ |
30 | struct trace_probe tp; | 46 | struct trace_probe tp; |
31 | }; | 47 | }; |
32 | 48 | ||
49 | static bool is_trace_kprobe(struct dyn_event *ev) | ||
50 | { | ||
51 | return ev->ops == &trace_kprobe_ops; | ||
52 | } | ||
53 | |||
54 | static struct trace_kprobe *to_trace_kprobe(struct dyn_event *ev) | ||
55 | { | ||
56 | return container_of(ev, struct trace_kprobe, devent); | ||
57 | } | ||
58 | |||
59 | /** | ||
60 | * for_each_trace_kprobe - iterate over the trace_kprobe list | ||
61 | * @pos: the struct trace_kprobe * for each entry | ||
62 | * @dpos: the struct dyn_event * to use as a loop cursor | ||
63 | */ | ||
64 | #define for_each_trace_kprobe(pos, dpos) \ | ||
65 | for_each_dyn_event(dpos) \ | ||
66 | if (is_trace_kprobe(dpos) && (pos = to_trace_kprobe(dpos))) | ||
67 | |||
33 | #define SIZEOF_TRACE_KPROBE(n) \ | 68 | #define SIZEOF_TRACE_KPROBE(n) \ |
34 | (offsetof(struct trace_kprobe, tp.args) + \ | 69 | (offsetof(struct trace_kprobe, tp.args) + \ |
35 | (sizeof(struct probe_arg) * (n))) | 70 | (sizeof(struct probe_arg) * (n))) |
@@ -81,6 +116,22 @@ static nokprobe_inline bool trace_kprobe_module_exist(struct trace_kprobe *tk) | |||
81 | return ret; | 116 | return ret; |
82 | } | 117 | } |
83 | 118 | ||
119 | static bool trace_kprobe_is_busy(struct dyn_event *ev) | ||
120 | { | ||
121 | struct trace_kprobe *tk = to_trace_kprobe(ev); | ||
122 | |||
123 | return trace_probe_is_enabled(&tk->tp); | ||
124 | } | ||
125 | |||
126 | static bool trace_kprobe_match(const char *system, const char *event, | ||
127 | struct dyn_event *ev) | ||
128 | { | ||
129 | struct trace_kprobe *tk = to_trace_kprobe(ev); | ||
130 | |||
131 | return strcmp(trace_event_name(&tk->tp.call), event) == 0 && | ||
132 | (!system || strcmp(tk->tp.call.class->system, system) == 0); | ||
133 | } | ||
134 | |||
84 | static nokprobe_inline unsigned long trace_kprobe_nhit(struct trace_kprobe *tk) | 135 | static nokprobe_inline unsigned long trace_kprobe_nhit(struct trace_kprobe *tk) |
85 | { | 136 | { |
86 | unsigned long nhit = 0; | 137 | unsigned long nhit = 0; |
@@ -128,9 +179,6 @@ bool trace_kprobe_error_injectable(struct trace_event_call *call) | |||
128 | static int register_kprobe_event(struct trace_kprobe *tk); | 179 | static int register_kprobe_event(struct trace_kprobe *tk); |
129 | static int unregister_kprobe_event(struct trace_kprobe *tk); | 180 | static int unregister_kprobe_event(struct trace_kprobe *tk); |
130 | 181 | ||
131 | static DEFINE_MUTEX(probe_lock); | ||
132 | static LIST_HEAD(probe_list); | ||
133 | |||
134 | static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs); | 182 | static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs); |
135 | static int kretprobe_dispatcher(struct kretprobe_instance *ri, | 183 | static int kretprobe_dispatcher(struct kretprobe_instance *ri, |
136 | struct pt_regs *regs); | 184 | struct pt_regs *regs); |
@@ -192,7 +240,7 @@ static struct trace_kprobe *alloc_trace_kprobe(const char *group, | |||
192 | if (!tk->tp.class.system) | 240 | if (!tk->tp.class.system) |
193 | goto error; | 241 | goto error; |
194 | 242 | ||
195 | INIT_LIST_HEAD(&tk->list); | 243 | dyn_event_init(&tk->devent, &trace_kprobe_ops); |
196 | INIT_LIST_HEAD(&tk->tp.files); | 244 | INIT_LIST_HEAD(&tk->tp.files); |
197 | return tk; | 245 | return tk; |
198 | error: | 246 | error: |
@@ -207,6 +255,9 @@ static void free_trace_kprobe(struct trace_kprobe *tk) | |||
207 | { | 255 | { |
208 | int i; | 256 | int i; |
209 | 257 | ||
258 | if (!tk) | ||
259 | return; | ||
260 | |||
210 | for (i = 0; i < tk->tp.nr_args; i++) | 261 | for (i = 0; i < tk->tp.nr_args; i++) |
211 | traceprobe_free_probe_arg(&tk->tp.args[i]); | 262 | traceprobe_free_probe_arg(&tk->tp.args[i]); |
212 | 263 | ||
@@ -220,9 +271,10 @@ static void free_trace_kprobe(struct trace_kprobe *tk) | |||
220 | static struct trace_kprobe *find_trace_kprobe(const char *event, | 271 | static struct trace_kprobe *find_trace_kprobe(const char *event, |
221 | const char *group) | 272 | const char *group) |
222 | { | 273 | { |
274 | struct dyn_event *pos; | ||
223 | struct trace_kprobe *tk; | 275 | struct trace_kprobe *tk; |
224 | 276 | ||
225 | list_for_each_entry(tk, &probe_list, list) | 277 | for_each_trace_kprobe(tk, pos) |
226 | if (strcmp(trace_event_name(&tk->tp.call), event) == 0 && | 278 | if (strcmp(trace_event_name(&tk->tp.call), event) == 0 && |
227 | strcmp(tk->tp.call.class->system, group) == 0) | 279 | strcmp(tk->tp.call.class->system, group) == 0) |
228 | return tk; | 280 | return tk; |
@@ -321,7 +373,7 @@ disable_trace_kprobe(struct trace_kprobe *tk, struct trace_event_file *file) | |||
321 | * created with perf_event_open. We don't need to wait for these | 373 | * created with perf_event_open. We don't need to wait for these |
322 | * trace_kprobes | 374 | * trace_kprobes |
323 | */ | 375 | */ |
324 | if (list_empty(&tk->list)) | 376 | if (list_empty(&tk->devent.list)) |
325 | wait = 0; | 377 | wait = 0; |
326 | out: | 378 | out: |
327 | if (wait) { | 379 | if (wait) { |
@@ -419,7 +471,7 @@ static void __unregister_trace_kprobe(struct trace_kprobe *tk) | |||
419 | } | 471 | } |
420 | } | 472 | } |
421 | 473 | ||
422 | /* Unregister a trace_probe and probe_event: call with locking probe_lock */ | 474 | /* Unregister a trace_probe and probe_event */ |
423 | static int unregister_trace_kprobe(struct trace_kprobe *tk) | 475 | static int unregister_trace_kprobe(struct trace_kprobe *tk) |
424 | { | 476 | { |
425 | /* Enabled event can not be unregistered */ | 477 | /* Enabled event can not be unregistered */ |
@@ -431,7 +483,7 @@ static int unregister_trace_kprobe(struct trace_kprobe *tk) | |||
431 | return -EBUSY; | 483 | return -EBUSY; |
432 | 484 | ||
433 | __unregister_trace_kprobe(tk); | 485 | __unregister_trace_kprobe(tk); |
434 | list_del(&tk->list); | 486 | dyn_event_remove(&tk->devent); |
435 | 487 | ||
436 | return 0; | 488 | return 0; |
437 | } | 489 | } |
@@ -442,7 +494,7 @@ static int register_trace_kprobe(struct trace_kprobe *tk) | |||
442 | struct trace_kprobe *old_tk; | 494 | struct trace_kprobe *old_tk; |
443 | int ret; | 495 | int ret; |
444 | 496 | ||
445 | mutex_lock(&probe_lock); | 497 | mutex_lock(&event_mutex); |
446 | 498 | ||
447 | /* Delete old (same name) event if exist */ | 499 | /* Delete old (same name) event if exist */ |
448 | old_tk = find_trace_kprobe(trace_event_name(&tk->tp.call), | 500 | old_tk = find_trace_kprobe(trace_event_name(&tk->tp.call), |
@@ -471,10 +523,10 @@ static int register_trace_kprobe(struct trace_kprobe *tk) | |||
471 | if (ret < 0) | 523 | if (ret < 0) |
472 | unregister_kprobe_event(tk); | 524 | unregister_kprobe_event(tk); |
473 | else | 525 | else |
474 | list_add_tail(&tk->list, &probe_list); | 526 | dyn_event_add(&tk->devent); |
475 | 527 | ||
476 | end: | 528 | end: |
477 | mutex_unlock(&probe_lock); | 529 | mutex_unlock(&event_mutex); |
478 | return ret; | 530 | return ret; |
479 | } | 531 | } |
480 | 532 | ||
@@ -483,6 +535,7 @@ static int trace_kprobe_module_callback(struct notifier_block *nb, | |||
483 | unsigned long val, void *data) | 535 | unsigned long val, void *data) |
484 | { | 536 | { |
485 | struct module *mod = data; | 537 | struct module *mod = data; |
538 | struct dyn_event *pos; | ||
486 | struct trace_kprobe *tk; | 539 | struct trace_kprobe *tk; |
487 | int ret; | 540 | int ret; |
488 | 541 | ||
@@ -490,8 +543,8 @@ static int trace_kprobe_module_callback(struct notifier_block *nb, | |||
490 | return NOTIFY_DONE; | 543 | return NOTIFY_DONE; |
491 | 544 | ||
492 | /* Update probes on coming module */ | 545 | /* Update probes on coming module */ |
493 | mutex_lock(&probe_lock); | 546 | mutex_lock(&event_mutex); |
494 | list_for_each_entry(tk, &probe_list, list) { | 547 | for_each_trace_kprobe(tk, pos) { |
495 | if (trace_kprobe_within_module(tk, mod)) { | 548 | if (trace_kprobe_within_module(tk, mod)) { |
496 | /* Don't need to check busy - this should have gone. */ | 549 | /* Don't need to check busy - this should have gone. */ |
497 | __unregister_trace_kprobe(tk); | 550 | __unregister_trace_kprobe(tk); |
@@ -502,7 +555,7 @@ static int trace_kprobe_module_callback(struct notifier_block *nb, | |||
502 | mod->name, ret); | 555 | mod->name, ret); |
503 | } | 556 | } |
504 | } | 557 | } |
505 | mutex_unlock(&probe_lock); | 558 | mutex_unlock(&event_mutex); |
506 | 559 | ||
507 | return NOTIFY_DONE; | 560 | return NOTIFY_DONE; |
508 | } | 561 | } |
@@ -520,7 +573,7 @@ static inline void sanitize_event_name(char *name) | |||
520 | *name = '_'; | 573 | *name = '_'; |
521 | } | 574 | } |
522 | 575 | ||
523 | static int create_trace_kprobe(int argc, char **argv) | 576 | static int trace_kprobe_create(int argc, const char *argv[]) |
524 | { | 577 | { |
525 | /* | 578 | /* |
526 | * Argument syntax: | 579 | * Argument syntax: |
@@ -544,37 +597,37 @@ static int create_trace_kprobe(int argc, char **argv) | |||
544 | * FETCHARG:TYPE : use TYPE instead of unsigned long. | 597 | * FETCHARG:TYPE : use TYPE instead of unsigned long. |
545 | */ | 598 | */ |
546 | struct trace_kprobe *tk; | 599 | struct trace_kprobe *tk; |
547 | int i, ret = 0; | 600 | int i, len, ret = 0; |
548 | bool is_return = false, is_delete = false; | 601 | bool is_return = false; |
549 | char *symbol = NULL, *event = NULL, *group = NULL; | 602 | char *symbol = NULL, *tmp = NULL; |
603 | const char *event = NULL, *group = KPROBE_EVENT_SYSTEM; | ||
550 | int maxactive = 0; | 604 | int maxactive = 0; |
551 | char *arg; | ||
552 | long offset = 0; | 605 | long offset = 0; |
553 | void *addr = NULL; | 606 | void *addr = NULL; |
554 | char buf[MAX_EVENT_NAME_LEN]; | 607 | char buf[MAX_EVENT_NAME_LEN]; |
555 | unsigned int flags = TPARG_FL_KERNEL; | 608 | unsigned int flags = TPARG_FL_KERNEL; |
556 | 609 | ||
557 | /* argc must be >= 1 */ | 610 | /* argc must be >= 1 */ |
558 | if (argv[0][0] == 'p') | 611 | if (argv[0][0] == 'r') { |
559 | is_return = false; | ||
560 | else if (argv[0][0] == 'r') { | ||
561 | is_return = true; | 612 | is_return = true; |
562 | flags |= TPARG_FL_RETURN; | 613 | flags |= TPARG_FL_RETURN; |
563 | } else if (argv[0][0] == '-') | 614 | } else if (argv[0][0] != 'p' || argc < 2) |
564 | is_delete = true; | 615 | return -ECANCELED; |
565 | else { | ||
566 | pr_info("Probe definition must be started with 'p', 'r' or" | ||
567 | " '-'.\n"); | ||
568 | return -EINVAL; | ||
569 | } | ||
570 | 616 | ||
571 | event = strchr(&argv[0][1], ':'); | 617 | event = strchr(&argv[0][1], ':'); |
572 | if (event) { | 618 | if (event) |
573 | event[0] = '\0'; | ||
574 | event++; | 619 | event++; |
575 | } | 620 | |
576 | if (is_return && isdigit(argv[0][1])) { | 621 | if (is_return && isdigit(argv[0][1])) { |
577 | ret = kstrtouint(&argv[0][1], 0, &maxactive); | 622 | if (event) |
623 | len = event - &argv[0][1] - 1; | ||
624 | else | ||
625 | len = strlen(&argv[0][1]); | ||
626 | if (len > MAX_EVENT_NAME_LEN - 1) | ||
627 | return -E2BIG; | ||
628 | memcpy(buf, &argv[0][1], len); | ||
629 | buf[len] = '\0'; | ||
630 | ret = kstrtouint(buf, 0, &maxactive); | ||
578 | if (ret) { | 631 | if (ret) { |
579 | pr_info("Failed to parse maxactive.\n"); | 632 | pr_info("Failed to parse maxactive.\n"); |
580 | return ret; | 633 | return ret; |
@@ -589,74 +642,37 @@ static int create_trace_kprobe(int argc, char **argv) | |||
589 | } | 642 | } |
590 | } | 643 | } |
591 | 644 | ||
592 | if (event) { | ||
593 | char *slash; | ||
594 | |||
595 | slash = strchr(event, '/'); | ||
596 | if (slash) { | ||
597 | group = event; | ||
598 | event = slash + 1; | ||
599 | slash[0] = '\0'; | ||
600 | if (strlen(group) == 0) { | ||
601 | pr_info("Group name is not specified\n"); | ||
602 | return -EINVAL; | ||
603 | } | ||
604 | } | ||
605 | if (strlen(event) == 0) { | ||
606 | pr_info("Event name is not specified\n"); | ||
607 | return -EINVAL; | ||
608 | } | ||
609 | } | ||
610 | if (!group) | ||
611 | group = KPROBE_EVENT_SYSTEM; | ||
612 | |||
613 | if (is_delete) { | ||
614 | if (!event) { | ||
615 | pr_info("Delete command needs an event name.\n"); | ||
616 | return -EINVAL; | ||
617 | } | ||
618 | mutex_lock(&probe_lock); | ||
619 | tk = find_trace_kprobe(event, group); | ||
620 | if (!tk) { | ||
621 | mutex_unlock(&probe_lock); | ||
622 | pr_info("Event %s/%s doesn't exist.\n", group, event); | ||
623 | return -ENOENT; | ||
624 | } | ||
625 | /* delete an event */ | ||
626 | ret = unregister_trace_kprobe(tk); | ||
627 | if (ret == 0) | ||
628 | free_trace_kprobe(tk); | ||
629 | mutex_unlock(&probe_lock); | ||
630 | return ret; | ||
631 | } | ||
632 | |||
633 | if (argc < 2) { | ||
634 | pr_info("Probe point is not specified.\n"); | ||
635 | return -EINVAL; | ||
636 | } | ||
637 | |||
638 | /* try to parse an address. if that fails, try to read the | 645 | /* try to parse an address. if that fails, try to read the |
639 | * input as a symbol. */ | 646 | * input as a symbol. */ |
640 | if (kstrtoul(argv[1], 0, (unsigned long *)&addr)) { | 647 | if (kstrtoul(argv[1], 0, (unsigned long *)&addr)) { |
648 | /* Check whether uprobe event specified */ | ||
649 | if (strchr(argv[1], '/') && strchr(argv[1], ':')) | ||
650 | return -ECANCELED; | ||
641 | /* a symbol specified */ | 651 | /* a symbol specified */ |
642 | symbol = argv[1]; | 652 | symbol = kstrdup(argv[1], GFP_KERNEL); |
653 | if (!symbol) | ||
654 | return -ENOMEM; | ||
643 | /* TODO: support .init module functions */ | 655 | /* TODO: support .init module functions */ |
644 | ret = traceprobe_split_symbol_offset(symbol, &offset); | 656 | ret = traceprobe_split_symbol_offset(symbol, &offset); |
645 | if (ret || offset < 0 || offset > UINT_MAX) { | 657 | if (ret || offset < 0 || offset > UINT_MAX) { |
646 | pr_info("Failed to parse either an address or a symbol.\n"); | 658 | pr_info("Failed to parse either an address or a symbol.\n"); |
647 | return ret; | 659 | goto out; |
648 | } | 660 | } |
649 | if (kprobe_on_func_entry(NULL, symbol, offset)) | 661 | if (kprobe_on_func_entry(NULL, symbol, offset)) |
650 | flags |= TPARG_FL_FENTRY; | 662 | flags |= TPARG_FL_FENTRY; |
651 | if (offset && is_return && !(flags & TPARG_FL_FENTRY)) { | 663 | if (offset && is_return && !(flags & TPARG_FL_FENTRY)) { |
652 | pr_info("Given offset is not valid for return probe.\n"); | 664 | pr_info("Given offset is not valid for return probe.\n"); |
653 | return -EINVAL; | 665 | ret = -EINVAL; |
666 | goto out; | ||
654 | } | 667 | } |
655 | } | 668 | } |
656 | argc -= 2; argv += 2; | 669 | argc -= 2; argv += 2; |
657 | 670 | ||
658 | /* setup a probe */ | 671 | if (event) { |
659 | if (!event) { | 672 | ret = traceprobe_parse_event_name(&event, &group, buf); |
673 | if (ret) | ||
674 | goto out; | ||
675 | } else { | ||
660 | /* Make a new event name */ | 676 | /* Make a new event name */ |
661 | if (symbol) | 677 | if (symbol) |
662 | snprintf(buf, MAX_EVENT_NAME_LEN, "%c_%s_%ld", | 678 | snprintf(buf, MAX_EVENT_NAME_LEN, "%c_%s_%ld", |
@@ -667,121 +683,67 @@ static int create_trace_kprobe(int argc, char **argv) | |||
667 | sanitize_event_name(buf); | 683 | sanitize_event_name(buf); |
668 | event = buf; | 684 | event = buf; |
669 | } | 685 | } |
686 | |||
687 | /* setup a probe */ | ||
670 | tk = alloc_trace_kprobe(group, event, addr, symbol, offset, maxactive, | 688 | tk = alloc_trace_kprobe(group, event, addr, symbol, offset, maxactive, |
671 | argc, is_return); | 689 | argc, is_return); |
672 | if (IS_ERR(tk)) { | 690 | if (IS_ERR(tk)) { |
673 | pr_info("Failed to allocate trace_probe.(%d)\n", | 691 | pr_info("Failed to allocate trace_probe.(%d)\n", |
674 | (int)PTR_ERR(tk)); | 692 | (int)PTR_ERR(tk)); |
675 | return PTR_ERR(tk); | 693 | ret = PTR_ERR(tk); |
694 | goto out; | ||
676 | } | 695 | } |
677 | 696 | ||
678 | /* parse arguments */ | 697 | /* parse arguments */ |
679 | ret = 0; | ||
680 | for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) { | 698 | for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) { |
681 | struct probe_arg *parg = &tk->tp.args[i]; | 699 | tmp = kstrdup(argv[i], GFP_KERNEL); |
682 | 700 | if (!tmp) { | |
683 | /* Increment count for freeing args in error case */ | ||
684 | tk->tp.nr_args++; | ||
685 | |||
686 | /* Parse argument name */ | ||
687 | arg = strchr(argv[i], '='); | ||
688 | if (arg) { | ||
689 | *arg++ = '\0'; | ||
690 | parg->name = kstrdup(argv[i], GFP_KERNEL); | ||
691 | } else { | ||
692 | arg = argv[i]; | ||
693 | /* If argument name is omitted, set "argN" */ | ||
694 | snprintf(buf, MAX_EVENT_NAME_LEN, "arg%d", i + 1); | ||
695 | parg->name = kstrdup(buf, GFP_KERNEL); | ||
696 | } | ||
697 | |||
698 | if (!parg->name) { | ||
699 | pr_info("Failed to allocate argument[%d] name.\n", i); | ||
700 | ret = -ENOMEM; | 701 | ret = -ENOMEM; |
701 | goto error; | 702 | goto error; |
702 | } | 703 | } |
703 | 704 | ||
704 | if (!is_good_name(parg->name)) { | 705 | ret = traceprobe_parse_probe_arg(&tk->tp, i, tmp, flags); |
705 | pr_info("Invalid argument[%d] name: %s\n", | 706 | kfree(tmp); |
706 | i, parg->name); | 707 | if (ret) |
707 | ret = -EINVAL; | ||
708 | goto error; | ||
709 | } | ||
710 | |||
711 | if (traceprobe_conflict_field_name(parg->name, | ||
712 | tk->tp.args, i)) { | ||
713 | pr_info("Argument[%d] name '%s' conflicts with " | ||
714 | "another field.\n", i, argv[i]); | ||
715 | ret = -EINVAL; | ||
716 | goto error; | ||
717 | } | ||
718 | |||
719 | /* Parse fetch argument */ | ||
720 | ret = traceprobe_parse_probe_arg(arg, &tk->tp.size, parg, | ||
721 | flags); | ||
722 | if (ret) { | ||
723 | pr_info("Parse error at argument[%d]. (%d)\n", i, ret); | ||
724 | goto error; | 708 | goto error; |
725 | } | ||
726 | } | 709 | } |
727 | 710 | ||
728 | ret = register_trace_kprobe(tk); | 711 | ret = register_trace_kprobe(tk); |
729 | if (ret) | 712 | if (ret) |
730 | goto error; | 713 | goto error; |
731 | return 0; | 714 | out: |
715 | kfree(symbol); | ||
716 | return ret; | ||
732 | 717 | ||
733 | error: | 718 | error: |
734 | free_trace_kprobe(tk); | 719 | free_trace_kprobe(tk); |
735 | return ret; | 720 | goto out; |
736 | } | 721 | } |
737 | 722 | ||
738 | static int release_all_trace_kprobes(void) | 723 | static int create_or_delete_trace_kprobe(int argc, char **argv) |
739 | { | 724 | { |
740 | struct trace_kprobe *tk; | 725 | int ret; |
741 | int ret = 0; | ||
742 | |||
743 | mutex_lock(&probe_lock); | ||
744 | /* Ensure no probe is in use. */ | ||
745 | list_for_each_entry(tk, &probe_list, list) | ||
746 | if (trace_probe_is_enabled(&tk->tp)) { | ||
747 | ret = -EBUSY; | ||
748 | goto end; | ||
749 | } | ||
750 | /* TODO: Use batch unregistration */ | ||
751 | while (!list_empty(&probe_list)) { | ||
752 | tk = list_entry(probe_list.next, struct trace_kprobe, list); | ||
753 | ret = unregister_trace_kprobe(tk); | ||
754 | if (ret) | ||
755 | goto end; | ||
756 | free_trace_kprobe(tk); | ||
757 | } | ||
758 | |||
759 | end: | ||
760 | mutex_unlock(&probe_lock); | ||
761 | 726 | ||
762 | return ret; | 727 | if (argv[0][0] == '-') |
763 | } | 728 | return dyn_event_release(argc, argv, &trace_kprobe_ops); |
764 | 729 | ||
765 | /* Probes listing interfaces */ | 730 | ret = trace_kprobe_create(argc, (const char **)argv); |
766 | static void *probes_seq_start(struct seq_file *m, loff_t *pos) | 731 | return ret == -ECANCELED ? -EINVAL : ret; |
767 | { | ||
768 | mutex_lock(&probe_lock); | ||
769 | return seq_list_start(&probe_list, *pos); | ||
770 | } | 732 | } |
771 | 733 | ||
772 | static void *probes_seq_next(struct seq_file *m, void *v, loff_t *pos) | 734 | static int trace_kprobe_release(struct dyn_event *ev) |
773 | { | 735 | { |
774 | return seq_list_next(v, &probe_list, pos); | 736 | struct trace_kprobe *tk = to_trace_kprobe(ev); |
775 | } | 737 | int ret = unregister_trace_kprobe(tk); |
776 | 738 | ||
777 | static void probes_seq_stop(struct seq_file *m, void *v) | 739 | if (!ret) |
778 | { | 740 | free_trace_kprobe(tk); |
779 | mutex_unlock(&probe_lock); | 741 | return ret; |
780 | } | 742 | } |
781 | 743 | ||
782 | static int probes_seq_show(struct seq_file *m, void *v) | 744 | static int trace_kprobe_show(struct seq_file *m, struct dyn_event *ev) |
783 | { | 745 | { |
784 | struct trace_kprobe *tk = v; | 746 | struct trace_kprobe *tk = to_trace_kprobe(ev); |
785 | int i; | 747 | int i; |
786 | 748 | ||
787 | seq_putc(m, trace_kprobe_is_return(tk) ? 'r' : 'p'); | 749 | seq_putc(m, trace_kprobe_is_return(tk) ? 'r' : 'p'); |
@@ -803,10 +765,20 @@ static int probes_seq_show(struct seq_file *m, void *v) | |||
803 | return 0; | 765 | return 0; |
804 | } | 766 | } |
805 | 767 | ||
768 | static int probes_seq_show(struct seq_file *m, void *v) | ||
769 | { | ||
770 | struct dyn_event *ev = v; | ||
771 | |||
772 | if (!is_trace_kprobe(ev)) | ||
773 | return 0; | ||
774 | |||
775 | return trace_kprobe_show(m, ev); | ||
776 | } | ||
777 | |||
806 | static const struct seq_operations probes_seq_op = { | 778 | static const struct seq_operations probes_seq_op = { |
807 | .start = probes_seq_start, | 779 | .start = dyn_event_seq_start, |
808 | .next = probes_seq_next, | 780 | .next = dyn_event_seq_next, |
809 | .stop = probes_seq_stop, | 781 | .stop = dyn_event_seq_stop, |
810 | .show = probes_seq_show | 782 | .show = probes_seq_show |
811 | }; | 783 | }; |
812 | 784 | ||
@@ -815,7 +787,7 @@ static int probes_open(struct inode *inode, struct file *file) | |||
815 | int ret; | 787 | int ret; |
816 | 788 | ||
817 | if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) { | 789 | if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) { |
818 | ret = release_all_trace_kprobes(); | 790 | ret = dyn_events_release_all(&trace_kprobe_ops); |
819 | if (ret < 0) | 791 | if (ret < 0) |
820 | return ret; | 792 | return ret; |
821 | } | 793 | } |
@@ -827,7 +799,7 @@ static ssize_t probes_write(struct file *file, const char __user *buffer, | |||
827 | size_t count, loff_t *ppos) | 799 | size_t count, loff_t *ppos) |
828 | { | 800 | { |
829 | return trace_parse_run_command(file, buffer, count, ppos, | 801 | return trace_parse_run_command(file, buffer, count, ppos, |
830 | create_trace_kprobe); | 802 | create_or_delete_trace_kprobe); |
831 | } | 803 | } |
832 | 804 | ||
833 | static const struct file_operations kprobe_events_ops = { | 805 | static const struct file_operations kprobe_events_ops = { |
@@ -842,8 +814,13 @@ static const struct file_operations kprobe_events_ops = { | |||
842 | /* Probes profiling interfaces */ | 814 | /* Probes profiling interfaces */ |
843 | static int probes_profile_seq_show(struct seq_file *m, void *v) | 815 | static int probes_profile_seq_show(struct seq_file *m, void *v) |
844 | { | 816 | { |
845 | struct trace_kprobe *tk = v; | 817 | struct dyn_event *ev = v; |
818 | struct trace_kprobe *tk; | ||
819 | |||
820 | if (!is_trace_kprobe(ev)) | ||
821 | return 0; | ||
846 | 822 | ||
823 | tk = to_trace_kprobe(ev); | ||
847 | seq_printf(m, " %-44s %15lu %15lu\n", | 824 | seq_printf(m, " %-44s %15lu %15lu\n", |
848 | trace_event_name(&tk->tp.call), | 825 | trace_event_name(&tk->tp.call), |
849 | trace_kprobe_nhit(tk), | 826 | trace_kprobe_nhit(tk), |
@@ -853,9 +830,9 @@ static int probes_profile_seq_show(struct seq_file *m, void *v) | |||
853 | } | 830 | } |
854 | 831 | ||
855 | static const struct seq_operations profile_seq_op = { | 832 | static const struct seq_operations profile_seq_op = { |
856 | .start = probes_seq_start, | 833 | .start = dyn_event_seq_start, |
857 | .next = probes_seq_next, | 834 | .next = dyn_event_seq_next, |
858 | .stop = probes_seq_stop, | 835 | .stop = dyn_event_seq_stop, |
859 | .show = probes_profile_seq_show | 836 | .show = probes_profile_seq_show |
860 | }; | 837 | }; |
861 | 838 | ||
@@ -1408,7 +1385,7 @@ create_local_trace_kprobe(char *func, void *addr, unsigned long offs, | |||
1408 | char *event; | 1385 | char *event; |
1409 | 1386 | ||
1410 | /* | 1387 | /* |
1411 | * local trace_kprobes are not added to probe_list, so they are never | 1388 | * local trace_kprobes are not added to dyn_event, so they are never |
1412 | * searched in find_trace_kprobe(). Therefore, there is no concern of | 1389 | * searched in find_trace_kprobe(). Therefore, there is no concern of |
1413 | * duplicated name here. | 1390 | * duplicated name here. |
1414 | */ | 1391 | */ |
@@ -1466,6 +1443,11 @@ static __init int init_kprobe_trace(void) | |||
1466 | { | 1443 | { |
1467 | struct dentry *d_tracer; | 1444 | struct dentry *d_tracer; |
1468 | struct dentry *entry; | 1445 | struct dentry *entry; |
1446 | int ret; | ||
1447 | |||
1448 | ret = dyn_event_register(&trace_kprobe_ops); | ||
1449 | if (ret) | ||
1450 | return ret; | ||
1469 | 1451 | ||
1470 | if (register_module_notifier(&trace_kprobe_module_nb)) | 1452 | if (register_module_notifier(&trace_kprobe_module_nb)) |
1471 | return -EINVAL; | 1453 | return -EINVAL; |
@@ -1523,9 +1505,8 @@ static __init int kprobe_trace_self_tests_init(void) | |||
1523 | 1505 | ||
1524 | pr_info("Testing kprobe tracing: "); | 1506 | pr_info("Testing kprobe tracing: "); |
1525 | 1507 | ||
1526 | ret = trace_run_command("p:testprobe kprobe_trace_selftest_target " | 1508 | ret = trace_run_command("p:testprobe kprobe_trace_selftest_target $stack $stack0 +0($stack)", |
1527 | "$stack $stack0 +0($stack)", | 1509 | create_or_delete_trace_kprobe); |
1528 | create_trace_kprobe); | ||
1529 | if (WARN_ON_ONCE(ret)) { | 1510 | if (WARN_ON_ONCE(ret)) { |
1530 | pr_warn("error on probing function entry.\n"); | 1511 | pr_warn("error on probing function entry.\n"); |
1531 | warn++; | 1512 | warn++; |
@@ -1545,8 +1526,8 @@ static __init int kprobe_trace_self_tests_init(void) | |||
1545 | } | 1526 | } |
1546 | } | 1527 | } |
1547 | 1528 | ||
1548 | ret = trace_run_command("r:testprobe2 kprobe_trace_selftest_target " | 1529 | ret = trace_run_command("r:testprobe2 kprobe_trace_selftest_target $retval", |
1549 | "$retval", create_trace_kprobe); | 1530 | create_or_delete_trace_kprobe); |
1550 | if (WARN_ON_ONCE(ret)) { | 1531 | if (WARN_ON_ONCE(ret)) { |
1551 | pr_warn("error on probing function return.\n"); | 1532 | pr_warn("error on probing function return.\n"); |
1552 | warn++; | 1533 | warn++; |
@@ -1616,20 +1597,24 @@ static __init int kprobe_trace_self_tests_init(void) | |||
1616 | disable_trace_kprobe(tk, file); | 1597 | disable_trace_kprobe(tk, file); |
1617 | } | 1598 | } |
1618 | 1599 | ||
1619 | ret = trace_run_command("-:testprobe", create_trace_kprobe); | 1600 | ret = trace_run_command("-:testprobe", create_or_delete_trace_kprobe); |
1620 | if (WARN_ON_ONCE(ret)) { | 1601 | if (WARN_ON_ONCE(ret)) { |
1621 | pr_warn("error on deleting a probe.\n"); | 1602 | pr_warn("error on deleting a probe.\n"); |
1622 | warn++; | 1603 | warn++; |
1623 | } | 1604 | } |
1624 | 1605 | ||
1625 | ret = trace_run_command("-:testprobe2", create_trace_kprobe); | 1606 | ret = trace_run_command("-:testprobe2", create_or_delete_trace_kprobe); |
1626 | if (WARN_ON_ONCE(ret)) { | 1607 | if (WARN_ON_ONCE(ret)) { |
1627 | pr_warn("error on deleting a probe.\n"); | 1608 | pr_warn("error on deleting a probe.\n"); |
1628 | warn++; | 1609 | warn++; |
1629 | } | 1610 | } |
1630 | 1611 | ||
1631 | end: | 1612 | end: |
1632 | release_all_trace_kprobes(); | 1613 | ret = dyn_events_release_all(&trace_kprobe_ops); |
1614 | if (WARN_ON_ONCE(ret)) { | ||
1615 | pr_warn("error on cleaning up probes.\n"); | ||
1616 | warn++; | ||
1617 | } | ||
1633 | /* | 1618 | /* |
1634 | * Wait for the optimizer work to finish. Otherwise it might fiddle | 1619 | * Wait for the optimizer work to finish. Otherwise it might fiddle |
1635 | * with probes in already freed __init text. | 1620 | * with probes in already freed __init text. |
diff --git a/kernel/trace/trace_output.c b/kernel/trace/trace_output.c index 6e6cc64faa38..54373d93e251 100644 --- a/kernel/trace/trace_output.c +++ b/kernel/trace/trace_output.c | |||
@@ -339,43 +339,24 @@ static inline const char *kretprobed(const char *name) | |||
339 | #endif /* CONFIG_KRETPROBES */ | 339 | #endif /* CONFIG_KRETPROBES */ |
340 | 340 | ||
341 | static void | 341 | static void |
342 | seq_print_sym_short(struct trace_seq *s, const char *fmt, unsigned long address) | 342 | seq_print_sym(struct trace_seq *s, unsigned long address, bool offset) |
343 | { | 343 | { |
344 | char str[KSYM_SYMBOL_LEN]; | ||
345 | #ifdef CONFIG_KALLSYMS | 344 | #ifdef CONFIG_KALLSYMS |
346 | const char *name; | ||
347 | |||
348 | kallsyms_lookup(address, NULL, NULL, NULL, str); | ||
349 | |||
350 | name = kretprobed(str); | ||
351 | |||
352 | if (name && strlen(name)) { | ||
353 | trace_seq_printf(s, fmt, name); | ||
354 | return; | ||
355 | } | ||
356 | #endif | ||
357 | snprintf(str, KSYM_SYMBOL_LEN, "0x%08lx", address); | ||
358 | trace_seq_printf(s, fmt, str); | ||
359 | } | ||
360 | |||
361 | static void | ||
362 | seq_print_sym_offset(struct trace_seq *s, const char *fmt, | ||
363 | unsigned long address) | ||
364 | { | ||
365 | char str[KSYM_SYMBOL_LEN]; | 345 | char str[KSYM_SYMBOL_LEN]; |
366 | #ifdef CONFIG_KALLSYMS | ||
367 | const char *name; | 346 | const char *name; |
368 | 347 | ||
369 | sprint_symbol(str, address); | 348 | if (offset) |
349 | sprint_symbol(str, address); | ||
350 | else | ||
351 | kallsyms_lookup(address, NULL, NULL, NULL, str); | ||
370 | name = kretprobed(str); | 352 | name = kretprobed(str); |
371 | 353 | ||
372 | if (name && strlen(name)) { | 354 | if (name && strlen(name)) { |
373 | trace_seq_printf(s, fmt, name); | 355 | trace_seq_puts(s, name); |
374 | return; | 356 | return; |
375 | } | 357 | } |
376 | #endif | 358 | #endif |
377 | snprintf(str, KSYM_SYMBOL_LEN, "0x%08lx", address); | 359 | trace_seq_printf(s, "0x%08lx", address); |
378 | trace_seq_printf(s, fmt, str); | ||
379 | } | 360 | } |
380 | 361 | ||
381 | #ifndef CONFIG_64BIT | 362 | #ifndef CONFIG_64BIT |
@@ -424,10 +405,7 @@ seq_print_ip_sym(struct trace_seq *s, unsigned long ip, unsigned long sym_flags) | |||
424 | goto out; | 405 | goto out; |
425 | } | 406 | } |
426 | 407 | ||
427 | if (sym_flags & TRACE_ITER_SYM_OFFSET) | 408 | seq_print_sym(s, ip, sym_flags & TRACE_ITER_SYM_OFFSET); |
428 | seq_print_sym_offset(s, "%s", ip); | ||
429 | else | ||
430 | seq_print_sym_short(s, "%s", ip); | ||
431 | 409 | ||
432 | if (sym_flags & TRACE_ITER_SYM_ADDR) | 410 | if (sym_flags & TRACE_ITER_SYM_ADDR) |
433 | trace_seq_printf(s, " <" IP_FMT ">", ip); | 411 | trace_seq_printf(s, " <" IP_FMT ">", ip); |
diff --git a/kernel/trace/trace_probe.c b/kernel/trace/trace_probe.c index bd30e9398d2a..9962cb5da8ac 100644 --- a/kernel/trace/trace_probe.c +++ b/kernel/trace/trace_probe.c | |||
@@ -154,24 +154,52 @@ int traceprobe_split_symbol_offset(char *symbol, long *offset) | |||
154 | return 0; | 154 | return 0; |
155 | } | 155 | } |
156 | 156 | ||
157 | /* @buf must has MAX_EVENT_NAME_LEN size */ | ||
158 | int traceprobe_parse_event_name(const char **pevent, const char **pgroup, | ||
159 | char *buf) | ||
160 | { | ||
161 | const char *slash, *event = *pevent; | ||
162 | |||
163 | slash = strchr(event, '/'); | ||
164 | if (slash) { | ||
165 | if (slash == event) { | ||
166 | pr_info("Group name is not specified\n"); | ||
167 | return -EINVAL; | ||
168 | } | ||
169 | if (slash - event + 1 > MAX_EVENT_NAME_LEN) { | ||
170 | pr_info("Group name is too long\n"); | ||
171 | return -E2BIG; | ||
172 | } | ||
173 | strlcpy(buf, event, slash - event + 1); | ||
174 | *pgroup = buf; | ||
175 | *pevent = slash + 1; | ||
176 | } | ||
177 | if (strlen(event) == 0) { | ||
178 | pr_info("Event name is not specified\n"); | ||
179 | return -EINVAL; | ||
180 | } | ||
181 | return 0; | ||
182 | } | ||
183 | |||
157 | #define PARAM_MAX_STACK (THREAD_SIZE / sizeof(unsigned long)) | 184 | #define PARAM_MAX_STACK (THREAD_SIZE / sizeof(unsigned long)) |
158 | 185 | ||
159 | static int parse_probe_vars(char *arg, const struct fetch_type *t, | 186 | static int parse_probe_vars(char *arg, const struct fetch_type *t, |
160 | struct fetch_insn *code, unsigned int flags) | 187 | struct fetch_insn *code, unsigned int flags) |
161 | { | 188 | { |
162 | int ret = 0; | ||
163 | unsigned long param; | 189 | unsigned long param; |
190 | int ret = 0; | ||
191 | int len; | ||
164 | 192 | ||
165 | if (strcmp(arg, "retval") == 0) { | 193 | if (strcmp(arg, "retval") == 0) { |
166 | if (flags & TPARG_FL_RETURN) | 194 | if (flags & TPARG_FL_RETURN) |
167 | code->op = FETCH_OP_RETVAL; | 195 | code->op = FETCH_OP_RETVAL; |
168 | else | 196 | else |
169 | ret = -EINVAL; | 197 | ret = -EINVAL; |
170 | } else if (strncmp(arg, "stack", 5) == 0) { | 198 | } else if ((len = str_has_prefix(arg, "stack"))) { |
171 | if (arg[5] == '\0') { | 199 | if (arg[len] == '\0') { |
172 | code->op = FETCH_OP_STACKP; | 200 | code->op = FETCH_OP_STACKP; |
173 | } else if (isdigit(arg[5])) { | 201 | } else if (isdigit(arg[len])) { |
174 | ret = kstrtoul(arg + 5, 10, ¶m); | 202 | ret = kstrtoul(arg + len, 10, ¶m); |
175 | if (ret || ((flags & TPARG_FL_KERNEL) && | 203 | if (ret || ((flags & TPARG_FL_KERNEL) && |
176 | param > PARAM_MAX_STACK)) | 204 | param > PARAM_MAX_STACK)) |
177 | ret = -EINVAL; | 205 | ret = -EINVAL; |
@@ -186,10 +214,10 @@ static int parse_probe_vars(char *arg, const struct fetch_type *t, | |||
186 | #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API | 214 | #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API |
187 | } else if (((flags & TPARG_FL_MASK) == | 215 | } else if (((flags & TPARG_FL_MASK) == |
188 | (TPARG_FL_KERNEL | TPARG_FL_FENTRY)) && | 216 | (TPARG_FL_KERNEL | TPARG_FL_FENTRY)) && |
189 | strncmp(arg, "arg", 3) == 0) { | 217 | (len = str_has_prefix(arg, "arg"))) { |
190 | if (!isdigit(arg[3])) | 218 | if (!isdigit(arg[len])) |
191 | return -EINVAL; | 219 | return -EINVAL; |
192 | ret = kstrtoul(arg + 3, 10, ¶m); | 220 | ret = kstrtoul(arg + len, 10, ¶m); |
193 | if (ret || !param || param > PARAM_MAX_STACK) | 221 | if (ret || !param || param > PARAM_MAX_STACK) |
194 | return -EINVAL; | 222 | return -EINVAL; |
195 | code->op = FETCH_OP_ARG; | 223 | code->op = FETCH_OP_ARG; |
@@ -348,7 +376,7 @@ static int __parse_bitfield_probe_arg(const char *bf, | |||
348 | } | 376 | } |
349 | 377 | ||
350 | /* String length checking wrapper */ | 378 | /* String length checking wrapper */ |
351 | int traceprobe_parse_probe_arg(char *arg, ssize_t *size, | 379 | static int traceprobe_parse_probe_arg_body(char *arg, ssize_t *size, |
352 | struct probe_arg *parg, unsigned int flags) | 380 | struct probe_arg *parg, unsigned int flags) |
353 | { | 381 | { |
354 | struct fetch_insn *code, *scode, *tmp = NULL; | 382 | struct fetch_insn *code, *scode, *tmp = NULL; |
@@ -491,8 +519,8 @@ fail: | |||
491 | } | 519 | } |
492 | 520 | ||
493 | /* Return 1 if name is reserved or already used by another argument */ | 521 | /* Return 1 if name is reserved or already used by another argument */ |
494 | int traceprobe_conflict_field_name(const char *name, | 522 | static int traceprobe_conflict_field_name(const char *name, |
495 | struct probe_arg *args, int narg) | 523 | struct probe_arg *args, int narg) |
496 | { | 524 | { |
497 | int i; | 525 | int i; |
498 | 526 | ||
@@ -507,6 +535,47 @@ int traceprobe_conflict_field_name(const char *name, | |||
507 | return 0; | 535 | return 0; |
508 | } | 536 | } |
509 | 537 | ||
538 | int traceprobe_parse_probe_arg(struct trace_probe *tp, int i, char *arg, | ||
539 | unsigned int flags) | ||
540 | { | ||
541 | struct probe_arg *parg = &tp->args[i]; | ||
542 | char *body; | ||
543 | int ret; | ||
544 | |||
545 | /* Increment count for freeing args in error case */ | ||
546 | tp->nr_args++; | ||
547 | |||
548 | body = strchr(arg, '='); | ||
549 | if (body) { | ||
550 | parg->name = kmemdup_nul(arg, body - arg, GFP_KERNEL); | ||
551 | body++; | ||
552 | } else { | ||
553 | /* If argument name is omitted, set "argN" */ | ||
554 | parg->name = kasprintf(GFP_KERNEL, "arg%d", i + 1); | ||
555 | body = arg; | ||
556 | } | ||
557 | if (!parg->name) | ||
558 | return -ENOMEM; | ||
559 | |||
560 | if (!is_good_name(parg->name)) { | ||
561 | pr_info("Invalid argument[%d] name: %s\n", | ||
562 | i, parg->name); | ||
563 | return -EINVAL; | ||
564 | } | ||
565 | |||
566 | if (traceprobe_conflict_field_name(parg->name, tp->args, i)) { | ||
567 | pr_info("Argument[%d]: '%s' conflicts with another field.\n", | ||
568 | i, parg->name); | ||
569 | return -EINVAL; | ||
570 | } | ||
571 | |||
572 | /* Parse fetch argument */ | ||
573 | ret = traceprobe_parse_probe_arg_body(body, &tp->size, parg, flags); | ||
574 | if (ret) | ||
575 | pr_info("Parse error at argument[%d]. (%d)\n", i, ret); | ||
576 | return ret; | ||
577 | } | ||
578 | |||
510 | void traceprobe_free_probe_arg(struct probe_arg *arg) | 579 | void traceprobe_free_probe_arg(struct probe_arg *arg) |
511 | { | 580 | { |
512 | struct fetch_insn *code = arg->code; | 581 | struct fetch_insn *code = arg->code; |
diff --git a/kernel/trace/trace_probe.h b/kernel/trace/trace_probe.h index 974afc1a3e73..8a63f8bc01bc 100644 --- a/kernel/trace/trace_probe.h +++ b/kernel/trace/trace_probe.h | |||
@@ -272,16 +272,15 @@ find_event_file_link(struct trace_probe *tp, struct trace_event_file *file) | |||
272 | #define TPARG_FL_FENTRY BIT(2) | 272 | #define TPARG_FL_FENTRY BIT(2) |
273 | #define TPARG_FL_MASK GENMASK(2, 0) | 273 | #define TPARG_FL_MASK GENMASK(2, 0) |
274 | 274 | ||
275 | extern int traceprobe_parse_probe_arg(char *arg, ssize_t *size, | 275 | extern int traceprobe_parse_probe_arg(struct trace_probe *tp, int i, |
276 | struct probe_arg *parg, unsigned int flags); | 276 | char *arg, unsigned int flags); |
277 | |||
278 | extern int traceprobe_conflict_field_name(const char *name, | ||
279 | struct probe_arg *args, int narg); | ||
280 | 277 | ||
281 | extern int traceprobe_update_arg(struct probe_arg *arg); | 278 | extern int traceprobe_update_arg(struct probe_arg *arg); |
282 | extern void traceprobe_free_probe_arg(struct probe_arg *arg); | 279 | extern void traceprobe_free_probe_arg(struct probe_arg *arg); |
283 | 280 | ||
284 | extern int traceprobe_split_symbol_offset(char *symbol, long *offset); | 281 | extern int traceprobe_split_symbol_offset(char *symbol, long *offset); |
282 | extern int traceprobe_parse_event_name(const char **pevent, | ||
283 | const char **pgroup, char *buf); | ||
285 | 284 | ||
286 | extern int traceprobe_set_print_fmt(struct trace_probe *tp, bool is_return); | 285 | extern int traceprobe_set_print_fmt(struct trace_probe *tp, bool is_return); |
287 | 286 | ||
diff --git a/kernel/trace/trace_sched_wakeup.c b/kernel/trace/trace_sched_wakeup.c index 7d04b9890755..4ea7e6845efb 100644 --- a/kernel/trace/trace_sched_wakeup.c +++ b/kernel/trace/trace_sched_wakeup.c | |||
@@ -35,26 +35,19 @@ static arch_spinlock_t wakeup_lock = | |||
35 | 35 | ||
36 | static void wakeup_reset(struct trace_array *tr); | 36 | static void wakeup_reset(struct trace_array *tr); |
37 | static void __wakeup_reset(struct trace_array *tr); | 37 | static void __wakeup_reset(struct trace_array *tr); |
38 | static int start_func_tracer(struct trace_array *tr, int graph); | ||
39 | static void stop_func_tracer(struct trace_array *tr, int graph); | ||
38 | 40 | ||
39 | static int save_flags; | 41 | static int save_flags; |
40 | 42 | ||
41 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | 43 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
42 | static int wakeup_display_graph(struct trace_array *tr, int set); | ||
43 | # define is_graph(tr) ((tr)->trace_flags & TRACE_ITER_DISPLAY_GRAPH) | 44 | # define is_graph(tr) ((tr)->trace_flags & TRACE_ITER_DISPLAY_GRAPH) |
44 | #else | 45 | #else |
45 | static inline int wakeup_display_graph(struct trace_array *tr, int set) | ||
46 | { | ||
47 | return 0; | ||
48 | } | ||
49 | # define is_graph(tr) false | 46 | # define is_graph(tr) false |
50 | #endif | 47 | #endif |
51 | 48 | ||
52 | |||
53 | #ifdef CONFIG_FUNCTION_TRACER | 49 | #ifdef CONFIG_FUNCTION_TRACER |
54 | 50 | ||
55 | static int wakeup_graph_entry(struct ftrace_graph_ent *trace); | ||
56 | static void wakeup_graph_return(struct ftrace_graph_ret *trace); | ||
57 | |||
58 | static bool function_enabled; | 51 | static bool function_enabled; |
59 | 52 | ||
60 | /* | 53 | /* |
@@ -104,122 +97,8 @@ out_enable: | |||
104 | return 0; | 97 | return 0; |
105 | } | 98 | } |
106 | 99 | ||
107 | /* | ||
108 | * wakeup uses its own tracer function to keep the overhead down: | ||
109 | */ | ||
110 | static void | ||
111 | wakeup_tracer_call(unsigned long ip, unsigned long parent_ip, | ||
112 | struct ftrace_ops *op, struct pt_regs *pt_regs) | ||
113 | { | ||
114 | struct trace_array *tr = wakeup_trace; | ||
115 | struct trace_array_cpu *data; | ||
116 | unsigned long flags; | ||
117 | int pc; | ||
118 | |||
119 | if (!func_prolog_preempt_disable(tr, &data, &pc)) | ||
120 | return; | ||
121 | |||
122 | local_irq_save(flags); | ||
123 | trace_function(tr, ip, parent_ip, flags, pc); | ||
124 | local_irq_restore(flags); | ||
125 | |||
126 | atomic_dec(&data->disabled); | ||
127 | preempt_enable_notrace(); | ||
128 | } | ||
129 | |||
130 | static int register_wakeup_function(struct trace_array *tr, int graph, int set) | ||
131 | { | ||
132 | int ret; | ||
133 | |||
134 | /* 'set' is set if TRACE_ITER_FUNCTION is about to be set */ | ||
135 | if (function_enabled || (!set && !(tr->trace_flags & TRACE_ITER_FUNCTION))) | ||
136 | return 0; | ||
137 | |||
138 | if (graph) | ||
139 | ret = register_ftrace_graph(&wakeup_graph_return, | ||
140 | &wakeup_graph_entry); | ||
141 | else | ||
142 | ret = register_ftrace_function(tr->ops); | ||
143 | |||
144 | if (!ret) | ||
145 | function_enabled = true; | ||
146 | |||
147 | return ret; | ||
148 | } | ||
149 | |||
150 | static void unregister_wakeup_function(struct trace_array *tr, int graph) | ||
151 | { | ||
152 | if (!function_enabled) | ||
153 | return; | ||
154 | |||
155 | if (graph) | ||
156 | unregister_ftrace_graph(); | ||
157 | else | ||
158 | unregister_ftrace_function(tr->ops); | ||
159 | |||
160 | function_enabled = false; | ||
161 | } | ||
162 | |||
163 | static int wakeup_function_set(struct trace_array *tr, u32 mask, int set) | ||
164 | { | ||
165 | if (!(mask & TRACE_ITER_FUNCTION)) | ||
166 | return 0; | ||
167 | |||
168 | if (set) | ||
169 | register_wakeup_function(tr, is_graph(tr), 1); | ||
170 | else | ||
171 | unregister_wakeup_function(tr, is_graph(tr)); | ||
172 | return 1; | ||
173 | } | ||
174 | #else | ||
175 | static int register_wakeup_function(struct trace_array *tr, int graph, int set) | ||
176 | { | ||
177 | return 0; | ||
178 | } | ||
179 | static void unregister_wakeup_function(struct trace_array *tr, int graph) { } | ||
180 | static int wakeup_function_set(struct trace_array *tr, u32 mask, int set) | ||
181 | { | ||
182 | return 0; | ||
183 | } | ||
184 | #endif /* CONFIG_FUNCTION_TRACER */ | ||
185 | |||
186 | static int wakeup_flag_changed(struct trace_array *tr, u32 mask, int set) | ||
187 | { | ||
188 | struct tracer *tracer = tr->current_trace; | ||
189 | |||
190 | if (wakeup_function_set(tr, mask, set)) | ||
191 | return 0; | ||
192 | |||
193 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | 100 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
194 | if (mask & TRACE_ITER_DISPLAY_GRAPH) | ||
195 | return wakeup_display_graph(tr, set); | ||
196 | #endif | ||
197 | 101 | ||
198 | return trace_keep_overwrite(tracer, mask, set); | ||
199 | } | ||
200 | |||
201 | static int start_func_tracer(struct trace_array *tr, int graph) | ||
202 | { | ||
203 | int ret; | ||
204 | |||
205 | ret = register_wakeup_function(tr, graph, 0); | ||
206 | |||
207 | if (!ret && tracing_is_enabled()) | ||
208 | tracer_enabled = 1; | ||
209 | else | ||
210 | tracer_enabled = 0; | ||
211 | |||
212 | return ret; | ||
213 | } | ||
214 | |||
215 | static void stop_func_tracer(struct trace_array *tr, int graph) | ||
216 | { | ||
217 | tracer_enabled = 0; | ||
218 | |||
219 | unregister_wakeup_function(tr, graph); | ||
220 | } | ||
221 | |||
222 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | ||
223 | static int wakeup_display_graph(struct trace_array *tr, int set) | 102 | static int wakeup_display_graph(struct trace_array *tr, int set) |
224 | { | 103 | { |
225 | if (!(is_graph(tr) ^ set)) | 104 | if (!(is_graph(tr) ^ set)) |
@@ -283,6 +162,11 @@ static void wakeup_graph_return(struct ftrace_graph_ret *trace) | |||
283 | return; | 162 | return; |
284 | } | 163 | } |
285 | 164 | ||
165 | static struct fgraph_ops fgraph_wakeup_ops = { | ||
166 | .entryfunc = &wakeup_graph_entry, | ||
167 | .retfunc = &wakeup_graph_return, | ||
168 | }; | ||
169 | |||
286 | static void wakeup_trace_open(struct trace_iterator *iter) | 170 | static void wakeup_trace_open(struct trace_iterator *iter) |
287 | { | 171 | { |
288 | if (is_graph(iter->tr)) | 172 | if (is_graph(iter->tr)) |
@@ -318,20 +202,87 @@ static void wakeup_print_header(struct seq_file *s) | |||
318 | else | 202 | else |
319 | trace_default_header(s); | 203 | trace_default_header(s); |
320 | } | 204 | } |
205 | #endif /* else CONFIG_FUNCTION_GRAPH_TRACER */ | ||
321 | 206 | ||
207 | /* | ||
208 | * wakeup uses its own tracer function to keep the overhead down: | ||
209 | */ | ||
322 | static void | 210 | static void |
323 | __trace_function(struct trace_array *tr, | 211 | wakeup_tracer_call(unsigned long ip, unsigned long parent_ip, |
324 | unsigned long ip, unsigned long parent_ip, | 212 | struct ftrace_ops *op, struct pt_regs *pt_regs) |
325 | unsigned long flags, int pc) | ||
326 | { | 213 | { |
327 | if (is_graph(tr)) | 214 | struct trace_array *tr = wakeup_trace; |
328 | trace_graph_function(tr, ip, parent_ip, flags, pc); | 215 | struct trace_array_cpu *data; |
216 | unsigned long flags; | ||
217 | int pc; | ||
218 | |||
219 | if (!func_prolog_preempt_disable(tr, &data, &pc)) | ||
220 | return; | ||
221 | |||
222 | local_irq_save(flags); | ||
223 | trace_function(tr, ip, parent_ip, flags, pc); | ||
224 | local_irq_restore(flags); | ||
225 | |||
226 | atomic_dec(&data->disabled); | ||
227 | preempt_enable_notrace(); | ||
228 | } | ||
229 | |||
230 | static int register_wakeup_function(struct trace_array *tr, int graph, int set) | ||
231 | { | ||
232 | int ret; | ||
233 | |||
234 | /* 'set' is set if TRACE_ITER_FUNCTION is about to be set */ | ||
235 | if (function_enabled || (!set && !(tr->trace_flags & TRACE_ITER_FUNCTION))) | ||
236 | return 0; | ||
237 | |||
238 | if (graph) | ||
239 | ret = register_ftrace_graph(&fgraph_wakeup_ops); | ||
329 | else | 240 | else |
330 | trace_function(tr, ip, parent_ip, flags, pc); | 241 | ret = register_ftrace_function(tr->ops); |
242 | |||
243 | if (!ret) | ||
244 | function_enabled = true; | ||
245 | |||
246 | return ret; | ||
331 | } | 247 | } |
332 | #else | ||
333 | #define __trace_function trace_function | ||
334 | 248 | ||
249 | static void unregister_wakeup_function(struct trace_array *tr, int graph) | ||
250 | { | ||
251 | if (!function_enabled) | ||
252 | return; | ||
253 | |||
254 | if (graph) | ||
255 | unregister_ftrace_graph(&fgraph_wakeup_ops); | ||
256 | else | ||
257 | unregister_ftrace_function(tr->ops); | ||
258 | |||
259 | function_enabled = false; | ||
260 | } | ||
261 | |||
262 | static int wakeup_function_set(struct trace_array *tr, u32 mask, int set) | ||
263 | { | ||
264 | if (!(mask & TRACE_ITER_FUNCTION)) | ||
265 | return 0; | ||
266 | |||
267 | if (set) | ||
268 | register_wakeup_function(tr, is_graph(tr), 1); | ||
269 | else | ||
270 | unregister_wakeup_function(tr, is_graph(tr)); | ||
271 | return 1; | ||
272 | } | ||
273 | #else /* CONFIG_FUNCTION_TRACER */ | ||
274 | static int register_wakeup_function(struct trace_array *tr, int graph, int set) | ||
275 | { | ||
276 | return 0; | ||
277 | } | ||
278 | static void unregister_wakeup_function(struct trace_array *tr, int graph) { } | ||
279 | static int wakeup_function_set(struct trace_array *tr, u32 mask, int set) | ||
280 | { | ||
281 | return 0; | ||
282 | } | ||
283 | #endif /* else CONFIG_FUNCTION_TRACER */ | ||
284 | |||
285 | #ifndef CONFIG_FUNCTION_GRAPH_TRACER | ||
335 | static enum print_line_t wakeup_print_line(struct trace_iterator *iter) | 286 | static enum print_line_t wakeup_print_line(struct trace_iterator *iter) |
336 | { | 287 | { |
337 | return TRACE_TYPE_UNHANDLED; | 288 | return TRACE_TYPE_UNHANDLED; |
@@ -340,23 +291,58 @@ static enum print_line_t wakeup_print_line(struct trace_iterator *iter) | |||
340 | static void wakeup_trace_open(struct trace_iterator *iter) { } | 291 | static void wakeup_trace_open(struct trace_iterator *iter) { } |
341 | static void wakeup_trace_close(struct trace_iterator *iter) { } | 292 | static void wakeup_trace_close(struct trace_iterator *iter) { } |
342 | 293 | ||
343 | #ifdef CONFIG_FUNCTION_TRACER | ||
344 | static int wakeup_graph_entry(struct ftrace_graph_ent *trace) | ||
345 | { | ||
346 | return -1; | ||
347 | } | ||
348 | static void wakeup_graph_return(struct ftrace_graph_ret *trace) { } | ||
349 | static void wakeup_print_header(struct seq_file *s) | 294 | static void wakeup_print_header(struct seq_file *s) |
350 | { | 295 | { |
351 | trace_default_header(s); | 296 | trace_default_header(s); |
352 | } | 297 | } |
353 | #else | 298 | #endif /* !CONFIG_FUNCTION_GRAPH_TRACER */ |
354 | static void wakeup_print_header(struct seq_file *s) | 299 | |
300 | static void | ||
301 | __trace_function(struct trace_array *tr, | ||
302 | unsigned long ip, unsigned long parent_ip, | ||
303 | unsigned long flags, int pc) | ||
355 | { | 304 | { |
356 | trace_latency_header(s); | 305 | if (is_graph(tr)) |
306 | trace_graph_function(tr, ip, parent_ip, flags, pc); | ||
307 | else | ||
308 | trace_function(tr, ip, parent_ip, flags, pc); | ||
309 | } | ||
310 | |||
311 | static int wakeup_flag_changed(struct trace_array *tr, u32 mask, int set) | ||
312 | { | ||
313 | struct tracer *tracer = tr->current_trace; | ||
314 | |||
315 | if (wakeup_function_set(tr, mask, set)) | ||
316 | return 0; | ||
317 | |||
318 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | ||
319 | if (mask & TRACE_ITER_DISPLAY_GRAPH) | ||
320 | return wakeup_display_graph(tr, set); | ||
321 | #endif | ||
322 | |||
323 | return trace_keep_overwrite(tracer, mask, set); | ||
324 | } | ||
325 | |||
326 | static int start_func_tracer(struct trace_array *tr, int graph) | ||
327 | { | ||
328 | int ret; | ||
329 | |||
330 | ret = register_wakeup_function(tr, graph, 0); | ||
331 | |||
332 | if (!ret && tracing_is_enabled()) | ||
333 | tracer_enabled = 1; | ||
334 | else | ||
335 | tracer_enabled = 0; | ||
336 | |||
337 | return ret; | ||
338 | } | ||
339 | |||
340 | static void stop_func_tracer(struct trace_array *tr, int graph) | ||
341 | { | ||
342 | tracer_enabled = 0; | ||
343 | |||
344 | unregister_wakeup_function(tr, graph); | ||
357 | } | 345 | } |
358 | #endif /* CONFIG_FUNCTION_TRACER */ | ||
359 | #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ | ||
360 | 346 | ||
361 | /* | 347 | /* |
362 | * Should this new latency be reported/recorded? | 348 | * Should this new latency be reported/recorded? |
diff --git a/kernel/trace/trace_selftest.c b/kernel/trace/trace_selftest.c index 11e9daa4a568..9d402e7fc949 100644 --- a/kernel/trace/trace_selftest.c +++ b/kernel/trace/trace_selftest.c | |||
@@ -741,6 +741,11 @@ static int trace_graph_entry_watchdog(struct ftrace_graph_ent *trace) | |||
741 | return trace_graph_entry(trace); | 741 | return trace_graph_entry(trace); |
742 | } | 742 | } |
743 | 743 | ||
744 | static struct fgraph_ops fgraph_ops __initdata = { | ||
745 | .entryfunc = &trace_graph_entry_watchdog, | ||
746 | .retfunc = &trace_graph_return, | ||
747 | }; | ||
748 | |||
744 | /* | 749 | /* |
745 | * Pretty much the same than for the function tracer from which the selftest | 750 | * Pretty much the same than for the function tracer from which the selftest |
746 | * has been borrowed. | 751 | * has been borrowed. |
@@ -765,8 +770,7 @@ trace_selftest_startup_function_graph(struct tracer *trace, | |||
765 | */ | 770 | */ |
766 | tracing_reset_online_cpus(&tr->trace_buffer); | 771 | tracing_reset_online_cpus(&tr->trace_buffer); |
767 | set_graph_array(tr); | 772 | set_graph_array(tr); |
768 | ret = register_ftrace_graph(&trace_graph_return, | 773 | ret = register_ftrace_graph(&fgraph_ops); |
769 | &trace_graph_entry_watchdog); | ||
770 | if (ret) { | 774 | if (ret) { |
771 | warn_failed_init_tracer(trace, ret); | 775 | warn_failed_init_tracer(trace, ret); |
772 | goto out; | 776 | goto out; |
diff --git a/kernel/trace/trace_stack.c b/kernel/trace/trace_stack.c index 2b0d1ee3241c..eec648a0d673 100644 --- a/kernel/trace/trace_stack.c +++ b/kernel/trace/trace_stack.c | |||
@@ -286,7 +286,7 @@ __next(struct seq_file *m, loff_t *pos) | |||
286 | { | 286 | { |
287 | long n = *pos - 1; | 287 | long n = *pos - 1; |
288 | 288 | ||
289 | if (n > stack_trace_max.nr_entries || stack_dump_trace[n] == ULONG_MAX) | 289 | if (n >= stack_trace_max.nr_entries || stack_dump_trace[n] == ULONG_MAX) |
290 | return NULL; | 290 | return NULL; |
291 | 291 | ||
292 | m->private = (void *)n; | 292 | m->private = (void *)n; |
@@ -448,8 +448,10 @@ static char stack_trace_filter_buf[COMMAND_LINE_SIZE+1] __initdata; | |||
448 | 448 | ||
449 | static __init int enable_stacktrace(char *str) | 449 | static __init int enable_stacktrace(char *str) |
450 | { | 450 | { |
451 | if (strncmp(str, "_filter=", 8) == 0) | 451 | int len; |
452 | strncpy(stack_trace_filter_buf, str+8, COMMAND_LINE_SIZE); | 452 | |
453 | if ((len = str_has_prefix(str, "_filter="))) | ||
454 | strncpy(stack_trace_filter_buf, str + len, COMMAND_LINE_SIZE); | ||
453 | 455 | ||
454 | stack_tracer_enabled = 1; | 456 | stack_tracer_enabled = 1; |
455 | last_stack_tracer_enabled = 1; | 457 | last_stack_tracer_enabled = 1; |
diff --git a/kernel/trace/trace_uprobe.c b/kernel/trace/trace_uprobe.c index 31ea48eceda1..e335576b9411 100644 --- a/kernel/trace/trace_uprobe.c +++ b/kernel/trace/trace_uprobe.c | |||
@@ -7,6 +7,7 @@ | |||
7 | */ | 7 | */ |
8 | #define pr_fmt(fmt) "trace_kprobe: " fmt | 8 | #define pr_fmt(fmt) "trace_kprobe: " fmt |
9 | 9 | ||
10 | #include <linux/ctype.h> | ||
10 | #include <linux/module.h> | 11 | #include <linux/module.h> |
11 | #include <linux/uaccess.h> | 12 | #include <linux/uaccess.h> |
12 | #include <linux/uprobes.h> | 13 | #include <linux/uprobes.h> |
@@ -14,6 +15,7 @@ | |||
14 | #include <linux/string.h> | 15 | #include <linux/string.h> |
15 | #include <linux/rculist.h> | 16 | #include <linux/rculist.h> |
16 | 17 | ||
18 | #include "trace_dynevent.h" | ||
17 | #include "trace_probe.h" | 19 | #include "trace_probe.h" |
18 | #include "trace_probe_tmpl.h" | 20 | #include "trace_probe_tmpl.h" |
19 | 21 | ||
@@ -37,11 +39,26 @@ struct trace_uprobe_filter { | |||
37 | struct list_head perf_events; | 39 | struct list_head perf_events; |
38 | }; | 40 | }; |
39 | 41 | ||
42 | static int trace_uprobe_create(int argc, const char **argv); | ||
43 | static int trace_uprobe_show(struct seq_file *m, struct dyn_event *ev); | ||
44 | static int trace_uprobe_release(struct dyn_event *ev); | ||
45 | static bool trace_uprobe_is_busy(struct dyn_event *ev); | ||
46 | static bool trace_uprobe_match(const char *system, const char *event, | ||
47 | struct dyn_event *ev); | ||
48 | |||
49 | static struct dyn_event_operations trace_uprobe_ops = { | ||
50 | .create = trace_uprobe_create, | ||
51 | .show = trace_uprobe_show, | ||
52 | .is_busy = trace_uprobe_is_busy, | ||
53 | .free = trace_uprobe_release, | ||
54 | .match = trace_uprobe_match, | ||
55 | }; | ||
56 | |||
40 | /* | 57 | /* |
41 | * uprobe event core functions | 58 | * uprobe event core functions |
42 | */ | 59 | */ |
43 | struct trace_uprobe { | 60 | struct trace_uprobe { |
44 | struct list_head list; | 61 | struct dyn_event devent; |
45 | struct trace_uprobe_filter filter; | 62 | struct trace_uprobe_filter filter; |
46 | struct uprobe_consumer consumer; | 63 | struct uprobe_consumer consumer; |
47 | struct path path; | 64 | struct path path; |
@@ -53,6 +70,25 @@ struct trace_uprobe { | |||
53 | struct trace_probe tp; | 70 | struct trace_probe tp; |
54 | }; | 71 | }; |
55 | 72 | ||
73 | static bool is_trace_uprobe(struct dyn_event *ev) | ||
74 | { | ||
75 | return ev->ops == &trace_uprobe_ops; | ||
76 | } | ||
77 | |||
78 | static struct trace_uprobe *to_trace_uprobe(struct dyn_event *ev) | ||
79 | { | ||
80 | return container_of(ev, struct trace_uprobe, devent); | ||
81 | } | ||
82 | |||
83 | /** | ||
84 | * for_each_trace_uprobe - iterate over the trace_uprobe list | ||
85 | * @pos: the struct trace_uprobe * for each entry | ||
86 | * @dpos: the struct dyn_event * to use as a loop cursor | ||
87 | */ | ||
88 | #define for_each_trace_uprobe(pos, dpos) \ | ||
89 | for_each_dyn_event(dpos) \ | ||
90 | if (is_trace_uprobe(dpos) && (pos = to_trace_uprobe(dpos))) | ||
91 | |||
56 | #define SIZEOF_TRACE_UPROBE(n) \ | 92 | #define SIZEOF_TRACE_UPROBE(n) \ |
57 | (offsetof(struct trace_uprobe, tp.args) + \ | 93 | (offsetof(struct trace_uprobe, tp.args) + \ |
58 | (sizeof(struct probe_arg) * (n))) | 94 | (sizeof(struct probe_arg) * (n))) |
@@ -60,9 +96,6 @@ struct trace_uprobe { | |||
60 | static int register_uprobe_event(struct trace_uprobe *tu); | 96 | static int register_uprobe_event(struct trace_uprobe *tu); |
61 | static int unregister_uprobe_event(struct trace_uprobe *tu); | 97 | static int unregister_uprobe_event(struct trace_uprobe *tu); |
62 | 98 | ||
63 | static DEFINE_MUTEX(uprobe_lock); | ||
64 | static LIST_HEAD(uprobe_list); | ||
65 | |||
66 | struct uprobe_dispatch_data { | 99 | struct uprobe_dispatch_data { |
67 | struct trace_uprobe *tu; | 100 | struct trace_uprobe *tu; |
68 | unsigned long bp_addr; | 101 | unsigned long bp_addr; |
@@ -209,6 +242,22 @@ static inline bool is_ret_probe(struct trace_uprobe *tu) | |||
209 | return tu->consumer.ret_handler != NULL; | 242 | return tu->consumer.ret_handler != NULL; |
210 | } | 243 | } |
211 | 244 | ||
245 | static bool trace_uprobe_is_busy(struct dyn_event *ev) | ||
246 | { | ||
247 | struct trace_uprobe *tu = to_trace_uprobe(ev); | ||
248 | |||
249 | return trace_probe_is_enabled(&tu->tp); | ||
250 | } | ||
251 | |||
252 | static bool trace_uprobe_match(const char *system, const char *event, | ||
253 | struct dyn_event *ev) | ||
254 | { | ||
255 | struct trace_uprobe *tu = to_trace_uprobe(ev); | ||
256 | |||
257 | return strcmp(trace_event_name(&tu->tp.call), event) == 0 && | ||
258 | (!system || strcmp(tu->tp.call.class->system, system) == 0); | ||
259 | } | ||
260 | |||
212 | /* | 261 | /* |
213 | * Allocate new trace_uprobe and initialize it (including uprobes). | 262 | * Allocate new trace_uprobe and initialize it (including uprobes). |
214 | */ | 263 | */ |
@@ -236,7 +285,7 @@ alloc_trace_uprobe(const char *group, const char *event, int nargs, bool is_ret) | |||
236 | if (!tu->tp.class.system) | 285 | if (!tu->tp.class.system) |
237 | goto error; | 286 | goto error; |
238 | 287 | ||
239 | INIT_LIST_HEAD(&tu->list); | 288 | dyn_event_init(&tu->devent, &trace_uprobe_ops); |
240 | INIT_LIST_HEAD(&tu->tp.files); | 289 | INIT_LIST_HEAD(&tu->tp.files); |
241 | tu->consumer.handler = uprobe_dispatcher; | 290 | tu->consumer.handler = uprobe_dispatcher; |
242 | if (is_ret) | 291 | if (is_ret) |
@@ -255,6 +304,9 @@ static void free_trace_uprobe(struct trace_uprobe *tu) | |||
255 | { | 304 | { |
256 | int i; | 305 | int i; |
257 | 306 | ||
307 | if (!tu) | ||
308 | return; | ||
309 | |||
258 | for (i = 0; i < tu->tp.nr_args; i++) | 310 | for (i = 0; i < tu->tp.nr_args; i++) |
259 | traceprobe_free_probe_arg(&tu->tp.args[i]); | 311 | traceprobe_free_probe_arg(&tu->tp.args[i]); |
260 | 312 | ||
@@ -267,9 +319,10 @@ static void free_trace_uprobe(struct trace_uprobe *tu) | |||
267 | 319 | ||
268 | static struct trace_uprobe *find_probe_event(const char *event, const char *group) | 320 | static struct trace_uprobe *find_probe_event(const char *event, const char *group) |
269 | { | 321 | { |
322 | struct dyn_event *pos; | ||
270 | struct trace_uprobe *tu; | 323 | struct trace_uprobe *tu; |
271 | 324 | ||
272 | list_for_each_entry(tu, &uprobe_list, list) | 325 | for_each_trace_uprobe(tu, pos) |
273 | if (strcmp(trace_event_name(&tu->tp.call), event) == 0 && | 326 | if (strcmp(trace_event_name(&tu->tp.call), event) == 0 && |
274 | strcmp(tu->tp.call.class->system, group) == 0) | 327 | strcmp(tu->tp.call.class->system, group) == 0) |
275 | return tu; | 328 | return tu; |
@@ -277,7 +330,7 @@ static struct trace_uprobe *find_probe_event(const char *event, const char *grou | |||
277 | return NULL; | 330 | return NULL; |
278 | } | 331 | } |
279 | 332 | ||
280 | /* Unregister a trace_uprobe and probe_event: call with locking uprobe_lock */ | 333 | /* Unregister a trace_uprobe and probe_event */ |
281 | static int unregister_trace_uprobe(struct trace_uprobe *tu) | 334 | static int unregister_trace_uprobe(struct trace_uprobe *tu) |
282 | { | 335 | { |
283 | int ret; | 336 | int ret; |
@@ -286,7 +339,7 @@ static int unregister_trace_uprobe(struct trace_uprobe *tu) | |||
286 | if (ret) | 339 | if (ret) |
287 | return ret; | 340 | return ret; |
288 | 341 | ||
289 | list_del(&tu->list); | 342 | dyn_event_remove(&tu->devent); |
290 | free_trace_uprobe(tu); | 343 | free_trace_uprobe(tu); |
291 | return 0; | 344 | return 0; |
292 | } | 345 | } |
@@ -302,13 +355,14 @@ static int unregister_trace_uprobe(struct trace_uprobe *tu) | |||
302 | */ | 355 | */ |
303 | static struct trace_uprobe *find_old_trace_uprobe(struct trace_uprobe *new) | 356 | static struct trace_uprobe *find_old_trace_uprobe(struct trace_uprobe *new) |
304 | { | 357 | { |
358 | struct dyn_event *pos; | ||
305 | struct trace_uprobe *tmp, *old = NULL; | 359 | struct trace_uprobe *tmp, *old = NULL; |
306 | struct inode *new_inode = d_real_inode(new->path.dentry); | 360 | struct inode *new_inode = d_real_inode(new->path.dentry); |
307 | 361 | ||
308 | old = find_probe_event(trace_event_name(&new->tp.call), | 362 | old = find_probe_event(trace_event_name(&new->tp.call), |
309 | new->tp.call.class->system); | 363 | new->tp.call.class->system); |
310 | 364 | ||
311 | list_for_each_entry(tmp, &uprobe_list, list) { | 365 | for_each_trace_uprobe(tmp, pos) { |
312 | if ((old ? old != tmp : true) && | 366 | if ((old ? old != tmp : true) && |
313 | new_inode == d_real_inode(tmp->path.dentry) && | 367 | new_inode == d_real_inode(tmp->path.dentry) && |
314 | new->offset == tmp->offset && | 368 | new->offset == tmp->offset && |
@@ -326,7 +380,7 @@ static int register_trace_uprobe(struct trace_uprobe *tu) | |||
326 | struct trace_uprobe *old_tu; | 380 | struct trace_uprobe *old_tu; |
327 | int ret; | 381 | int ret; |
328 | 382 | ||
329 | mutex_lock(&uprobe_lock); | 383 | mutex_lock(&event_mutex); |
330 | 384 | ||
331 | /* register as an event */ | 385 | /* register as an event */ |
332 | old_tu = find_old_trace_uprobe(tu); | 386 | old_tu = find_old_trace_uprobe(tu); |
@@ -348,10 +402,10 @@ static int register_trace_uprobe(struct trace_uprobe *tu) | |||
348 | goto end; | 402 | goto end; |
349 | } | 403 | } |
350 | 404 | ||
351 | list_add_tail(&tu->list, &uprobe_list); | 405 | dyn_event_add(&tu->devent); |
352 | 406 | ||
353 | end: | 407 | end: |
354 | mutex_unlock(&uprobe_lock); | 408 | mutex_unlock(&event_mutex); |
355 | 409 | ||
356 | return ret; | 410 | return ret; |
357 | } | 411 | } |
@@ -362,91 +416,49 @@ end: | |||
362 | * | 416 | * |
363 | * - Remove uprobe: -:[GRP/]EVENT | 417 | * - Remove uprobe: -:[GRP/]EVENT |
364 | */ | 418 | */ |
365 | static int create_trace_uprobe(int argc, char **argv) | 419 | static int trace_uprobe_create(int argc, const char **argv) |
366 | { | 420 | { |
367 | struct trace_uprobe *tu; | 421 | struct trace_uprobe *tu; |
368 | char *arg, *event, *group, *filename, *rctr, *rctr_end; | 422 | const char *event = NULL, *group = UPROBE_EVENT_SYSTEM; |
423 | char *arg, *filename, *rctr, *rctr_end, *tmp; | ||
369 | char buf[MAX_EVENT_NAME_LEN]; | 424 | char buf[MAX_EVENT_NAME_LEN]; |
370 | struct path path; | 425 | struct path path; |
371 | unsigned long offset, ref_ctr_offset; | 426 | unsigned long offset, ref_ctr_offset; |
372 | bool is_delete, is_return; | 427 | bool is_return = false; |
373 | int i, ret; | 428 | int i, ret; |
374 | 429 | ||
375 | ret = 0; | 430 | ret = 0; |
376 | is_delete = false; | ||
377 | is_return = false; | ||
378 | event = NULL; | ||
379 | group = NULL; | ||
380 | ref_ctr_offset = 0; | 431 | ref_ctr_offset = 0; |
381 | 432 | ||
382 | /* argc must be >= 1 */ | 433 | /* argc must be >= 1 */ |
383 | if (argv[0][0] == '-') | 434 | if (argv[0][0] == 'r') |
384 | is_delete = true; | ||
385 | else if (argv[0][0] == 'r') | ||
386 | is_return = true; | 435 | is_return = true; |
387 | else if (argv[0][0] != 'p') { | 436 | else if (argv[0][0] != 'p' || argc < 2) |
388 | pr_info("Probe definition must be started with 'p', 'r' or '-'.\n"); | 437 | return -ECANCELED; |
389 | return -EINVAL; | ||
390 | } | ||
391 | 438 | ||
392 | if (argv[0][1] == ':') { | 439 | if (argv[0][1] == ':') |
393 | event = &argv[0][2]; | 440 | event = &argv[0][2]; |
394 | arg = strchr(event, '/'); | ||
395 | |||
396 | if (arg) { | ||
397 | group = event; | ||
398 | event = arg + 1; | ||
399 | event[-1] = '\0'; | ||
400 | |||
401 | if (strlen(group) == 0) { | ||
402 | pr_info("Group name is not specified\n"); | ||
403 | return -EINVAL; | ||
404 | } | ||
405 | } | ||
406 | if (strlen(event) == 0) { | ||
407 | pr_info("Event name is not specified\n"); | ||
408 | return -EINVAL; | ||
409 | } | ||
410 | } | ||
411 | if (!group) | ||
412 | group = UPROBE_EVENT_SYSTEM; | ||
413 | 441 | ||
414 | if (is_delete) { | 442 | if (!strchr(argv[1], '/')) |
415 | int ret; | 443 | return -ECANCELED; |
416 | 444 | ||
417 | if (!event) { | 445 | filename = kstrdup(argv[1], GFP_KERNEL); |
418 | pr_info("Delete command needs an event name.\n"); | 446 | if (!filename) |
419 | return -EINVAL; | 447 | return -ENOMEM; |
420 | } | ||
421 | mutex_lock(&uprobe_lock); | ||
422 | tu = find_probe_event(event, group); | ||
423 | |||
424 | if (!tu) { | ||
425 | mutex_unlock(&uprobe_lock); | ||
426 | pr_info("Event %s/%s doesn't exist.\n", group, event); | ||
427 | return -ENOENT; | ||
428 | } | ||
429 | /* delete an event */ | ||
430 | ret = unregister_trace_uprobe(tu); | ||
431 | mutex_unlock(&uprobe_lock); | ||
432 | return ret; | ||
433 | } | ||
434 | 448 | ||
435 | if (argc < 2) { | ||
436 | pr_info("Probe point is not specified.\n"); | ||
437 | return -EINVAL; | ||
438 | } | ||
439 | /* Find the last occurrence, in case the path contains ':' too. */ | 449 | /* Find the last occurrence, in case the path contains ':' too. */ |
440 | arg = strrchr(argv[1], ':'); | 450 | arg = strrchr(filename, ':'); |
441 | if (!arg) | 451 | if (!arg || !isdigit(arg[1])) { |
442 | return -EINVAL; | 452 | kfree(filename); |
453 | return -ECANCELED; | ||
454 | } | ||
443 | 455 | ||
444 | *arg++ = '\0'; | 456 | *arg++ = '\0'; |
445 | filename = argv[1]; | ||
446 | ret = kern_path(filename, LOOKUP_FOLLOW, &path); | 457 | ret = kern_path(filename, LOOKUP_FOLLOW, &path); |
447 | if (ret) | 458 | if (ret) { |
459 | kfree(filename); | ||
448 | return ret; | 460 | return ret; |
449 | 461 | } | |
450 | if (!d_is_reg(path.dentry)) { | 462 | if (!d_is_reg(path.dentry)) { |
451 | ret = -EINVAL; | 463 | ret = -EINVAL; |
452 | goto fail_address_parse; | 464 | goto fail_address_parse; |
@@ -480,7 +492,11 @@ static int create_trace_uprobe(int argc, char **argv) | |||
480 | argv += 2; | 492 | argv += 2; |
481 | 493 | ||
482 | /* setup a probe */ | 494 | /* setup a probe */ |
483 | if (!event) { | 495 | if (event) { |
496 | ret = traceprobe_parse_event_name(&event, &group, buf); | ||
497 | if (ret) | ||
498 | goto fail_address_parse; | ||
499 | } else { | ||
484 | char *tail; | 500 | char *tail; |
485 | char *ptr; | 501 | char *ptr; |
486 | 502 | ||
@@ -508,60 +524,21 @@ static int create_trace_uprobe(int argc, char **argv) | |||
508 | tu->offset = offset; | 524 | tu->offset = offset; |
509 | tu->ref_ctr_offset = ref_ctr_offset; | 525 | tu->ref_ctr_offset = ref_ctr_offset; |
510 | tu->path = path; | 526 | tu->path = path; |
511 | tu->filename = kstrdup(filename, GFP_KERNEL); | 527 | tu->filename = filename; |
512 | |||
513 | if (!tu->filename) { | ||
514 | pr_info("Failed to allocate filename.\n"); | ||
515 | ret = -ENOMEM; | ||
516 | goto error; | ||
517 | } | ||
518 | 528 | ||
519 | /* parse arguments */ | 529 | /* parse arguments */ |
520 | ret = 0; | ||
521 | for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) { | 530 | for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) { |
522 | struct probe_arg *parg = &tu->tp.args[i]; | 531 | tmp = kstrdup(argv[i], GFP_KERNEL); |
523 | 532 | if (!tmp) { | |
524 | /* Increment count for freeing args in error case */ | ||
525 | tu->tp.nr_args++; | ||
526 | |||
527 | /* Parse argument name */ | ||
528 | arg = strchr(argv[i], '='); | ||
529 | if (arg) { | ||
530 | *arg++ = '\0'; | ||
531 | parg->name = kstrdup(argv[i], GFP_KERNEL); | ||
532 | } else { | ||
533 | arg = argv[i]; | ||
534 | /* If argument name is omitted, set "argN" */ | ||
535 | snprintf(buf, MAX_EVENT_NAME_LEN, "arg%d", i + 1); | ||
536 | parg->name = kstrdup(buf, GFP_KERNEL); | ||
537 | } | ||
538 | |||
539 | if (!parg->name) { | ||
540 | pr_info("Failed to allocate argument[%d] name.\n", i); | ||
541 | ret = -ENOMEM; | 533 | ret = -ENOMEM; |
542 | goto error; | 534 | goto error; |
543 | } | 535 | } |
544 | 536 | ||
545 | if (!is_good_name(parg->name)) { | 537 | ret = traceprobe_parse_probe_arg(&tu->tp, i, tmp, |
546 | pr_info("Invalid argument[%d] name: %s\n", i, parg->name); | ||
547 | ret = -EINVAL; | ||
548 | goto error; | ||
549 | } | ||
550 | |||
551 | if (traceprobe_conflict_field_name(parg->name, tu->tp.args, i)) { | ||
552 | pr_info("Argument[%d] name '%s' conflicts with " | ||
553 | "another field.\n", i, argv[i]); | ||
554 | ret = -EINVAL; | ||
555 | goto error; | ||
556 | } | ||
557 | |||
558 | /* Parse fetch argument */ | ||
559 | ret = traceprobe_parse_probe_arg(arg, &tu->tp.size, parg, | ||
560 | is_return ? TPARG_FL_RETURN : 0); | 538 | is_return ? TPARG_FL_RETURN : 0); |
561 | if (ret) { | 539 | kfree(tmp); |
562 | pr_info("Parse error at argument[%d]. (%d)\n", i, ret); | 540 | if (ret) |
563 | goto error; | 541 | goto error; |
564 | } | ||
565 | } | 542 | } |
566 | 543 | ||
567 | ret = register_trace_uprobe(tu); | 544 | ret = register_trace_uprobe(tu); |
@@ -575,48 +552,35 @@ error: | |||
575 | 552 | ||
576 | fail_address_parse: | 553 | fail_address_parse: |
577 | path_put(&path); | 554 | path_put(&path); |
555 | kfree(filename); | ||
578 | 556 | ||
579 | pr_info("Failed to parse address or file.\n"); | 557 | pr_info("Failed to parse address or file.\n"); |
580 | 558 | ||
581 | return ret; | 559 | return ret; |
582 | } | 560 | } |
583 | 561 | ||
584 | static int cleanup_all_probes(void) | 562 | static int create_or_delete_trace_uprobe(int argc, char **argv) |
585 | { | 563 | { |
586 | struct trace_uprobe *tu; | 564 | int ret; |
587 | int ret = 0; | ||
588 | 565 | ||
589 | mutex_lock(&uprobe_lock); | 566 | if (argv[0][0] == '-') |
590 | while (!list_empty(&uprobe_list)) { | 567 | return dyn_event_release(argc, argv, &trace_uprobe_ops); |
591 | tu = list_entry(uprobe_list.next, struct trace_uprobe, list); | ||
592 | ret = unregister_trace_uprobe(tu); | ||
593 | if (ret) | ||
594 | break; | ||
595 | } | ||
596 | mutex_unlock(&uprobe_lock); | ||
597 | return ret; | ||
598 | } | ||
599 | 568 | ||
600 | /* Probes listing interfaces */ | 569 | ret = trace_uprobe_create(argc, (const char **)argv); |
601 | static void *probes_seq_start(struct seq_file *m, loff_t *pos) | 570 | return ret == -ECANCELED ? -EINVAL : ret; |
602 | { | ||
603 | mutex_lock(&uprobe_lock); | ||
604 | return seq_list_start(&uprobe_list, *pos); | ||
605 | } | 571 | } |
606 | 572 | ||
607 | static void *probes_seq_next(struct seq_file *m, void *v, loff_t *pos) | 573 | static int trace_uprobe_release(struct dyn_event *ev) |
608 | { | 574 | { |
609 | return seq_list_next(v, &uprobe_list, pos); | 575 | struct trace_uprobe *tu = to_trace_uprobe(ev); |
610 | } | ||
611 | 576 | ||
612 | static void probes_seq_stop(struct seq_file *m, void *v) | 577 | return unregister_trace_uprobe(tu); |
613 | { | ||
614 | mutex_unlock(&uprobe_lock); | ||
615 | } | 578 | } |
616 | 579 | ||
617 | static int probes_seq_show(struct seq_file *m, void *v) | 580 | /* Probes listing interfaces */ |
581 | static int trace_uprobe_show(struct seq_file *m, struct dyn_event *ev) | ||
618 | { | 582 | { |
619 | struct trace_uprobe *tu = v; | 583 | struct trace_uprobe *tu = to_trace_uprobe(ev); |
620 | char c = is_ret_probe(tu) ? 'r' : 'p'; | 584 | char c = is_ret_probe(tu) ? 'r' : 'p'; |
621 | int i; | 585 | int i; |
622 | 586 | ||
@@ -634,11 +598,21 @@ static int probes_seq_show(struct seq_file *m, void *v) | |||
634 | return 0; | 598 | return 0; |
635 | } | 599 | } |
636 | 600 | ||
601 | static int probes_seq_show(struct seq_file *m, void *v) | ||
602 | { | ||
603 | struct dyn_event *ev = v; | ||
604 | |||
605 | if (!is_trace_uprobe(ev)) | ||
606 | return 0; | ||
607 | |||
608 | return trace_uprobe_show(m, ev); | ||
609 | } | ||
610 | |||
637 | static const struct seq_operations probes_seq_op = { | 611 | static const struct seq_operations probes_seq_op = { |
638 | .start = probes_seq_start, | 612 | .start = dyn_event_seq_start, |
639 | .next = probes_seq_next, | 613 | .next = dyn_event_seq_next, |
640 | .stop = probes_seq_stop, | 614 | .stop = dyn_event_seq_stop, |
641 | .show = probes_seq_show | 615 | .show = probes_seq_show |
642 | }; | 616 | }; |
643 | 617 | ||
644 | static int probes_open(struct inode *inode, struct file *file) | 618 | static int probes_open(struct inode *inode, struct file *file) |
@@ -646,7 +620,7 @@ static int probes_open(struct inode *inode, struct file *file) | |||
646 | int ret; | 620 | int ret; |
647 | 621 | ||
648 | if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) { | 622 | if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) { |
649 | ret = cleanup_all_probes(); | 623 | ret = dyn_events_release_all(&trace_uprobe_ops); |
650 | if (ret) | 624 | if (ret) |
651 | return ret; | 625 | return ret; |
652 | } | 626 | } |
@@ -657,7 +631,8 @@ static int probes_open(struct inode *inode, struct file *file) | |||
657 | static ssize_t probes_write(struct file *file, const char __user *buffer, | 631 | static ssize_t probes_write(struct file *file, const char __user *buffer, |
658 | size_t count, loff_t *ppos) | 632 | size_t count, loff_t *ppos) |
659 | { | 633 | { |
660 | return trace_parse_run_command(file, buffer, count, ppos, create_trace_uprobe); | 634 | return trace_parse_run_command(file, buffer, count, ppos, |
635 | create_or_delete_trace_uprobe); | ||
661 | } | 636 | } |
662 | 637 | ||
663 | static const struct file_operations uprobe_events_ops = { | 638 | static const struct file_operations uprobe_events_ops = { |
@@ -672,17 +647,22 @@ static const struct file_operations uprobe_events_ops = { | |||
672 | /* Probes profiling interfaces */ | 647 | /* Probes profiling interfaces */ |
673 | static int probes_profile_seq_show(struct seq_file *m, void *v) | 648 | static int probes_profile_seq_show(struct seq_file *m, void *v) |
674 | { | 649 | { |
675 | struct trace_uprobe *tu = v; | 650 | struct dyn_event *ev = v; |
651 | struct trace_uprobe *tu; | ||
676 | 652 | ||
653 | if (!is_trace_uprobe(ev)) | ||
654 | return 0; | ||
655 | |||
656 | tu = to_trace_uprobe(ev); | ||
677 | seq_printf(m, " %s %-44s %15lu\n", tu->filename, | 657 | seq_printf(m, " %s %-44s %15lu\n", tu->filename, |
678 | trace_event_name(&tu->tp.call), tu->nhit); | 658 | trace_event_name(&tu->tp.call), tu->nhit); |
679 | return 0; | 659 | return 0; |
680 | } | 660 | } |
681 | 661 | ||
682 | static const struct seq_operations profile_seq_op = { | 662 | static const struct seq_operations profile_seq_op = { |
683 | .start = probes_seq_start, | 663 | .start = dyn_event_seq_start, |
684 | .next = probes_seq_next, | 664 | .next = dyn_event_seq_next, |
685 | .stop = probes_seq_stop, | 665 | .stop = dyn_event_seq_stop, |
686 | .show = probes_profile_seq_show | 666 | .show = probes_profile_seq_show |
687 | }; | 667 | }; |
688 | 668 | ||
@@ -1384,7 +1364,7 @@ create_local_trace_uprobe(char *name, unsigned long offs, | |||
1384 | } | 1364 | } |
1385 | 1365 | ||
1386 | /* | 1366 | /* |
1387 | * local trace_kprobes are not added to probe_list, so they are never | 1367 | * local trace_kprobes are not added to dyn_event, so they are never |
1388 | * searched in find_trace_kprobe(). Therefore, there is no concern of | 1368 | * searched in find_trace_kprobe(). Therefore, there is no concern of |
1389 | * duplicated name "DUMMY_EVENT" here. | 1369 | * duplicated name "DUMMY_EVENT" here. |
1390 | */ | 1370 | */ |
@@ -1432,6 +1412,11 @@ void destroy_local_trace_uprobe(struct trace_event_call *event_call) | |||
1432 | static __init int init_uprobe_trace(void) | 1412 | static __init int init_uprobe_trace(void) |
1433 | { | 1413 | { |
1434 | struct dentry *d_tracer; | 1414 | struct dentry *d_tracer; |
1415 | int ret; | ||
1416 | |||
1417 | ret = dyn_event_register(&trace_uprobe_ops); | ||
1418 | if (ret) | ||
1419 | return ret; | ||
1435 | 1420 | ||
1436 | d_tracer = tracing_init_dentry(); | 1421 | d_tracer = tracing_init_dentry(); |
1437 | if (IS_ERR(d_tracer)) | 1422 | if (IS_ERR(d_tracer)) |
diff --git a/lib/seq_buf.c b/lib/seq_buf.c index 11f2ae0f9099..bd807f545a9d 100644 --- a/lib/seq_buf.c +++ b/lib/seq_buf.c | |||
@@ -140,13 +140,17 @@ int seq_buf_bprintf(struct seq_buf *s, const char *fmt, const u32 *binary) | |||
140 | */ | 140 | */ |
141 | int seq_buf_puts(struct seq_buf *s, const char *str) | 141 | int seq_buf_puts(struct seq_buf *s, const char *str) |
142 | { | 142 | { |
143 | unsigned int len = strlen(str); | 143 | size_t len = strlen(str); |
144 | 144 | ||
145 | WARN_ON(s->size == 0); | 145 | WARN_ON(s->size == 0); |
146 | 146 | ||
147 | /* Add 1 to len for the trailing null byte which must be there */ | ||
148 | len += 1; | ||
149 | |||
147 | if (seq_buf_can_fit(s, len)) { | 150 | if (seq_buf_can_fit(s, len)) { |
148 | memcpy(s->buffer + s->len, str, len); | 151 | memcpy(s->buffer + s->len, str, len); |
149 | s->len += len; | 152 | /* Don't count the trailing null byte against the capacity */ |
153 | s->len += len - 1; | ||
150 | return 0; | 154 | return 0; |
151 | } | 155 | } |
152 | seq_buf_set_overflow(s); | 156 | seq_buf_set_overflow(s); |
diff --git a/scripts/recordmcount.c b/scripts/recordmcount.c index 895c40e8679f..a50a2aa963ad 100644 --- a/scripts/recordmcount.c +++ b/scripts/recordmcount.c | |||
@@ -397,7 +397,7 @@ static uint32_t (*w2)(uint16_t); | |||
397 | static int | 397 | static int |
398 | is_mcounted_section_name(char const *const txtname) | 398 | is_mcounted_section_name(char const *const txtname) |
399 | { | 399 | { |
400 | return strcmp(".text", txtname) == 0 || | 400 | return strncmp(".text", txtname, 5) == 0 || |
401 | strcmp(".init.text", txtname) == 0 || | 401 | strcmp(".init.text", txtname) == 0 || |
402 | strcmp(".ref.text", txtname) == 0 || | 402 | strcmp(".ref.text", txtname) == 0 || |
403 | strcmp(".sched.text", txtname) == 0 || | 403 | strcmp(".sched.text", txtname) == 0 || |
diff --git a/scripts/recordmcount.pl b/scripts/recordmcount.pl index f599031260d5..68841d01162c 100755 --- a/scripts/recordmcount.pl +++ b/scripts/recordmcount.pl | |||
@@ -142,6 +142,11 @@ my %text_sections = ( | |||
142 | ".text.unlikely" => 1, | 142 | ".text.unlikely" => 1, |
143 | ); | 143 | ); |
144 | 144 | ||
145 | # Acceptable section-prefixes to record. | ||
146 | my %text_section_prefixes = ( | ||
147 | ".text." => 1, | ||
148 | ); | ||
149 | |||
145 | # Note: we are nice to C-programmers here, thus we skip the '||='-idiom. | 150 | # Note: we are nice to C-programmers here, thus we skip the '||='-idiom. |
146 | $objdump = 'objdump' if (!$objdump); | 151 | $objdump = 'objdump' if (!$objdump); |
147 | $objcopy = 'objcopy' if (!$objcopy); | 152 | $objcopy = 'objcopy' if (!$objcopy); |
@@ -519,6 +524,14 @@ while (<IN>) { | |||
519 | 524 | ||
520 | # Only record text sections that we know are safe | 525 | # Only record text sections that we know are safe |
521 | $read_function = defined($text_sections{$1}); | 526 | $read_function = defined($text_sections{$1}); |
527 | if (!$read_function) { | ||
528 | foreach my $prefix (keys %text_section_prefixes) { | ||
529 | if (substr($1, 0, length $prefix) eq $prefix) { | ||
530 | $read_function = 1; | ||
531 | last; | ||
532 | } | ||
533 | } | ||
534 | } | ||
522 | # print out any recorded offsets | 535 | # print out any recorded offsets |
523 | update_funcs(); | 536 | update_funcs(); |
524 | 537 | ||
diff --git a/tools/testing/selftests/ftrace/test.d/dynevent/add_remove_kprobe.tc b/tools/testing/selftests/ftrace/test.d/dynevent/add_remove_kprobe.tc new file mode 100644 index 000000000000..c6d8387dbbb8 --- /dev/null +++ b/tools/testing/selftests/ftrace/test.d/dynevent/add_remove_kprobe.tc | |||
@@ -0,0 +1,30 @@ | |||
1 | #!/bin/sh | ||
2 | # SPDX-License-Identifier: GPL-2.0 | ||
3 | # description: Generic dynamic event - add/remove kprobe events | ||
4 | |||
5 | [ -f dynamic_events ] || exit_unsupported | ||
6 | |||
7 | grep -q "place: \[<module>:\]<symbol>" README || exit_unsupported | ||
8 | grep -q "place (kretprobe): \[<module>:\]<symbol>" README || exit_unsupported | ||
9 | |||
10 | echo 0 > events/enable | ||
11 | echo > dynamic_events | ||
12 | |||
13 | PLACE=_do_fork | ||
14 | |||
15 | echo "p:myevent1 $PLACE" >> dynamic_events | ||
16 | echo "r:myevent2 $PLACE" >> dynamic_events | ||
17 | |||
18 | grep -q myevent1 dynamic_events | ||
19 | grep -q myevent2 dynamic_events | ||
20 | test -d events/kprobes/myevent1 | ||
21 | test -d events/kprobes/myevent2 | ||
22 | |||
23 | echo "-:myevent2" >> dynamic_events | ||
24 | |||
25 | grep -q myevent1 dynamic_events | ||
26 | ! grep -q myevent2 dynamic_events | ||
27 | |||
28 | echo > dynamic_events | ||
29 | |||
30 | clear_trace | ||
diff --git a/tools/testing/selftests/ftrace/test.d/dynevent/add_remove_synth.tc b/tools/testing/selftests/ftrace/test.d/dynevent/add_remove_synth.tc new file mode 100644 index 000000000000..62b77b5941d0 --- /dev/null +++ b/tools/testing/selftests/ftrace/test.d/dynevent/add_remove_synth.tc | |||
@@ -0,0 +1,27 @@ | |||
1 | #!/bin/sh | ||
2 | # SPDX-License-Identifier: GPL-2.0 | ||
3 | # description: Generic dynamic event - add/remove synthetic events | ||
4 | |||
5 | [ -f dynamic_events ] || exit_unsupported | ||
6 | |||
7 | grep -q "s:\[synthetic/\]" README || exit_unsupported | ||
8 | |||
9 | echo 0 > events/enable | ||
10 | echo > dynamic_events | ||
11 | |||
12 | echo "s:latency1 u64 lat; pid_t pid;" >> dynamic_events | ||
13 | echo "s:latency2 u64 lat; pid_t pid;" >> dynamic_events | ||
14 | |||
15 | grep -q latency1 dynamic_events | ||
16 | grep -q latency2 dynamic_events | ||
17 | test -d events/synthetic/latency1 | ||
18 | test -d events/synthetic/latency2 | ||
19 | |||
20 | echo "-:synthetic/latency2" >> dynamic_events | ||
21 | |||
22 | grep -q latency1 dynamic_events | ||
23 | ! grep -q latency2 dynamic_events | ||
24 | |||
25 | echo > dynamic_events | ||
26 | |||
27 | clear_trace | ||
diff --git a/tools/testing/selftests/ftrace/test.d/dynevent/clear_select_events.tc b/tools/testing/selftests/ftrace/test.d/dynevent/clear_select_events.tc new file mode 100644 index 000000000000..e0842109cb57 --- /dev/null +++ b/tools/testing/selftests/ftrace/test.d/dynevent/clear_select_events.tc | |||
@@ -0,0 +1,50 @@ | |||
1 | #!/bin/sh | ||
2 | # SPDX-License-Identifier: GPL-2.0 | ||
3 | # description: Generic dynamic event - selective clear (compatibility) | ||
4 | |||
5 | [ -f dynamic_events ] || exit_unsupported | ||
6 | |||
7 | grep -q "place: \[<module>:\]<symbol>" README || exit_unsupported | ||
8 | grep -q "place (kretprobe): \[<module>:\]<symbol>" README || exit_unsupported | ||
9 | |||
10 | grep -q "s:\[synthetic/\]" README || exit_unsupported | ||
11 | |||
12 | [ -f synthetic_events ] || exit_unsupported | ||
13 | [ -f kprobe_events ] || exit_unsupported | ||
14 | |||
15 | echo 0 > events/enable | ||
16 | echo > dynamic_events | ||
17 | |||
18 | PLACE=_do_fork | ||
19 | |||
20 | setup_events() { | ||
21 | echo "p:myevent1 $PLACE" >> dynamic_events | ||
22 | echo "s:latency1 u64 lat; pid_t pid;" >> dynamic_events | ||
23 | echo "r:myevent2 $PLACE" >> dynamic_events | ||
24 | echo "s:latency2 u64 lat; pid_t pid;" >> dynamic_events | ||
25 | |||
26 | grep -q myevent1 dynamic_events | ||
27 | grep -q myevent2 dynamic_events | ||
28 | grep -q latency1 dynamic_events | ||
29 | grep -q latency2 dynamic_events | ||
30 | } | ||
31 | |||
32 | setup_events | ||
33 | echo > synthetic_events | ||
34 | |||
35 | grep -q myevent1 dynamic_events | ||
36 | grep -q myevent2 dynamic_events | ||
37 | ! grep -q latency1 dynamic_events | ||
38 | ! grep -q latency2 dynamic_events | ||
39 | |||
40 | echo > dynamic_events | ||
41 | |||
42 | setup_events | ||
43 | echo > kprobe_events | ||
44 | |||
45 | ! grep -q myevent1 dynamic_events | ||
46 | ! grep -q myevent2 dynamic_events | ||
47 | grep -q latency1 dynamic_events | ||
48 | grep -q latency2 dynamic_events | ||
49 | |||
50 | echo > dynamic_events | ||
diff --git a/tools/testing/selftests/ftrace/test.d/dynevent/generic_clear_event.tc b/tools/testing/selftests/ftrace/test.d/dynevent/generic_clear_event.tc new file mode 100644 index 000000000000..901922e97878 --- /dev/null +++ b/tools/testing/selftests/ftrace/test.d/dynevent/generic_clear_event.tc | |||
@@ -0,0 +1,49 @@ | |||
1 | #!/bin/sh | ||
2 | # SPDX-License-Identifier: GPL-2.0 | ||
3 | # description: Generic dynamic event - generic clear event | ||
4 | |||
5 | [ -f dynamic_events ] || exit_unsupported | ||
6 | |||
7 | grep -q "place: \[<module>:\]<symbol>" README || exit_unsupported | ||
8 | grep -q "place (kretprobe): \[<module>:\]<symbol>" README || exit_unsupported | ||
9 | |||
10 | grep -q "s:\[synthetic/\]" README || exit_unsupported | ||
11 | |||
12 | echo 0 > events/enable | ||
13 | echo > dynamic_events | ||
14 | |||
15 | PLACE=_do_fork | ||
16 | |||
17 | setup_events() { | ||
18 | echo "p:myevent1 $PLACE" >> dynamic_events | ||
19 | echo "s:latency1 u64 lat; pid_t pid;" >> dynamic_events | ||
20 | echo "r:myevent2 $PLACE" >> dynamic_events | ||
21 | echo "s:latency2 u64 lat; pid_t pid;" >> dynamic_events | ||
22 | |||
23 | grep -q myevent1 dynamic_events | ||
24 | grep -q myevent2 dynamic_events | ||
25 | grep -q latency1 dynamic_events | ||
26 | grep -q latency2 dynamic_events | ||
27 | } | ||
28 | |||
29 | setup_events | ||
30 | |||
31 | echo "!p:myevent1 $PLACE" >> dynamic_events | ||
32 | ! grep -q myevent1 dynamic_events | ||
33 | grep -q myevent2 dynamic_events | ||
34 | grep -q latency1 dynamic_events | ||
35 | grep -q latency2 dynamic_events | ||
36 | |||
37 | echo "!s:latency1 u64 lat; pid_t pid;" >> dynamic_events | ||
38 | grep -q myevent2 dynamic_events | ||
39 | ! grep -q latency1 dynamic_events | ||
40 | grep -q latency2 dynamic_events | ||
41 | |||
42 | echo "!r:myevent2 $PLACE" >> dynamic_events | ||
43 | ! grep -q myevent2 dynamic_events | ||
44 | grep -q latency2 dynamic_events | ||
45 | |||
46 | echo "!s:latency2 u64 lat; pid_t pid;" >> dynamic_events | ||
47 | ! grep -q latency2 dynamic_events | ||
48 | |||
49 | echo > dynamic_events | ||