aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--Documentation/trace/kprobetrace.rst3
-rw-r--r--Documentation/trace/uprobetracer.rst4
-rw-r--r--arch/arm64/kernel/ftrace.c1
-rw-r--r--arch/arm64/kernel/perf_callchain.c2
-rw-r--r--arch/arm64/kernel/process.c2
-rw-r--r--arch/arm64/kernel/return_address.c2
-rw-r--r--arch/arm64/kernel/stacktrace.c15
-rw-r--r--arch/arm64/kernel/time.c2
-rw-r--r--arch/arm64/kernel/traps.c2
-rw-r--r--arch/powerpc/kernel/process.c13
-rw-r--r--arch/sh/kernel/dumpstack.c11
-rw-r--r--arch/sh/kernel/dwarf.c9
-rw-r--r--arch/sparc/kernel/perf_event.c8
-rw-r--r--arch/sparc/kernel/stacktrace.c8
-rw-r--r--arch/sparc/kernel/traps_64.c7
-rw-r--r--arch/x86/kernel/ftrace.c41
-rw-r--r--arch/x86/kernel/ftrace_64.S8
-rw-r--r--include/linux/ftrace.h36
-rw-r--r--include/linux/ring_buffer.h4
-rw-r--r--include/linux/string.h20
-rw-r--r--kernel/trace/Kconfig6
-rw-r--r--kernel/trace/Makefile2
-rw-r--r--kernel/trace/fgraph.c626
-rw-r--r--kernel/trace/ftrace.c490
-rw-r--r--kernel/trace/ftrace_internal.h75
-rw-r--r--kernel/trace/ring_buffer.c94
-rw-r--r--kernel/trace/trace.c82
-rw-r--r--kernel/trace/trace.h13
-rw-r--r--kernel/trace/trace_dynevent.c217
-rw-r--r--kernel/trace/trace_dynevent.h119
-rw-r--r--kernel/trace/trace_events.c10
-rw-r--r--kernel/trace/trace_events_hist.c597
-rw-r--r--kernel/trace/trace_functions_graph.c334
-rw-r--r--kernel/trace/trace_irqsoff.c18
-rw-r--r--kernel/trace/trace_kprobe.c353
-rw-r--r--kernel/trace/trace_output.c38
-rw-r--r--kernel/trace/trace_probe.c91
-rw-r--r--kernel/trace/trace_probe.h9
-rw-r--r--kernel/trace/trace_sched_wakeup.c270
-rw-r--r--kernel/trace/trace_selftest.c8
-rw-r--r--kernel/trace/trace_stack.c8
-rw-r--r--kernel/trace/trace_uprobe.c301
-rw-r--r--lib/seq_buf.c8
-rw-r--r--scripts/recordmcount.c2
-rwxr-xr-xscripts/recordmcount.pl13
-rw-r--r--tools/testing/selftests/ftrace/test.d/dynevent/add_remove_kprobe.tc30
-rw-r--r--tools/testing/selftests/ftrace/test.d/dynevent/add_remove_synth.tc27
-rw-r--r--tools/testing/selftests/ftrace/test.d/dynevent/clear_select_events.tc50
-rw-r--r--tools/testing/selftests/ftrace/test.d/dynevent/generic_clear_event.tc49
49 files changed, 2494 insertions, 1644 deletions
diff --git a/Documentation/trace/kprobetrace.rst b/Documentation/trace/kprobetrace.rst
index 47e765c2f2c3..235ce2ab131a 100644
--- a/Documentation/trace/kprobetrace.rst
+++ b/Documentation/trace/kprobetrace.rst
@@ -20,6 +20,9 @@ current_tracer. Instead of that, add probe points via
20/sys/kernel/debug/tracing/kprobe_events, and enable it via 20/sys/kernel/debug/tracing/kprobe_events, and enable it via
21/sys/kernel/debug/tracing/events/kprobes/<EVENT>/enable. 21/sys/kernel/debug/tracing/events/kprobes/<EVENT>/enable.
22 22
23You can also use /sys/kernel/debug/tracing/dynamic_events instead of
24kprobe_events. That interface will provide unified access to other
25dynamic events too.
23 26
24Synopsis of kprobe_events 27Synopsis of kprobe_events
25------------------------- 28-------------------------
diff --git a/Documentation/trace/uprobetracer.rst b/Documentation/trace/uprobetracer.rst
index d0822811527a..4c3bfde2ba47 100644
--- a/Documentation/trace/uprobetracer.rst
+++ b/Documentation/trace/uprobetracer.rst
@@ -18,6 +18,10 @@ current_tracer. Instead of that, add probe points via
18However unlike kprobe-event tracer, the uprobe event interface expects the 18However unlike kprobe-event tracer, the uprobe event interface expects the
19user to calculate the offset of the probepoint in the object. 19user to calculate the offset of the probepoint in the object.
20 20
21You can also use /sys/kernel/debug/tracing/dynamic_events instead of
22uprobe_events. That interface will provide unified access to other
23dynamic events too.
24
21Synopsis of uprobe_tracer 25Synopsis of uprobe_tracer
22------------------------- 26-------------------------
23:: 27::
diff --git a/arch/arm64/kernel/ftrace.c b/arch/arm64/kernel/ftrace.c
index c1f30f854fb3..8e4431a8821f 100644
--- a/arch/arm64/kernel/ftrace.c
+++ b/arch/arm64/kernel/ftrace.c
@@ -193,6 +193,7 @@ int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec,
193 193
194void arch_ftrace_update_code(int command) 194void arch_ftrace_update_code(int command)
195{ 195{
196 command |= FTRACE_MAY_SLEEP;
196 ftrace_modify_all_code(command); 197 ftrace_modify_all_code(command);
197} 198}
198 199
diff --git a/arch/arm64/kernel/perf_callchain.c b/arch/arm64/kernel/perf_callchain.c
index 94754f07f67a..a34c26afacb0 100644
--- a/arch/arm64/kernel/perf_callchain.c
+++ b/arch/arm64/kernel/perf_callchain.c
@@ -168,7 +168,7 @@ void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry,
168 frame.fp = regs->regs[29]; 168 frame.fp = regs->regs[29];
169 frame.pc = regs->pc; 169 frame.pc = regs->pc;
170#ifdef CONFIG_FUNCTION_GRAPH_TRACER 170#ifdef CONFIG_FUNCTION_GRAPH_TRACER
171 frame.graph = current->curr_ret_stack; 171 frame.graph = 0;
172#endif 172#endif
173 173
174 walk_stackframe(current, &frame, callchain_trace, entry); 174 walk_stackframe(current, &frame, callchain_trace, entry);
diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c
index e0a443730e04..a0f985a6ac50 100644
--- a/arch/arm64/kernel/process.c
+++ b/arch/arm64/kernel/process.c
@@ -461,7 +461,7 @@ unsigned long get_wchan(struct task_struct *p)
461 frame.fp = thread_saved_fp(p); 461 frame.fp = thread_saved_fp(p);
462 frame.pc = thread_saved_pc(p); 462 frame.pc = thread_saved_pc(p);
463#ifdef CONFIG_FUNCTION_GRAPH_TRACER 463#ifdef CONFIG_FUNCTION_GRAPH_TRACER
464 frame.graph = p->curr_ret_stack; 464 frame.graph = 0;
465#endif 465#endif
466 do { 466 do {
467 if (unwind_frame(p, &frame)) 467 if (unwind_frame(p, &frame))
diff --git a/arch/arm64/kernel/return_address.c b/arch/arm64/kernel/return_address.c
index 933adbc0f654..53c40196b607 100644
--- a/arch/arm64/kernel/return_address.c
+++ b/arch/arm64/kernel/return_address.c
@@ -44,7 +44,7 @@ void *return_address(unsigned int level)
44 frame.fp = (unsigned long)__builtin_frame_address(0); 44 frame.fp = (unsigned long)__builtin_frame_address(0);
45 frame.pc = (unsigned long)return_address; /* dummy */ 45 frame.pc = (unsigned long)return_address; /* dummy */
46#ifdef CONFIG_FUNCTION_GRAPH_TRACER 46#ifdef CONFIG_FUNCTION_GRAPH_TRACER
47 frame.graph = current->curr_ret_stack; 47 frame.graph = 0;
48#endif 48#endif
49 49
50 walk_stackframe(current, &frame, save_return_addr, &data); 50 walk_stackframe(current, &frame, save_return_addr, &data);
diff --git a/arch/arm64/kernel/stacktrace.c b/arch/arm64/kernel/stacktrace.c
index 4989f7ea1e59..1a29f2695ff2 100644
--- a/arch/arm64/kernel/stacktrace.c
+++ b/arch/arm64/kernel/stacktrace.c
@@ -59,18 +59,17 @@ int notrace unwind_frame(struct task_struct *tsk, struct stackframe *frame)
59#ifdef CONFIG_FUNCTION_GRAPH_TRACER 59#ifdef CONFIG_FUNCTION_GRAPH_TRACER
60 if (tsk->ret_stack && 60 if (tsk->ret_stack &&
61 (frame->pc == (unsigned long)return_to_handler)) { 61 (frame->pc == (unsigned long)return_to_handler)) {
62 if (WARN_ON_ONCE(frame->graph == -1)) 62 struct ftrace_ret_stack *ret_stack;
63 return -EINVAL;
64 if (frame->graph < -1)
65 frame->graph += FTRACE_NOTRACE_DEPTH;
66
67 /* 63 /*
68 * This is a case where function graph tracer has 64 * This is a case where function graph tracer has
69 * modified a return address (LR) in a stack frame 65 * modified a return address (LR) in a stack frame
70 * to hook a function return. 66 * to hook a function return.
71 * So replace it to an original value. 67 * So replace it to an original value.
72 */ 68 */
73 frame->pc = tsk->ret_stack[frame->graph--].ret; 69 ret_stack = ftrace_graph_get_ret_stack(tsk, frame->graph++);
70 if (WARN_ON_ONCE(!ret_stack))
71 return -EINVAL;
72 frame->pc = ret_stack->ret;
74 } 73 }
75#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 74#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
76 75
@@ -137,7 +136,7 @@ void save_stack_trace_regs(struct pt_regs *regs, struct stack_trace *trace)
137 frame.fp = regs->regs[29]; 136 frame.fp = regs->regs[29];
138 frame.pc = regs->pc; 137 frame.pc = regs->pc;
139#ifdef CONFIG_FUNCTION_GRAPH_TRACER 138#ifdef CONFIG_FUNCTION_GRAPH_TRACER
140 frame.graph = current->curr_ret_stack; 139 frame.graph = 0;
141#endif 140#endif
142 141
143 walk_stackframe(current, &frame, save_trace, &data); 142 walk_stackframe(current, &frame, save_trace, &data);
@@ -168,7 +167,7 @@ static noinline void __save_stack_trace(struct task_struct *tsk,
168 frame.pc = (unsigned long)__save_stack_trace; 167 frame.pc = (unsigned long)__save_stack_trace;
169 } 168 }
170#ifdef CONFIG_FUNCTION_GRAPH_TRACER 169#ifdef CONFIG_FUNCTION_GRAPH_TRACER
171 frame.graph = tsk->curr_ret_stack; 170 frame.graph = 0;
172#endif 171#endif
173 172
174 walk_stackframe(tsk, &frame, save_trace, &data); 173 walk_stackframe(tsk, &frame, save_trace, &data);
diff --git a/arch/arm64/kernel/time.c b/arch/arm64/kernel/time.c
index f258636273c9..a777ae90044d 100644
--- a/arch/arm64/kernel/time.c
+++ b/arch/arm64/kernel/time.c
@@ -52,7 +52,7 @@ unsigned long profile_pc(struct pt_regs *regs)
52 frame.fp = regs->regs[29]; 52 frame.fp = regs->regs[29];
53 frame.pc = regs->pc; 53 frame.pc = regs->pc;
54#ifdef CONFIG_FUNCTION_GRAPH_TRACER 54#ifdef CONFIG_FUNCTION_GRAPH_TRACER
55 frame.graph = current->curr_ret_stack; 55 frame.graph = 0;
56#endif 56#endif
57 do { 57 do {
58 int ret = unwind_frame(NULL, &frame); 58 int ret = unwind_frame(NULL, &frame);
diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c
index cdc71cf70aad..4e2fb877f8d5 100644
--- a/arch/arm64/kernel/traps.c
+++ b/arch/arm64/kernel/traps.c
@@ -123,7 +123,7 @@ void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk)
123 frame.pc = thread_saved_pc(tsk); 123 frame.pc = thread_saved_pc(tsk);
124 } 124 }
125#ifdef CONFIG_FUNCTION_GRAPH_TRACER 125#ifdef CONFIG_FUNCTION_GRAPH_TRACER
126 frame.graph = tsk->curr_ret_stack; 126 frame.graph = 0;
127#endif 127#endif
128 128
129 skip = !!regs; 129 skip = !!regs;
diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c
index 96f34730010f..ce393df243aa 100644
--- a/arch/powerpc/kernel/process.c
+++ b/arch/powerpc/kernel/process.c
@@ -2061,9 +2061,10 @@ void show_stack(struct task_struct *tsk, unsigned long *stack)
2061 int count = 0; 2061 int count = 0;
2062 int firstframe = 1; 2062 int firstframe = 1;
2063#ifdef CONFIG_FUNCTION_GRAPH_TRACER 2063#ifdef CONFIG_FUNCTION_GRAPH_TRACER
2064 int curr_frame = current->curr_ret_stack; 2064 struct ftrace_ret_stack *ret_stack;
2065 extern void return_to_handler(void); 2065 extern void return_to_handler(void);
2066 unsigned long rth = (unsigned long)return_to_handler; 2066 unsigned long rth = (unsigned long)return_to_handler;
2067 int curr_frame = 0;
2067#endif 2068#endif
2068 2069
2069 sp = (unsigned long) stack; 2070 sp = (unsigned long) stack;
@@ -2089,9 +2090,13 @@ void show_stack(struct task_struct *tsk, unsigned long *stack)
2089 printk("["REG"] ["REG"] %pS", sp, ip, (void *)ip); 2090 printk("["REG"] ["REG"] %pS", sp, ip, (void *)ip);
2090#ifdef CONFIG_FUNCTION_GRAPH_TRACER 2091#ifdef CONFIG_FUNCTION_GRAPH_TRACER
2091 if ((ip == rth) && curr_frame >= 0) { 2092 if ((ip == rth) && curr_frame >= 0) {
2092 pr_cont(" (%pS)", 2093 ret_stack = ftrace_graph_get_ret_stack(current,
2093 (void *)current->ret_stack[curr_frame].ret); 2094 curr_frame++);
2094 curr_frame--; 2095 if (ret_stack)
2096 pr_cont(" (%pS)",
2097 (void *)ret_stack->ret);
2098 else
2099 curr_frame = -1;
2095 } 2100 }
2096#endif 2101#endif
2097 if (firstframe) 2102 if (firstframe)
diff --git a/arch/sh/kernel/dumpstack.c b/arch/sh/kernel/dumpstack.c
index 93c6c0e691ee..9f1c9c11d62d 100644
--- a/arch/sh/kernel/dumpstack.c
+++ b/arch/sh/kernel/dumpstack.c
@@ -56,17 +56,20 @@ print_ftrace_graph_addr(unsigned long addr, void *data,
56 struct thread_info *tinfo, int *graph) 56 struct thread_info *tinfo, int *graph)
57{ 57{
58 struct task_struct *task = tinfo->task; 58 struct task_struct *task = tinfo->task;
59 struct ftrace_ret_stack *ret_stack;
59 unsigned long ret_addr; 60 unsigned long ret_addr;
60 int index = task->curr_ret_stack;
61 61
62 if (addr != (unsigned long)return_to_handler) 62 if (addr != (unsigned long)return_to_handler)
63 return; 63 return;
64 64
65 if (!task->ret_stack || index < *graph) 65 if (!task->ret_stack)
66 return; 66 return;
67 67
68 index -= *graph; 68 ret_stack = ftrace_graph_get_ret_stack(task, *graph);
69 ret_addr = task->ret_stack[index].ret; 69 if (!ret_stack)
70 return;
71
72 ret_addr = ret_stack->ret;
70 73
71 ops->address(data, ret_addr, 1); 74 ops->address(data, ret_addr, 1);
72 75
diff --git a/arch/sh/kernel/dwarf.c b/arch/sh/kernel/dwarf.c
index 9e1d26c8a0c4..c5b426506d16 100644
--- a/arch/sh/kernel/dwarf.c
+++ b/arch/sh/kernel/dwarf.c
@@ -605,17 +605,18 @@ struct dwarf_frame *dwarf_unwind_stack(unsigned long pc,
605 * expected to find the real return address. 605 * expected to find the real return address.
606 */ 606 */
607 if (pc == (unsigned long)&return_to_handler) { 607 if (pc == (unsigned long)&return_to_handler) {
608 int index = current->curr_ret_stack; 608 struct ftrace_ret_stack *ret_stack;
609 609
610 ret_stack = ftrace_graph_get_ret_stack(current, 0);
611 if (ret_stack)
612 pc = ret_stack->ret;
610 /* 613 /*
611 * We currently have no way of tracking how many 614 * We currently have no way of tracking how many
612 * return_to_handler()'s we've seen. If there is more 615 * return_to_handler()'s we've seen. If there is more
613 * than one patched return address on our stack, 616 * than one patched return address on our stack,
614 * complain loudly. 617 * complain loudly.
615 */ 618 */
616 WARN_ON(index > 0); 619 WARN_ON(ftrace_graph_get_ret_stack(current, 1);
617
618 pc = current->ret_stack[index].ret;
619 } 620 }
620#endif 621#endif
621 622
diff --git a/arch/sparc/kernel/perf_event.c b/arch/sparc/kernel/perf_event.c
index 47c871394ccb..6de7c684c29f 100644
--- a/arch/sparc/kernel/perf_event.c
+++ b/arch/sparc/kernel/perf_event.c
@@ -1767,9 +1767,11 @@ void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry,
1767 perf_callchain_store(entry, pc); 1767 perf_callchain_store(entry, pc);
1768#ifdef CONFIG_FUNCTION_GRAPH_TRACER 1768#ifdef CONFIG_FUNCTION_GRAPH_TRACER
1769 if ((pc + 8UL) == (unsigned long) &return_to_handler) { 1769 if ((pc + 8UL) == (unsigned long) &return_to_handler) {
1770 int index = current->curr_ret_stack; 1770 struct ftrace_ret_stack *ret_stack;
1771 if (current->ret_stack && index >= graph) { 1771 ret_stack = ftrace_graph_get_ret_stack(current,
1772 pc = current->ret_stack[index - graph].ret; 1772 graph);
1773 if (ret_stack) {
1774 pc = ret_stack->ret;
1773 perf_callchain_store(entry, pc); 1775 perf_callchain_store(entry, pc);
1774 graph++; 1776 graph++;
1775 } 1777 }
diff --git a/arch/sparc/kernel/stacktrace.c b/arch/sparc/kernel/stacktrace.c
index be4c14cccc05..dd654e651500 100644
--- a/arch/sparc/kernel/stacktrace.c
+++ b/arch/sparc/kernel/stacktrace.c
@@ -57,9 +57,11 @@ static void __save_stack_trace(struct thread_info *tp,
57 trace->entries[trace->nr_entries++] = pc; 57 trace->entries[trace->nr_entries++] = pc;
58#ifdef CONFIG_FUNCTION_GRAPH_TRACER 58#ifdef CONFIG_FUNCTION_GRAPH_TRACER
59 if ((pc + 8UL) == (unsigned long) &return_to_handler) { 59 if ((pc + 8UL) == (unsigned long) &return_to_handler) {
60 int index = t->curr_ret_stack; 60 struct ftrace_ret_stack *ret_stack;
61 if (t->ret_stack && index >= graph) { 61 ret_stack = ftrace_graph_get_ret_stack(t,
62 pc = t->ret_stack[index - graph].ret; 62 graph);
63 if (ret_stack) {
64 pc = ret_stack->ret;
63 if (trace->nr_entries < 65 if (trace->nr_entries <
64 trace->max_entries) 66 trace->max_entries)
65 trace->entries[trace->nr_entries++] = pc; 67 trace->entries[trace->nr_entries++] = pc;
diff --git a/arch/sparc/kernel/traps_64.c b/arch/sparc/kernel/traps_64.c
index aa624ed79db1..0cd02a64a451 100644
--- a/arch/sparc/kernel/traps_64.c
+++ b/arch/sparc/kernel/traps_64.c
@@ -2502,9 +2502,10 @@ void show_stack(struct task_struct *tsk, unsigned long *_ksp)
2502 printk(" [%016lx] %pS\n", pc, (void *) pc); 2502 printk(" [%016lx] %pS\n", pc, (void *) pc);
2503#ifdef CONFIG_FUNCTION_GRAPH_TRACER 2503#ifdef CONFIG_FUNCTION_GRAPH_TRACER
2504 if ((pc + 8UL) == (unsigned long) &return_to_handler) { 2504 if ((pc + 8UL) == (unsigned long) &return_to_handler) {
2505 int index = tsk->curr_ret_stack; 2505 struct ftrace_ret_stack *ret_stack;
2506 if (tsk->ret_stack && index >= graph) { 2506 ret_stack = ftrace_graph_get_ret_stack(tsk, graph);
2507 pc = tsk->ret_stack[index - graph].ret; 2507 if (ret_stack) {
2508 pc = ret_stack->ret;
2508 printk(" [%016lx] %pS\n", pc, (void *) pc); 2509 printk(" [%016lx] %pS\n", pc, (void *) pc);
2509 graph++; 2510 graph++;
2510 } 2511 }
diff --git a/arch/x86/kernel/ftrace.c b/arch/x86/kernel/ftrace.c
index 7ee8067cbf45..8257a59704ae 100644
--- a/arch/x86/kernel/ftrace.c
+++ b/arch/x86/kernel/ftrace.c
@@ -733,18 +733,20 @@ union ftrace_op_code_union {
733 } __attribute__((packed)); 733 } __attribute__((packed));
734}; 734};
735 735
736#define RET_SIZE 1
737
736static unsigned long 738static unsigned long
737create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size) 739create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size)
738{ 740{
739 unsigned const char *jmp;
740 unsigned long start_offset; 741 unsigned long start_offset;
741 unsigned long end_offset; 742 unsigned long end_offset;
742 unsigned long op_offset; 743 unsigned long op_offset;
743 unsigned long offset; 744 unsigned long offset;
744 unsigned long size; 745 unsigned long size;
745 unsigned long ip; 746 unsigned long retq;
746 unsigned long *ptr; 747 unsigned long *ptr;
747 void *trampoline; 748 void *trampoline;
749 void *ip;
748 /* 48 8b 15 <offset> is movq <offset>(%rip), %rdx */ 750 /* 48 8b 15 <offset> is movq <offset>(%rip), %rdx */
749 unsigned const char op_ref[] = { 0x48, 0x8b, 0x15 }; 751 unsigned const char op_ref[] = { 0x48, 0x8b, 0x15 };
750 union ftrace_op_code_union op_ptr; 752 union ftrace_op_code_union op_ptr;
@@ -764,27 +766,27 @@ create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size)
764 766
765 /* 767 /*
766 * Allocate enough size to store the ftrace_caller code, 768 * Allocate enough size to store the ftrace_caller code,
767 * the jmp to ftrace_epilogue, as well as the address of 769 * the iret , as well as the address of the ftrace_ops this
768 * the ftrace_ops this trampoline is used for. 770 * trampoline is used for.
769 */ 771 */
770 trampoline = alloc_tramp(size + MCOUNT_INSN_SIZE + sizeof(void *)); 772 trampoline = alloc_tramp(size + RET_SIZE + sizeof(void *));
771 if (!trampoline) 773 if (!trampoline)
772 return 0; 774 return 0;
773 775
774 *tramp_size = size + MCOUNT_INSN_SIZE + sizeof(void *); 776 *tramp_size = size + RET_SIZE + sizeof(void *);
775 777
776 /* Copy ftrace_caller onto the trampoline memory */ 778 /* Copy ftrace_caller onto the trampoline memory */
777 ret = probe_kernel_read(trampoline, (void *)start_offset, size); 779 ret = probe_kernel_read(trampoline, (void *)start_offset, size);
778 if (WARN_ON(ret < 0)) { 780 if (WARN_ON(ret < 0))
779 tramp_free(trampoline, *tramp_size); 781 goto fail;
780 return 0;
781 }
782 782
783 ip = (unsigned long)trampoline + size; 783 ip = trampoline + size;
784 784
785 /* The trampoline ends with a jmp to ftrace_epilogue */ 785 /* The trampoline ends with ret(q) */
786 jmp = ftrace_jmp_replace(ip, (unsigned long)ftrace_epilogue); 786 retq = (unsigned long)ftrace_stub;
787 memcpy(trampoline + size, jmp, MCOUNT_INSN_SIZE); 787 ret = probe_kernel_read(ip, (void *)retq, RET_SIZE);
788 if (WARN_ON(ret < 0))
789 goto fail;
788 790
789 /* 791 /*
790 * The address of the ftrace_ops that is used for this trampoline 792 * The address of the ftrace_ops that is used for this trampoline
@@ -794,17 +796,15 @@ create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size)
794 * the global function_trace_op variable. 796 * the global function_trace_op variable.
795 */ 797 */
796 798
797 ptr = (unsigned long *)(trampoline + size + MCOUNT_INSN_SIZE); 799 ptr = (unsigned long *)(trampoline + size + RET_SIZE);
798 *ptr = (unsigned long)ops; 800 *ptr = (unsigned long)ops;
799 801
800 op_offset -= start_offset; 802 op_offset -= start_offset;
801 memcpy(&op_ptr, trampoline + op_offset, OP_REF_SIZE); 803 memcpy(&op_ptr, trampoline + op_offset, OP_REF_SIZE);
802 804
803 /* Are we pointing to the reference? */ 805 /* Are we pointing to the reference? */
804 if (WARN_ON(memcmp(op_ptr.op, op_ref, 3) != 0)) { 806 if (WARN_ON(memcmp(op_ptr.op, op_ref, 3) != 0))
805 tramp_free(trampoline, *tramp_size); 807 goto fail;
806 return 0;
807 }
808 808
809 /* Load the contents of ptr into the callback parameter */ 809 /* Load the contents of ptr into the callback parameter */
810 offset = (unsigned long)ptr; 810 offset = (unsigned long)ptr;
@@ -819,6 +819,9 @@ create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size)
819 ops->flags |= FTRACE_OPS_FL_ALLOC_TRAMP; 819 ops->flags |= FTRACE_OPS_FL_ALLOC_TRAMP;
820 820
821 return (unsigned long)trampoline; 821 return (unsigned long)trampoline;
822fail:
823 tramp_free(trampoline, *tramp_size);
824 return 0;
822} 825}
823 826
824static unsigned long calc_trampoline_call_offset(bool save_regs) 827static unsigned long calc_trampoline_call_offset(bool save_regs)
diff --git a/arch/x86/kernel/ftrace_64.S b/arch/x86/kernel/ftrace_64.S
index 91b2cff4b79a..75f2b36b41a6 100644
--- a/arch/x86/kernel/ftrace_64.S
+++ b/arch/x86/kernel/ftrace_64.S
@@ -171,9 +171,6 @@ GLOBAL(ftrace_call)
171 restore_mcount_regs 171 restore_mcount_regs
172 172
173 /* 173 /*
174 * The copied trampoline must call ftrace_epilogue as it
175 * still may need to call the function graph tracer.
176 *
177 * The code up to this label is copied into trampolines so 174 * The code up to this label is copied into trampolines so
178 * think twice before adding any new code or changing the 175 * think twice before adding any new code or changing the
179 * layout here. 176 * layout here.
@@ -185,7 +182,10 @@ GLOBAL(ftrace_graph_call)
185 jmp ftrace_stub 182 jmp ftrace_stub
186#endif 183#endif
187 184
188/* This is weak to keep gas from relaxing the jumps */ 185/*
186 * This is weak to keep gas from relaxing the jumps.
187 * It is also used to copy the retq for trampolines.
188 */
189WEAK(ftrace_stub) 189WEAK(ftrace_stub)
190 retq 190 retq
191ENDPROC(ftrace_caller) 191ENDPROC(ftrace_caller)
diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h
index 5c990e891d6a..730876187344 100644
--- a/include/linux/ftrace.h
+++ b/include/linux/ftrace.h
@@ -389,6 +389,7 @@ enum {
389 FTRACE_UPDATE_TRACE_FUNC = (1 << 2), 389 FTRACE_UPDATE_TRACE_FUNC = (1 << 2),
390 FTRACE_START_FUNC_RET = (1 << 3), 390 FTRACE_START_FUNC_RET = (1 << 3),
391 FTRACE_STOP_FUNC_RET = (1 << 4), 391 FTRACE_STOP_FUNC_RET = (1 << 4),
392 FTRACE_MAY_SLEEP = (1 << 5),
392}; 393};
393 394
394/* 395/*
@@ -752,6 +753,11 @@ typedef int (*trace_func_graph_ent_t)(struct ftrace_graph_ent *); /* entry */
752 753
753#ifdef CONFIG_FUNCTION_GRAPH_TRACER 754#ifdef CONFIG_FUNCTION_GRAPH_TRACER
754 755
756struct fgraph_ops {
757 trace_func_graph_ent_t entryfunc;
758 trace_func_graph_ret_t retfunc;
759};
760
755/* 761/*
756 * Stack of return addresses for functions 762 * Stack of return addresses for functions
757 * of a thread. 763 * of a thread.
@@ -783,6 +789,9 @@ extern int
783function_graph_enter(unsigned long ret, unsigned long func, 789function_graph_enter(unsigned long ret, unsigned long func,
784 unsigned long frame_pointer, unsigned long *retp); 790 unsigned long frame_pointer, unsigned long *retp);
785 791
792struct ftrace_ret_stack *
793ftrace_graph_get_ret_stack(struct task_struct *task, int idx);
794
786unsigned long ftrace_graph_ret_addr(struct task_struct *task, int *idx, 795unsigned long ftrace_graph_ret_addr(struct task_struct *task, int *idx,
787 unsigned long ret, unsigned long *retp); 796 unsigned long ret, unsigned long *retp);
788 797
@@ -793,11 +802,11 @@ unsigned long ftrace_graph_ret_addr(struct task_struct *task, int *idx,
793 */ 802 */
794#define __notrace_funcgraph notrace 803#define __notrace_funcgraph notrace
795 804
796#define FTRACE_NOTRACE_DEPTH 65536
797#define FTRACE_RETFUNC_DEPTH 50 805#define FTRACE_RETFUNC_DEPTH 50
798#define FTRACE_RETSTACK_ALLOC_SIZE 32 806#define FTRACE_RETSTACK_ALLOC_SIZE 32
799extern int register_ftrace_graph(trace_func_graph_ret_t retfunc, 807
800 trace_func_graph_ent_t entryfunc); 808extern int register_ftrace_graph(struct fgraph_ops *ops);
809extern void unregister_ftrace_graph(struct fgraph_ops *ops);
801 810
802extern bool ftrace_graph_is_dead(void); 811extern bool ftrace_graph_is_dead(void);
803extern void ftrace_graph_stop(void); 812extern void ftrace_graph_stop(void);
@@ -806,17 +815,10 @@ extern void ftrace_graph_stop(void);
806extern trace_func_graph_ret_t ftrace_graph_return; 815extern trace_func_graph_ret_t ftrace_graph_return;
807extern trace_func_graph_ent_t ftrace_graph_entry; 816extern trace_func_graph_ent_t ftrace_graph_entry;
808 817
809extern void unregister_ftrace_graph(void);
810
811extern void ftrace_graph_init_task(struct task_struct *t); 818extern void ftrace_graph_init_task(struct task_struct *t);
812extern void ftrace_graph_exit_task(struct task_struct *t); 819extern void ftrace_graph_exit_task(struct task_struct *t);
813extern void ftrace_graph_init_idle_task(struct task_struct *t, int cpu); 820extern void ftrace_graph_init_idle_task(struct task_struct *t, int cpu);
814 821
815static inline int task_curr_ret_stack(struct task_struct *t)
816{
817 return t->curr_ret_stack;
818}
819
820static inline void pause_graph_tracing(void) 822static inline void pause_graph_tracing(void)
821{ 823{
822 atomic_inc(&current->tracing_graph_pause); 824 atomic_inc(&current->tracing_graph_pause);
@@ -834,17 +836,9 @@ static inline void ftrace_graph_init_task(struct task_struct *t) { }
834static inline void ftrace_graph_exit_task(struct task_struct *t) { } 836static inline void ftrace_graph_exit_task(struct task_struct *t) { }
835static inline void ftrace_graph_init_idle_task(struct task_struct *t, int cpu) { } 837static inline void ftrace_graph_init_idle_task(struct task_struct *t, int cpu) { }
836 838
837static inline int register_ftrace_graph(trace_func_graph_ret_t retfunc, 839/* Define as macros as fgraph_ops may not be defined */
838 trace_func_graph_ent_t entryfunc) 840#define register_ftrace_graph(ops) ({ -1; })
839{ 841#define unregister_ftrace_graph(ops) do { } while (0)
840 return -1;
841}
842static inline void unregister_ftrace_graph(void) { }
843
844static inline int task_curr_ret_stack(struct task_struct *tsk)
845{
846 return -1;
847}
848 842
849static inline unsigned long 843static inline unsigned long
850ftrace_graph_ret_addr(struct task_struct *task, int *idx, unsigned long ret, 844ftrace_graph_ret_addr(struct task_struct *task, int *idx, unsigned long ret,
diff --git a/include/linux/ring_buffer.h b/include/linux/ring_buffer.h
index 0940fda59872..5b9ae62272bb 100644
--- a/include/linux/ring_buffer.h
+++ b/include/linux/ring_buffer.h
@@ -97,7 +97,7 @@ __ring_buffer_alloc(unsigned long size, unsigned flags, struct lock_class_key *k
97 __ring_buffer_alloc((size), (flags), &__key); \ 97 __ring_buffer_alloc((size), (flags), &__key); \
98}) 98})
99 99
100int ring_buffer_wait(struct ring_buffer *buffer, int cpu, bool full); 100int ring_buffer_wait(struct ring_buffer *buffer, int cpu, int full);
101__poll_t ring_buffer_poll_wait(struct ring_buffer *buffer, int cpu, 101__poll_t ring_buffer_poll_wait(struct ring_buffer *buffer, int cpu,
102 struct file *filp, poll_table *poll_table); 102 struct file *filp, poll_table *poll_table);
103 103
@@ -189,6 +189,8 @@ bool ring_buffer_time_stamp_abs(struct ring_buffer *buffer);
189 189
190size_t ring_buffer_page_len(void *page); 190size_t ring_buffer_page_len(void *page);
191 191
192size_t ring_buffer_nr_pages(struct ring_buffer *buffer, int cpu);
193size_t ring_buffer_nr_dirty_pages(struct ring_buffer *buffer, int cpu);
192 194
193void *ring_buffer_alloc_read_page(struct ring_buffer *buffer, int cpu); 195void *ring_buffer_alloc_read_page(struct ring_buffer *buffer, int cpu);
194void ring_buffer_free_read_page(struct ring_buffer *buffer, int cpu, void *data); 196void ring_buffer_free_read_page(struct ring_buffer *buffer, int cpu, void *data);
diff --git a/include/linux/string.h b/include/linux/string.h
index 27d0482e5e05..7927b875f80c 100644
--- a/include/linux/string.h
+++ b/include/linux/string.h
@@ -456,4 +456,24 @@ static inline void memcpy_and_pad(void *dest, size_t dest_len,
456 memcpy(dest, src, dest_len); 456 memcpy(dest, src, dest_len);
457} 457}
458 458
459/**
460 * str_has_prefix - Test if a string has a given prefix
461 * @str: The string to test
462 * @prefix: The string to see if @str starts with
463 *
464 * A common way to test a prefix of a string is to do:
465 * strncmp(str, prefix, sizeof(prefix) - 1)
466 *
467 * But this can lead to bugs due to typos, or if prefix is a pointer
468 * and not a constant. Instead use str_has_prefix().
469 *
470 * Returns: 0 if @str does not start with @prefix
471 strlen(@prefix) if @str does start with @prefix
472 */
473static __always_inline size_t str_has_prefix(const char *str, const char *prefix)
474{
475 size_t len = strlen(prefix);
476 return strncmp(str, prefix, len) == 0 ? len : 0;
477}
478
459#endif /* _LINUX_STRING_H_ */ 479#endif /* _LINUX_STRING_H_ */
diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig
index 5e3de28c7677..fa8b1fe824f3 100644
--- a/kernel/trace/Kconfig
+++ b/kernel/trace/Kconfig
@@ -461,6 +461,7 @@ config KPROBE_EVENTS
461 bool "Enable kprobes-based dynamic events" 461 bool "Enable kprobes-based dynamic events"
462 select TRACING 462 select TRACING
463 select PROBE_EVENTS 463 select PROBE_EVENTS
464 select DYNAMIC_EVENTS
464 default y 465 default y
465 help 466 help
466 This allows the user to add tracing events (similar to tracepoints) 467 This allows the user to add tracing events (similar to tracepoints)
@@ -500,6 +501,7 @@ config UPROBE_EVENTS
500 depends on PERF_EVENTS 501 depends on PERF_EVENTS
501 select UPROBES 502 select UPROBES
502 select PROBE_EVENTS 503 select PROBE_EVENTS
504 select DYNAMIC_EVENTS
503 select TRACING 505 select TRACING
504 default y 506 default y
505 help 507 help
@@ -518,6 +520,9 @@ config BPF_EVENTS
518 help 520 help
519 This allows the user to attach BPF programs to kprobe events. 521 This allows the user to attach BPF programs to kprobe events.
520 522
523config DYNAMIC_EVENTS
524 def_bool n
525
521config PROBE_EVENTS 526config PROBE_EVENTS
522 def_bool n 527 def_bool n
523 528
@@ -630,6 +635,7 @@ config HIST_TRIGGERS
630 depends on ARCH_HAVE_NMI_SAFE_CMPXCHG 635 depends on ARCH_HAVE_NMI_SAFE_CMPXCHG
631 select TRACING_MAP 636 select TRACING_MAP
632 select TRACING 637 select TRACING
638 select DYNAMIC_EVENTS
633 default n 639 default n
634 help 640 help
635 Hist triggers allow one or more arbitrary trace event fields 641 Hist triggers allow one or more arbitrary trace event fields
diff --git a/kernel/trace/Makefile b/kernel/trace/Makefile
index f81dadbc7c4a..c2b2148bb1d2 100644
--- a/kernel/trace/Makefile
+++ b/kernel/trace/Makefile
@@ -57,6 +57,7 @@ obj-$(CONFIG_MMIOTRACE) += trace_mmiotrace.o
57obj-$(CONFIG_FUNCTION_GRAPH_TRACER) += trace_functions_graph.o 57obj-$(CONFIG_FUNCTION_GRAPH_TRACER) += trace_functions_graph.o
58obj-$(CONFIG_TRACE_BRANCH_PROFILING) += trace_branch.o 58obj-$(CONFIG_TRACE_BRANCH_PROFILING) += trace_branch.o
59obj-$(CONFIG_BLK_DEV_IO_TRACE) += blktrace.o 59obj-$(CONFIG_BLK_DEV_IO_TRACE) += blktrace.o
60obj-$(CONFIG_FUNCTION_GRAPH_TRACER) += fgraph.o
60ifeq ($(CONFIG_BLOCK),y) 61ifeq ($(CONFIG_BLOCK),y)
61obj-$(CONFIG_EVENT_TRACING) += blktrace.o 62obj-$(CONFIG_EVENT_TRACING) += blktrace.o
62endif 63endif
@@ -78,6 +79,7 @@ endif
78ifeq ($(CONFIG_TRACING),y) 79ifeq ($(CONFIG_TRACING),y)
79obj-$(CONFIG_KGDB_KDB) += trace_kdb.o 80obj-$(CONFIG_KGDB_KDB) += trace_kdb.o
80endif 81endif
82obj-$(CONFIG_DYNAMIC_EVENTS) += trace_dynevent.o
81obj-$(CONFIG_PROBE_EVENTS) += trace_probe.o 83obj-$(CONFIG_PROBE_EVENTS) += trace_probe.o
82obj-$(CONFIG_UPROBE_EVENTS) += trace_uprobe.o 84obj-$(CONFIG_UPROBE_EVENTS) += trace_uprobe.o
83 85
diff --git a/kernel/trace/fgraph.c b/kernel/trace/fgraph.c
new file mode 100644
index 000000000000..8dfd5021b933
--- /dev/null
+++ b/kernel/trace/fgraph.c
@@ -0,0 +1,626 @@
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Infrastructure to took into function calls and returns.
4 * Copyright (c) 2008-2009 Frederic Weisbecker <fweisbec@gmail.com>
5 * Mostly borrowed from function tracer which
6 * is Copyright (c) Steven Rostedt <srostedt@redhat.com>
7 *
8 * Highly modified by Steven Rostedt (VMware).
9 */
10#include <linux/suspend.h>
11#include <linux/ftrace.h>
12#include <linux/slab.h>
13
14#include <trace/events/sched.h>
15
16#include "ftrace_internal.h"
17
18#ifdef CONFIG_DYNAMIC_FTRACE
19#define ASSIGN_OPS_HASH(opsname, val) \
20 .func_hash = val, \
21 .local_hash.regex_lock = __MUTEX_INITIALIZER(opsname.local_hash.regex_lock),
22#else
23#define ASSIGN_OPS_HASH(opsname, val)
24#endif
25
26static bool kill_ftrace_graph;
27int ftrace_graph_active;
28
29/* Both enabled by default (can be cleared by function_graph tracer flags */
30static bool fgraph_sleep_time = true;
31
32/**
33 * ftrace_graph_is_dead - returns true if ftrace_graph_stop() was called
34 *
35 * ftrace_graph_stop() is called when a severe error is detected in
36 * the function graph tracing. This function is called by the critical
37 * paths of function graph to keep those paths from doing any more harm.
38 */
39bool ftrace_graph_is_dead(void)
40{
41 return kill_ftrace_graph;
42}
43
44/**
45 * ftrace_graph_stop - set to permanently disable function graph tracincg
46 *
47 * In case of an error int function graph tracing, this is called
48 * to try to keep function graph tracing from causing any more harm.
49 * Usually this is pretty severe and this is called to try to at least
50 * get a warning out to the user.
51 */
52void ftrace_graph_stop(void)
53{
54 kill_ftrace_graph = true;
55}
56
57/* Add a function return address to the trace stack on thread info.*/
58static int
59ftrace_push_return_trace(unsigned long ret, unsigned long func,
60 unsigned long frame_pointer, unsigned long *retp)
61{
62 unsigned long long calltime;
63 int index;
64
65 if (unlikely(ftrace_graph_is_dead()))
66 return -EBUSY;
67
68 if (!current->ret_stack)
69 return -EBUSY;
70
71 /*
72 * We must make sure the ret_stack is tested before we read
73 * anything else.
74 */
75 smp_rmb();
76
77 /* The return trace stack is full */
78 if (current->curr_ret_stack == FTRACE_RETFUNC_DEPTH - 1) {
79 atomic_inc(&current->trace_overrun);
80 return -EBUSY;
81 }
82
83 calltime = trace_clock_local();
84
85 index = ++current->curr_ret_stack;
86 barrier();
87 current->ret_stack[index].ret = ret;
88 current->ret_stack[index].func = func;
89 current->ret_stack[index].calltime = calltime;
90#ifdef HAVE_FUNCTION_GRAPH_FP_TEST
91 current->ret_stack[index].fp = frame_pointer;
92#endif
93#ifdef HAVE_FUNCTION_GRAPH_RET_ADDR_PTR
94 current->ret_stack[index].retp = retp;
95#endif
96 return 0;
97}
98
99int function_graph_enter(unsigned long ret, unsigned long func,
100 unsigned long frame_pointer, unsigned long *retp)
101{
102 struct ftrace_graph_ent trace;
103
104 trace.func = func;
105 trace.depth = ++current->curr_ret_depth;
106
107 if (ftrace_push_return_trace(ret, func, frame_pointer, retp))
108 goto out;
109
110 /* Only trace if the calling function expects to */
111 if (!ftrace_graph_entry(&trace))
112 goto out_ret;
113
114 return 0;
115 out_ret:
116 current->curr_ret_stack--;
117 out:
118 current->curr_ret_depth--;
119 return -EBUSY;
120}
121
122/* Retrieve a function return address to the trace stack on thread info.*/
123static void
124ftrace_pop_return_trace(struct ftrace_graph_ret *trace, unsigned long *ret,
125 unsigned long frame_pointer)
126{
127 int index;
128
129 index = current->curr_ret_stack;
130
131 if (unlikely(index < 0 || index >= FTRACE_RETFUNC_DEPTH)) {
132 ftrace_graph_stop();
133 WARN_ON(1);
134 /* Might as well panic, otherwise we have no where to go */
135 *ret = (unsigned long)panic;
136 return;
137 }
138
139#ifdef HAVE_FUNCTION_GRAPH_FP_TEST
140 /*
141 * The arch may choose to record the frame pointer used
142 * and check it here to make sure that it is what we expect it
143 * to be. If gcc does not set the place holder of the return
144 * address in the frame pointer, and does a copy instead, then
145 * the function graph trace will fail. This test detects this
146 * case.
147 *
148 * Currently, x86_32 with optimize for size (-Os) makes the latest
149 * gcc do the above.
150 *
151 * Note, -mfentry does not use frame pointers, and this test
152 * is not needed if CC_USING_FENTRY is set.
153 */
154 if (unlikely(current->ret_stack[index].fp != frame_pointer)) {
155 ftrace_graph_stop();
156 WARN(1, "Bad frame pointer: expected %lx, received %lx\n"
157 " from func %ps return to %lx\n",
158 current->ret_stack[index].fp,
159 frame_pointer,
160 (void *)current->ret_stack[index].func,
161 current->ret_stack[index].ret);
162 *ret = (unsigned long)panic;
163 return;
164 }
165#endif
166
167 *ret = current->ret_stack[index].ret;
168 trace->func = current->ret_stack[index].func;
169 trace->calltime = current->ret_stack[index].calltime;
170 trace->overrun = atomic_read(&current->trace_overrun);
171 trace->depth = current->curr_ret_depth--;
172 /*
173 * We still want to trace interrupts coming in if
174 * max_depth is set to 1. Make sure the decrement is
175 * seen before ftrace_graph_return.
176 */
177 barrier();
178}
179
180/*
181 * Hibernation protection.
182 * The state of the current task is too much unstable during
183 * suspend/restore to disk. We want to protect against that.
184 */
185static int
186ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state,
187 void *unused)
188{
189 switch (state) {
190 case PM_HIBERNATION_PREPARE:
191 pause_graph_tracing();
192 break;
193
194 case PM_POST_HIBERNATION:
195 unpause_graph_tracing();
196 break;
197 }
198 return NOTIFY_DONE;
199}
200
201static struct notifier_block ftrace_suspend_notifier = {
202 .notifier_call = ftrace_suspend_notifier_call,
203};
204
205/*
206 * Send the trace to the ring-buffer.
207 * @return the original return address.
208 */
209unsigned long ftrace_return_to_handler(unsigned long frame_pointer)
210{
211 struct ftrace_graph_ret trace;
212 unsigned long ret;
213
214 ftrace_pop_return_trace(&trace, &ret, frame_pointer);
215 trace.rettime = trace_clock_local();
216 ftrace_graph_return(&trace);
217 /*
218 * The ftrace_graph_return() may still access the current
219 * ret_stack structure, we need to make sure the update of
220 * curr_ret_stack is after that.
221 */
222 barrier();
223 current->curr_ret_stack--;
224
225 if (unlikely(!ret)) {
226 ftrace_graph_stop();
227 WARN_ON(1);
228 /* Might as well panic. What else to do? */
229 ret = (unsigned long)panic;
230 }
231
232 return ret;
233}
234
235/**
236 * ftrace_graph_get_ret_stack - return the entry of the shadow stack
237 * @task: The task to read the shadow stack from
238 * @idx: Index down the shadow stack
239 *
240 * Return the ret_struct on the shadow stack of the @task at the
241 * call graph at @idx starting with zero. If @idx is zero, it
242 * will return the last saved ret_stack entry. If it is greater than
243 * zero, it will return the corresponding ret_stack for the depth
244 * of saved return addresses.
245 */
246struct ftrace_ret_stack *
247ftrace_graph_get_ret_stack(struct task_struct *task, int idx)
248{
249 idx = task->curr_ret_stack - idx;
250
251 if (idx >= 0 && idx <= task->curr_ret_stack)
252 return &task->ret_stack[idx];
253
254 return NULL;
255}
256
257/**
258 * ftrace_graph_ret_addr - convert a potentially modified stack return address
259 * to its original value
260 *
261 * This function can be called by stack unwinding code to convert a found stack
262 * return address ('ret') to its original value, in case the function graph
263 * tracer has modified it to be 'return_to_handler'. If the address hasn't
264 * been modified, the unchanged value of 'ret' is returned.
265 *
266 * 'idx' is a state variable which should be initialized by the caller to zero
267 * before the first call.
268 *
269 * 'retp' is a pointer to the return address on the stack. It's ignored if
270 * the arch doesn't have HAVE_FUNCTION_GRAPH_RET_ADDR_PTR defined.
271 */
272#ifdef HAVE_FUNCTION_GRAPH_RET_ADDR_PTR
273unsigned long ftrace_graph_ret_addr(struct task_struct *task, int *idx,
274 unsigned long ret, unsigned long *retp)
275{
276 int index = task->curr_ret_stack;
277 int i;
278
279 if (ret != (unsigned long)return_to_handler)
280 return ret;
281
282 if (index < 0)
283 return ret;
284
285 for (i = 0; i <= index; i++)
286 if (task->ret_stack[i].retp == retp)
287 return task->ret_stack[i].ret;
288
289 return ret;
290}
291#else /* !HAVE_FUNCTION_GRAPH_RET_ADDR_PTR */
292unsigned long ftrace_graph_ret_addr(struct task_struct *task, int *idx,
293 unsigned long ret, unsigned long *retp)
294{
295 int task_idx;
296
297 if (ret != (unsigned long)return_to_handler)
298 return ret;
299
300 task_idx = task->curr_ret_stack;
301
302 if (!task->ret_stack || task_idx < *idx)
303 return ret;
304
305 task_idx -= *idx;
306 (*idx)++;
307
308 return task->ret_stack[task_idx].ret;
309}
310#endif /* HAVE_FUNCTION_GRAPH_RET_ADDR_PTR */
311
312static struct ftrace_ops graph_ops = {
313 .func = ftrace_stub,
314 .flags = FTRACE_OPS_FL_RECURSION_SAFE |
315 FTRACE_OPS_FL_INITIALIZED |
316 FTRACE_OPS_FL_PID |
317 FTRACE_OPS_FL_STUB,
318#ifdef FTRACE_GRAPH_TRAMP_ADDR
319 .trampoline = FTRACE_GRAPH_TRAMP_ADDR,
320 /* trampoline_size is only needed for dynamically allocated tramps */
321#endif
322 ASSIGN_OPS_HASH(graph_ops, &global_ops.local_hash)
323};
324
325void ftrace_graph_sleep_time_control(bool enable)
326{
327 fgraph_sleep_time = enable;
328}
329
330int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)
331{
332 return 0;
333}
334
335/* The callbacks that hook a function */
336trace_func_graph_ret_t ftrace_graph_return =
337 (trace_func_graph_ret_t)ftrace_stub;
338trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
339static trace_func_graph_ent_t __ftrace_graph_entry = ftrace_graph_entry_stub;
340
341/* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
342static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
343{
344 int i;
345 int ret = 0;
346 int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
347 struct task_struct *g, *t;
348
349 for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
350 ret_stack_list[i] =
351 kmalloc_array(FTRACE_RETFUNC_DEPTH,
352 sizeof(struct ftrace_ret_stack),
353 GFP_KERNEL);
354 if (!ret_stack_list[i]) {
355 start = 0;
356 end = i;
357 ret = -ENOMEM;
358 goto free;
359 }
360 }
361
362 read_lock(&tasklist_lock);
363 do_each_thread(g, t) {
364 if (start == end) {
365 ret = -EAGAIN;
366 goto unlock;
367 }
368
369 if (t->ret_stack == NULL) {
370 atomic_set(&t->tracing_graph_pause, 0);
371 atomic_set(&t->trace_overrun, 0);
372 t->curr_ret_stack = -1;
373 t->curr_ret_depth = -1;
374 /* Make sure the tasks see the -1 first: */
375 smp_wmb();
376 t->ret_stack = ret_stack_list[start++];
377 }
378 } while_each_thread(g, t);
379
380unlock:
381 read_unlock(&tasklist_lock);
382free:
383 for (i = start; i < end; i++)
384 kfree(ret_stack_list[i]);
385 return ret;
386}
387
388static void
389ftrace_graph_probe_sched_switch(void *ignore, bool preempt,
390 struct task_struct *prev, struct task_struct *next)
391{
392 unsigned long long timestamp;
393 int index;
394
395 /*
396 * Does the user want to count the time a function was asleep.
397 * If so, do not update the time stamps.
398 */
399 if (fgraph_sleep_time)
400 return;
401
402 timestamp = trace_clock_local();
403
404 prev->ftrace_timestamp = timestamp;
405
406 /* only process tasks that we timestamped */
407 if (!next->ftrace_timestamp)
408 return;
409
410 /*
411 * Update all the counters in next to make up for the
412 * time next was sleeping.
413 */
414 timestamp -= next->ftrace_timestamp;
415
416 for (index = next->curr_ret_stack; index >= 0; index--)
417 next->ret_stack[index].calltime += timestamp;
418}
419
420static int ftrace_graph_entry_test(struct ftrace_graph_ent *trace)
421{
422 if (!ftrace_ops_test(&global_ops, trace->func, NULL))
423 return 0;
424 return __ftrace_graph_entry(trace);
425}
426
427/*
428 * The function graph tracer should only trace the functions defined
429 * by set_ftrace_filter and set_ftrace_notrace. If another function
430 * tracer ops is registered, the graph tracer requires testing the
431 * function against the global ops, and not just trace any function
432 * that any ftrace_ops registered.
433 */
434void update_function_graph_func(void)
435{
436 struct ftrace_ops *op;
437 bool do_test = false;
438
439 /*
440 * The graph and global ops share the same set of functions
441 * to test. If any other ops is on the list, then
442 * the graph tracing needs to test if its the function
443 * it should call.
444 */
445 do_for_each_ftrace_op(op, ftrace_ops_list) {
446 if (op != &global_ops && op != &graph_ops &&
447 op != &ftrace_list_end) {
448 do_test = true;
449 /* in double loop, break out with goto */
450 goto out;
451 }
452 } while_for_each_ftrace_op(op);
453 out:
454 if (do_test)
455 ftrace_graph_entry = ftrace_graph_entry_test;
456 else
457 ftrace_graph_entry = __ftrace_graph_entry;
458}
459
460static DEFINE_PER_CPU(struct ftrace_ret_stack *, idle_ret_stack);
461
462static void
463graph_init_task(struct task_struct *t, struct ftrace_ret_stack *ret_stack)
464{
465 atomic_set(&t->tracing_graph_pause, 0);
466 atomic_set(&t->trace_overrun, 0);
467 t->ftrace_timestamp = 0;
468 /* make curr_ret_stack visible before we add the ret_stack */
469 smp_wmb();
470 t->ret_stack = ret_stack;
471}
472
473/*
474 * Allocate a return stack for the idle task. May be the first
475 * time through, or it may be done by CPU hotplug online.
476 */
477void ftrace_graph_init_idle_task(struct task_struct *t, int cpu)
478{
479 t->curr_ret_stack = -1;
480 t->curr_ret_depth = -1;
481 /*
482 * The idle task has no parent, it either has its own
483 * stack or no stack at all.
484 */
485 if (t->ret_stack)
486 WARN_ON(t->ret_stack != per_cpu(idle_ret_stack, cpu));
487
488 if (ftrace_graph_active) {
489 struct ftrace_ret_stack *ret_stack;
490
491 ret_stack = per_cpu(idle_ret_stack, cpu);
492 if (!ret_stack) {
493 ret_stack =
494 kmalloc_array(FTRACE_RETFUNC_DEPTH,
495 sizeof(struct ftrace_ret_stack),
496 GFP_KERNEL);
497 if (!ret_stack)
498 return;
499 per_cpu(idle_ret_stack, cpu) = ret_stack;
500 }
501 graph_init_task(t, ret_stack);
502 }
503}
504
505/* Allocate a return stack for newly created task */
506void ftrace_graph_init_task(struct task_struct *t)
507{
508 /* Make sure we do not use the parent ret_stack */
509 t->ret_stack = NULL;
510 t->curr_ret_stack = -1;
511 t->curr_ret_depth = -1;
512
513 if (ftrace_graph_active) {
514 struct ftrace_ret_stack *ret_stack;
515
516 ret_stack = kmalloc_array(FTRACE_RETFUNC_DEPTH,
517 sizeof(struct ftrace_ret_stack),
518 GFP_KERNEL);
519 if (!ret_stack)
520 return;
521 graph_init_task(t, ret_stack);
522 }
523}
524
525void ftrace_graph_exit_task(struct task_struct *t)
526{
527 struct ftrace_ret_stack *ret_stack = t->ret_stack;
528
529 t->ret_stack = NULL;
530 /* NULL must become visible to IRQs before we free it: */
531 barrier();
532
533 kfree(ret_stack);
534}
535
536/* Allocate a return stack for each task */
537static int start_graph_tracing(void)
538{
539 struct ftrace_ret_stack **ret_stack_list;
540 int ret, cpu;
541
542 ret_stack_list = kmalloc_array(FTRACE_RETSTACK_ALLOC_SIZE,
543 sizeof(struct ftrace_ret_stack *),
544 GFP_KERNEL);
545
546 if (!ret_stack_list)
547 return -ENOMEM;
548
549 /* The cpu_boot init_task->ret_stack will never be freed */
550 for_each_online_cpu(cpu) {
551 if (!idle_task(cpu)->ret_stack)
552 ftrace_graph_init_idle_task(idle_task(cpu), cpu);
553 }
554
555 do {
556 ret = alloc_retstack_tasklist(ret_stack_list);
557 } while (ret == -EAGAIN);
558
559 if (!ret) {
560 ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
561 if (ret)
562 pr_info("ftrace_graph: Couldn't activate tracepoint"
563 " probe to kernel_sched_switch\n");
564 }
565
566 kfree(ret_stack_list);
567 return ret;
568}
569
570int register_ftrace_graph(struct fgraph_ops *gops)
571{
572 int ret = 0;
573
574 mutex_lock(&ftrace_lock);
575
576 /* we currently allow only one tracer registered at a time */
577 if (ftrace_graph_active) {
578 ret = -EBUSY;
579 goto out;
580 }
581
582 register_pm_notifier(&ftrace_suspend_notifier);
583
584 ftrace_graph_active++;
585 ret = start_graph_tracing();
586 if (ret) {
587 ftrace_graph_active--;
588 goto out;
589 }
590
591 ftrace_graph_return = gops->retfunc;
592
593 /*
594 * Update the indirect function to the entryfunc, and the
595 * function that gets called to the entry_test first. Then
596 * call the update fgraph entry function to determine if
597 * the entryfunc should be called directly or not.
598 */
599 __ftrace_graph_entry = gops->entryfunc;
600 ftrace_graph_entry = ftrace_graph_entry_test;
601 update_function_graph_func();
602
603 ret = ftrace_startup(&graph_ops, FTRACE_START_FUNC_RET);
604out:
605 mutex_unlock(&ftrace_lock);
606 return ret;
607}
608
609void unregister_ftrace_graph(struct fgraph_ops *gops)
610{
611 mutex_lock(&ftrace_lock);
612
613 if (unlikely(!ftrace_graph_active))
614 goto out;
615
616 ftrace_graph_active--;
617 ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
618 ftrace_graph_entry = ftrace_graph_entry_stub;
619 __ftrace_graph_entry = ftrace_graph_entry_stub;
620 ftrace_shutdown(&graph_ops, FTRACE_STOP_FUNC_RET);
621 unregister_pm_notifier(&ftrace_suspend_notifier);
622 unregister_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
623
624 out:
625 mutex_unlock(&ftrace_lock);
626}
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
index f0ff24173a0b..aac7847c0214 100644
--- a/kernel/trace/ftrace.c
+++ b/kernel/trace/ftrace.c
@@ -19,7 +19,6 @@
19#include <linux/sched/task.h> 19#include <linux/sched/task.h>
20#include <linux/kallsyms.h> 20#include <linux/kallsyms.h>
21#include <linux/seq_file.h> 21#include <linux/seq_file.h>
22#include <linux/suspend.h>
23#include <linux/tracefs.h> 22#include <linux/tracefs.h>
24#include <linux/hardirq.h> 23#include <linux/hardirq.h>
25#include <linux/kthread.h> 24#include <linux/kthread.h>
@@ -40,6 +39,7 @@
40#include <asm/sections.h> 39#include <asm/sections.h>
41#include <asm/setup.h> 40#include <asm/setup.h>
42 41
42#include "ftrace_internal.h"
43#include "trace_output.h" 43#include "trace_output.h"
44#include "trace_stat.h" 44#include "trace_stat.h"
45 45
@@ -77,7 +77,12 @@
77#define ASSIGN_OPS_HASH(opsname, val) 77#define ASSIGN_OPS_HASH(opsname, val)
78#endif 78#endif
79 79
80static struct ftrace_ops ftrace_list_end __read_mostly = { 80enum {
81 FTRACE_MODIFY_ENABLE_FL = (1 << 0),
82 FTRACE_MODIFY_MAY_SLEEP_FL = (1 << 1),
83};
84
85struct ftrace_ops ftrace_list_end __read_mostly = {
81 .func = ftrace_stub, 86 .func = ftrace_stub,
82 .flags = FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_STUB, 87 .flags = FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_STUB,
83 INIT_OPS_HASH(ftrace_list_end) 88 INIT_OPS_HASH(ftrace_list_end)
@@ -112,11 +117,11 @@ static void ftrace_update_trampoline(struct ftrace_ops *ops);
112 */ 117 */
113static int ftrace_disabled __read_mostly; 118static int ftrace_disabled __read_mostly;
114 119
115static DEFINE_MUTEX(ftrace_lock); 120DEFINE_MUTEX(ftrace_lock);
116 121
117static struct ftrace_ops __rcu *ftrace_ops_list __read_mostly = &ftrace_list_end; 122struct ftrace_ops __rcu *ftrace_ops_list __read_mostly = &ftrace_list_end;
118ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub; 123ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
119static struct ftrace_ops global_ops; 124struct ftrace_ops global_ops;
120 125
121#if ARCH_SUPPORTS_FTRACE_OPS 126#if ARCH_SUPPORTS_FTRACE_OPS
122static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip, 127static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
@@ -127,26 +132,6 @@ static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip);
127#define ftrace_ops_list_func ((ftrace_func_t)ftrace_ops_no_ops) 132#define ftrace_ops_list_func ((ftrace_func_t)ftrace_ops_no_ops)
128#endif 133#endif
129 134
130/*
131 * Traverse the ftrace_global_list, invoking all entries. The reason that we
132 * can use rcu_dereference_raw_notrace() is that elements removed from this list
133 * are simply leaked, so there is no need to interact with a grace-period
134 * mechanism. The rcu_dereference_raw_notrace() calls are needed to handle
135 * concurrent insertions into the ftrace_global_list.
136 *
137 * Silly Alpha and silly pointer-speculation compiler optimizations!
138 */
139#define do_for_each_ftrace_op(op, list) \
140 op = rcu_dereference_raw_notrace(list); \
141 do
142
143/*
144 * Optimized for just a single item in the list (as that is the normal case).
145 */
146#define while_for_each_ftrace_op(op) \
147 while (likely(op = rcu_dereference_raw_notrace((op)->next)) && \
148 unlikely((op) != &ftrace_list_end))
149
150static inline void ftrace_ops_init(struct ftrace_ops *ops) 135static inline void ftrace_ops_init(struct ftrace_ops *ops)
151{ 136{
152#ifdef CONFIG_DYNAMIC_FTRACE 137#ifdef CONFIG_DYNAMIC_FTRACE
@@ -186,18 +171,6 @@ static void ftrace_sync_ipi(void *data)
186 smp_rmb(); 171 smp_rmb();
187} 172}
188 173
189#ifdef CONFIG_FUNCTION_GRAPH_TRACER
190static void update_function_graph_func(void);
191
192/* Both enabled by default (can be cleared by function_graph tracer flags */
193static bool fgraph_sleep_time = true;
194static bool fgraph_graph_time = true;
195
196#else
197static inline void update_function_graph_func(void) { }
198#endif
199
200
201static ftrace_func_t ftrace_ops_get_list_func(struct ftrace_ops *ops) 174static ftrace_func_t ftrace_ops_get_list_func(struct ftrace_ops *ops)
202{ 175{
203 /* 176 /*
@@ -334,7 +307,7 @@ static int remove_ftrace_ops(struct ftrace_ops __rcu **list,
334 307
335static void ftrace_update_trampoline(struct ftrace_ops *ops); 308static void ftrace_update_trampoline(struct ftrace_ops *ops);
336 309
337static int __register_ftrace_function(struct ftrace_ops *ops) 310int __register_ftrace_function(struct ftrace_ops *ops)
338{ 311{
339 if (ops->flags & FTRACE_OPS_FL_DELETED) 312 if (ops->flags & FTRACE_OPS_FL_DELETED)
340 return -EINVAL; 313 return -EINVAL;
@@ -375,7 +348,7 @@ static int __register_ftrace_function(struct ftrace_ops *ops)
375 return 0; 348 return 0;
376} 349}
377 350
378static int __unregister_ftrace_function(struct ftrace_ops *ops) 351int __unregister_ftrace_function(struct ftrace_ops *ops)
379{ 352{
380 int ret; 353 int ret;
381 354
@@ -815,9 +788,16 @@ function_profile_call(unsigned long ip, unsigned long parent_ip,
815} 788}
816 789
817#ifdef CONFIG_FUNCTION_GRAPH_TRACER 790#ifdef CONFIG_FUNCTION_GRAPH_TRACER
791static bool fgraph_graph_time = true;
792
793void ftrace_graph_graph_time_control(bool enable)
794{
795 fgraph_graph_time = enable;
796}
797
818static int profile_graph_entry(struct ftrace_graph_ent *trace) 798static int profile_graph_entry(struct ftrace_graph_ent *trace)
819{ 799{
820 int index = current->curr_ret_stack; 800 struct ftrace_ret_stack *ret_stack;
821 801
822 function_profile_call(trace->func, 0, NULL, NULL); 802 function_profile_call(trace->func, 0, NULL, NULL);
823 803
@@ -825,14 +805,16 @@ static int profile_graph_entry(struct ftrace_graph_ent *trace)
825 if (!current->ret_stack) 805 if (!current->ret_stack)
826 return 0; 806 return 0;
827 807
828 if (index >= 0 && index < FTRACE_RETFUNC_DEPTH) 808 ret_stack = ftrace_graph_get_ret_stack(current, 0);
829 current->ret_stack[index].subtime = 0; 809 if (ret_stack)
810 ret_stack->subtime = 0;
830 811
831 return 1; 812 return 1;
832} 813}
833 814
834static void profile_graph_return(struct ftrace_graph_ret *trace) 815static void profile_graph_return(struct ftrace_graph_ret *trace)
835{ 816{
817 struct ftrace_ret_stack *ret_stack;
836 struct ftrace_profile_stat *stat; 818 struct ftrace_profile_stat *stat;
837 unsigned long long calltime; 819 unsigned long long calltime;
838 struct ftrace_profile *rec; 820 struct ftrace_profile *rec;
@@ -850,16 +832,15 @@ static void profile_graph_return(struct ftrace_graph_ret *trace)
850 calltime = trace->rettime - trace->calltime; 832 calltime = trace->rettime - trace->calltime;
851 833
852 if (!fgraph_graph_time) { 834 if (!fgraph_graph_time) {
853 int index;
854
855 index = current->curr_ret_stack;
856 835
857 /* Append this call time to the parent time to subtract */ 836 /* Append this call time to the parent time to subtract */
858 if (index) 837 ret_stack = ftrace_graph_get_ret_stack(current, 1);
859 current->ret_stack[index - 1].subtime += calltime; 838 if (ret_stack)
839 ret_stack->subtime += calltime;
860 840
861 if (current->ret_stack[index].subtime < calltime) 841 ret_stack = ftrace_graph_get_ret_stack(current, 0);
862 calltime -= current->ret_stack[index].subtime; 842 if (ret_stack && ret_stack->subtime < calltime)
843 calltime -= ret_stack->subtime;
863 else 844 else
864 calltime = 0; 845 calltime = 0;
865 } 846 }
@@ -874,15 +855,19 @@ static void profile_graph_return(struct ftrace_graph_ret *trace)
874 local_irq_restore(flags); 855 local_irq_restore(flags);
875} 856}
876 857
858static struct fgraph_ops fprofiler_ops = {
859 .entryfunc = &profile_graph_entry,
860 .retfunc = &profile_graph_return,
861};
862
877static int register_ftrace_profiler(void) 863static int register_ftrace_profiler(void)
878{ 864{
879 return register_ftrace_graph(&profile_graph_return, 865 return register_ftrace_graph(&fprofiler_ops);
880 &profile_graph_entry);
881} 866}
882 867
883static void unregister_ftrace_profiler(void) 868static void unregister_ftrace_profiler(void)
884{ 869{
885 unregister_ftrace_graph(); 870 unregister_ftrace_graph(&fprofiler_ops);
886} 871}
887#else 872#else
888static struct ftrace_ops ftrace_profile_ops __read_mostly = { 873static struct ftrace_ops ftrace_profile_ops __read_mostly = {
@@ -1021,12 +1006,6 @@ static __init void ftrace_profile_tracefs(struct dentry *d_tracer)
1021} 1006}
1022#endif /* CONFIG_FUNCTION_PROFILER */ 1007#endif /* CONFIG_FUNCTION_PROFILER */
1023 1008
1024#ifdef CONFIG_FUNCTION_GRAPH_TRACER
1025static int ftrace_graph_active;
1026#else
1027# define ftrace_graph_active 0
1028#endif
1029
1030#ifdef CONFIG_DYNAMIC_FTRACE 1009#ifdef CONFIG_DYNAMIC_FTRACE
1031 1010
1032static struct ftrace_ops *removed_ops; 1011static struct ftrace_ops *removed_ops;
@@ -1067,7 +1046,7 @@ static const struct ftrace_hash empty_hash = {
1067}; 1046};
1068#define EMPTY_HASH ((struct ftrace_hash *)&empty_hash) 1047#define EMPTY_HASH ((struct ftrace_hash *)&empty_hash)
1069 1048
1070static struct ftrace_ops global_ops = { 1049struct ftrace_ops global_ops = {
1071 .func = ftrace_stub, 1050 .func = ftrace_stub,
1072 .local_hash.notrace_hash = EMPTY_HASH, 1051 .local_hash.notrace_hash = EMPTY_HASH,
1073 .local_hash.filter_hash = EMPTY_HASH, 1052 .local_hash.filter_hash = EMPTY_HASH,
@@ -1503,7 +1482,7 @@ static bool hash_contains_ip(unsigned long ip,
1503 * This needs to be called with preemption disabled as 1482 * This needs to be called with preemption disabled as
1504 * the hashes are freed with call_rcu(). 1483 * the hashes are freed with call_rcu().
1505 */ 1484 */
1506static int 1485int
1507ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs) 1486ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs)
1508{ 1487{
1509 struct ftrace_ops_hash hash; 1488 struct ftrace_ops_hash hash;
@@ -2415,10 +2394,12 @@ __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
2415 return -1; /* unknow ftrace bug */ 2394 return -1; /* unknow ftrace bug */
2416} 2395}
2417 2396
2418void __weak ftrace_replace_code(int enable) 2397void __weak ftrace_replace_code(int mod_flags)
2419{ 2398{
2420 struct dyn_ftrace *rec; 2399 struct dyn_ftrace *rec;
2421 struct ftrace_page *pg; 2400 struct ftrace_page *pg;
2401 int enable = mod_flags & FTRACE_MODIFY_ENABLE_FL;
2402 int schedulable = mod_flags & FTRACE_MODIFY_MAY_SLEEP_FL;
2422 int failed; 2403 int failed;
2423 2404
2424 if (unlikely(ftrace_disabled)) 2405 if (unlikely(ftrace_disabled))
@@ -2435,6 +2416,8 @@ void __weak ftrace_replace_code(int enable)
2435 /* Stop processing */ 2416 /* Stop processing */
2436 return; 2417 return;
2437 } 2418 }
2419 if (schedulable)
2420 cond_resched();
2438 } while_for_each_ftrace_rec(); 2421 } while_for_each_ftrace_rec();
2439} 2422}
2440 2423
@@ -2548,8 +2531,12 @@ int __weak ftrace_arch_code_modify_post_process(void)
2548void ftrace_modify_all_code(int command) 2531void ftrace_modify_all_code(int command)
2549{ 2532{
2550 int update = command & FTRACE_UPDATE_TRACE_FUNC; 2533 int update = command & FTRACE_UPDATE_TRACE_FUNC;
2534 int mod_flags = 0;
2551 int err = 0; 2535 int err = 0;
2552 2536
2537 if (command & FTRACE_MAY_SLEEP)
2538 mod_flags = FTRACE_MODIFY_MAY_SLEEP_FL;
2539
2553 /* 2540 /*
2554 * If the ftrace_caller calls a ftrace_ops func directly, 2541 * If the ftrace_caller calls a ftrace_ops func directly,
2555 * we need to make sure that it only traces functions it 2542 * we need to make sure that it only traces functions it
@@ -2567,9 +2554,9 @@ void ftrace_modify_all_code(int command)
2567 } 2554 }
2568 2555
2569 if (command & FTRACE_UPDATE_CALLS) 2556 if (command & FTRACE_UPDATE_CALLS)
2570 ftrace_replace_code(1); 2557 ftrace_replace_code(mod_flags | FTRACE_MODIFY_ENABLE_FL);
2571 else if (command & FTRACE_DISABLE_CALLS) 2558 else if (command & FTRACE_DISABLE_CALLS)
2572 ftrace_replace_code(0); 2559 ftrace_replace_code(mod_flags);
2573 2560
2574 if (update && ftrace_trace_function != ftrace_ops_list_func) { 2561 if (update && ftrace_trace_function != ftrace_ops_list_func) {
2575 function_trace_op = set_function_trace_op; 2562 function_trace_op = set_function_trace_op;
@@ -2682,7 +2669,7 @@ static void ftrace_startup_all(int command)
2682 update_all_ops = false; 2669 update_all_ops = false;
2683} 2670}
2684 2671
2685static int ftrace_startup(struct ftrace_ops *ops, int command) 2672int ftrace_startup(struct ftrace_ops *ops, int command)
2686{ 2673{
2687 int ret; 2674 int ret;
2688 2675
@@ -2724,7 +2711,7 @@ static int ftrace_startup(struct ftrace_ops *ops, int command)
2724 return 0; 2711 return 0;
2725} 2712}
2726 2713
2727static int ftrace_shutdown(struct ftrace_ops *ops, int command) 2714int ftrace_shutdown(struct ftrace_ops *ops, int command)
2728{ 2715{
2729 int ret; 2716 int ret;
2730 2717
@@ -6178,7 +6165,7 @@ void ftrace_init_trace_array(struct trace_array *tr)
6178} 6165}
6179#else 6166#else
6180 6167
6181static struct ftrace_ops global_ops = { 6168struct ftrace_ops global_ops = {
6182 .func = ftrace_stub, 6169 .func = ftrace_stub,
6183 .flags = FTRACE_OPS_FL_RECURSION_SAFE | 6170 .flags = FTRACE_OPS_FL_RECURSION_SAFE |
6184 FTRACE_OPS_FL_INITIALIZED | 6171 FTRACE_OPS_FL_INITIALIZED |
@@ -6195,31 +6182,10 @@ core_initcall(ftrace_nodyn_init);
6195static inline int ftrace_init_dyn_tracefs(struct dentry *d_tracer) { return 0; } 6182static inline int ftrace_init_dyn_tracefs(struct dentry *d_tracer) { return 0; }
6196static inline void ftrace_startup_enable(int command) { } 6183static inline void ftrace_startup_enable(int command) { }
6197static inline void ftrace_startup_all(int command) { } 6184static inline void ftrace_startup_all(int command) { }
6198/* Keep as macros so we do not need to define the commands */
6199# define ftrace_startup(ops, command) \
6200 ({ \
6201 int ___ret = __register_ftrace_function(ops); \
6202 if (!___ret) \
6203 (ops)->flags |= FTRACE_OPS_FL_ENABLED; \
6204 ___ret; \
6205 })
6206# define ftrace_shutdown(ops, command) \
6207 ({ \
6208 int ___ret = __unregister_ftrace_function(ops); \
6209 if (!___ret) \
6210 (ops)->flags &= ~FTRACE_OPS_FL_ENABLED; \
6211 ___ret; \
6212 })
6213 6185
6214# define ftrace_startup_sysctl() do { } while (0) 6186# define ftrace_startup_sysctl() do { } while (0)
6215# define ftrace_shutdown_sysctl() do { } while (0) 6187# define ftrace_shutdown_sysctl() do { } while (0)
6216 6188
6217static inline int
6218ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs)
6219{
6220 return 1;
6221}
6222
6223static void ftrace_update_trampoline(struct ftrace_ops *ops) 6189static void ftrace_update_trampoline(struct ftrace_ops *ops)
6224{ 6190{
6225} 6191}
@@ -6746,353 +6712,3 @@ ftrace_enable_sysctl(struct ctl_table *table, int write,
6746 mutex_unlock(&ftrace_lock); 6712 mutex_unlock(&ftrace_lock);
6747 return ret; 6713 return ret;
6748} 6714}
6749
6750#ifdef CONFIG_FUNCTION_GRAPH_TRACER
6751
6752static struct ftrace_ops graph_ops = {
6753 .func = ftrace_stub,
6754 .flags = FTRACE_OPS_FL_RECURSION_SAFE |
6755 FTRACE_OPS_FL_INITIALIZED |
6756 FTRACE_OPS_FL_PID |
6757 FTRACE_OPS_FL_STUB,
6758#ifdef FTRACE_GRAPH_TRAMP_ADDR
6759 .trampoline = FTRACE_GRAPH_TRAMP_ADDR,
6760 /* trampoline_size is only needed for dynamically allocated tramps */
6761#endif
6762 ASSIGN_OPS_HASH(graph_ops, &global_ops.local_hash)
6763};
6764
6765void ftrace_graph_sleep_time_control(bool enable)
6766{
6767 fgraph_sleep_time = enable;
6768}
6769
6770void ftrace_graph_graph_time_control(bool enable)
6771{
6772 fgraph_graph_time = enable;
6773}
6774
6775int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)
6776{
6777 return 0;
6778}
6779
6780/* The callbacks that hook a function */
6781trace_func_graph_ret_t ftrace_graph_return =
6782 (trace_func_graph_ret_t)ftrace_stub;
6783trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
6784static trace_func_graph_ent_t __ftrace_graph_entry = ftrace_graph_entry_stub;
6785
6786/* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
6787static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
6788{
6789 int i;
6790 int ret = 0;
6791 int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
6792 struct task_struct *g, *t;
6793
6794 for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
6795 ret_stack_list[i] =
6796 kmalloc_array(FTRACE_RETFUNC_DEPTH,
6797 sizeof(struct ftrace_ret_stack),
6798 GFP_KERNEL);
6799 if (!ret_stack_list[i]) {
6800 start = 0;
6801 end = i;
6802 ret = -ENOMEM;
6803 goto free;
6804 }
6805 }
6806
6807 read_lock(&tasklist_lock);
6808 do_each_thread(g, t) {
6809 if (start == end) {
6810 ret = -EAGAIN;
6811 goto unlock;
6812 }
6813
6814 if (t->ret_stack == NULL) {
6815 atomic_set(&t->tracing_graph_pause, 0);
6816 atomic_set(&t->trace_overrun, 0);
6817 t->curr_ret_stack = -1;
6818 t->curr_ret_depth = -1;
6819 /* Make sure the tasks see the -1 first: */
6820 smp_wmb();
6821 t->ret_stack = ret_stack_list[start++];
6822 }
6823 } while_each_thread(g, t);
6824
6825unlock:
6826 read_unlock(&tasklist_lock);
6827free:
6828 for (i = start; i < end; i++)
6829 kfree(ret_stack_list[i]);
6830 return ret;
6831}
6832
6833static void
6834ftrace_graph_probe_sched_switch(void *ignore, bool preempt,
6835 struct task_struct *prev, struct task_struct *next)
6836{
6837 unsigned long long timestamp;
6838 int index;
6839
6840 /*
6841 * Does the user want to count the time a function was asleep.
6842 * If so, do not update the time stamps.
6843 */
6844 if (fgraph_sleep_time)
6845 return;
6846
6847 timestamp = trace_clock_local();
6848
6849 prev->ftrace_timestamp = timestamp;
6850
6851 /* only process tasks that we timestamped */
6852 if (!next->ftrace_timestamp)
6853 return;
6854
6855 /*
6856 * Update all the counters in next to make up for the
6857 * time next was sleeping.
6858 */
6859 timestamp -= next->ftrace_timestamp;
6860
6861 for (index = next->curr_ret_stack; index >= 0; index--)
6862 next->ret_stack[index].calltime += timestamp;
6863}
6864
6865/* Allocate a return stack for each task */
6866static int start_graph_tracing(void)
6867{
6868 struct ftrace_ret_stack **ret_stack_list;
6869 int ret, cpu;
6870
6871 ret_stack_list = kmalloc_array(FTRACE_RETSTACK_ALLOC_SIZE,
6872 sizeof(struct ftrace_ret_stack *),
6873 GFP_KERNEL);
6874
6875 if (!ret_stack_list)
6876 return -ENOMEM;
6877
6878 /* The cpu_boot init_task->ret_stack will never be freed */
6879 for_each_online_cpu(cpu) {
6880 if (!idle_task(cpu)->ret_stack)
6881 ftrace_graph_init_idle_task(idle_task(cpu), cpu);
6882 }
6883
6884 do {
6885 ret = alloc_retstack_tasklist(ret_stack_list);
6886 } while (ret == -EAGAIN);
6887
6888 if (!ret) {
6889 ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
6890 if (ret)
6891 pr_info("ftrace_graph: Couldn't activate tracepoint"
6892 " probe to kernel_sched_switch\n");
6893 }
6894
6895 kfree(ret_stack_list);
6896 return ret;
6897}
6898
6899/*
6900 * Hibernation protection.
6901 * The state of the current task is too much unstable during
6902 * suspend/restore to disk. We want to protect against that.
6903 */
6904static int
6905ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state,
6906 void *unused)
6907{
6908 switch (state) {
6909 case PM_HIBERNATION_PREPARE:
6910 pause_graph_tracing();
6911 break;
6912
6913 case PM_POST_HIBERNATION:
6914 unpause_graph_tracing();
6915 break;
6916 }
6917 return NOTIFY_DONE;
6918}
6919
6920static int ftrace_graph_entry_test(struct ftrace_graph_ent *trace)
6921{
6922 if (!ftrace_ops_test(&global_ops, trace->func, NULL))
6923 return 0;
6924 return __ftrace_graph_entry(trace);
6925}
6926
6927/*
6928 * The function graph tracer should only trace the functions defined
6929 * by set_ftrace_filter and set_ftrace_notrace. If another function
6930 * tracer ops is registered, the graph tracer requires testing the
6931 * function against the global ops, and not just trace any function
6932 * that any ftrace_ops registered.
6933 */
6934static void update_function_graph_func(void)
6935{
6936 struct ftrace_ops *op;
6937 bool do_test = false;
6938
6939 /*
6940 * The graph and global ops share the same set of functions
6941 * to test. If any other ops is on the list, then
6942 * the graph tracing needs to test if its the function
6943 * it should call.
6944 */
6945 do_for_each_ftrace_op(op, ftrace_ops_list) {
6946 if (op != &global_ops && op != &graph_ops &&
6947 op != &ftrace_list_end) {
6948 do_test = true;
6949 /* in double loop, break out with goto */
6950 goto out;
6951 }
6952 } while_for_each_ftrace_op(op);
6953 out:
6954 if (do_test)
6955 ftrace_graph_entry = ftrace_graph_entry_test;
6956 else
6957 ftrace_graph_entry = __ftrace_graph_entry;
6958}
6959
6960static struct notifier_block ftrace_suspend_notifier = {
6961 .notifier_call = ftrace_suspend_notifier_call,
6962};
6963
6964int register_ftrace_graph(trace_func_graph_ret_t retfunc,
6965 trace_func_graph_ent_t entryfunc)
6966{
6967 int ret = 0;
6968
6969 mutex_lock(&ftrace_lock);
6970
6971 /* we currently allow only one tracer registered at a time */
6972 if (ftrace_graph_active) {
6973 ret = -EBUSY;
6974 goto out;
6975 }
6976
6977 register_pm_notifier(&ftrace_suspend_notifier);
6978
6979 ftrace_graph_active++;
6980 ret = start_graph_tracing();
6981 if (ret) {
6982 ftrace_graph_active--;
6983 goto out;
6984 }
6985
6986 ftrace_graph_return = retfunc;
6987
6988 /*
6989 * Update the indirect function to the entryfunc, and the
6990 * function that gets called to the entry_test first. Then
6991 * call the update fgraph entry function to determine if
6992 * the entryfunc should be called directly or not.
6993 */
6994 __ftrace_graph_entry = entryfunc;
6995 ftrace_graph_entry = ftrace_graph_entry_test;
6996 update_function_graph_func();
6997
6998 ret = ftrace_startup(&graph_ops, FTRACE_START_FUNC_RET);
6999out:
7000 mutex_unlock(&ftrace_lock);
7001 return ret;
7002}
7003
7004void unregister_ftrace_graph(void)
7005{
7006 mutex_lock(&ftrace_lock);
7007
7008 if (unlikely(!ftrace_graph_active))
7009 goto out;
7010
7011 ftrace_graph_active--;
7012 ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
7013 ftrace_graph_entry = ftrace_graph_entry_stub;
7014 __ftrace_graph_entry = ftrace_graph_entry_stub;
7015 ftrace_shutdown(&graph_ops, FTRACE_STOP_FUNC_RET);
7016 unregister_pm_notifier(&ftrace_suspend_notifier);
7017 unregister_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
7018
7019 out:
7020 mutex_unlock(&ftrace_lock);
7021}
7022
7023static DEFINE_PER_CPU(struct ftrace_ret_stack *, idle_ret_stack);
7024
7025static void
7026graph_init_task(struct task_struct *t, struct ftrace_ret_stack *ret_stack)
7027{
7028 atomic_set(&t->tracing_graph_pause, 0);
7029 atomic_set(&t->trace_overrun, 0);
7030 t->ftrace_timestamp = 0;
7031 /* make curr_ret_stack visible before we add the ret_stack */
7032 smp_wmb();
7033 t->ret_stack = ret_stack;
7034}
7035
7036/*
7037 * Allocate a return stack for the idle task. May be the first
7038 * time through, or it may be done by CPU hotplug online.
7039 */
7040void ftrace_graph_init_idle_task(struct task_struct *t, int cpu)
7041{
7042 t->curr_ret_stack = -1;
7043 t->curr_ret_depth = -1;
7044 /*
7045 * The idle task has no parent, it either has its own
7046 * stack or no stack at all.
7047 */
7048 if (t->ret_stack)
7049 WARN_ON(t->ret_stack != per_cpu(idle_ret_stack, cpu));
7050
7051 if (ftrace_graph_active) {
7052 struct ftrace_ret_stack *ret_stack;
7053
7054 ret_stack = per_cpu(idle_ret_stack, cpu);
7055 if (!ret_stack) {
7056 ret_stack =
7057 kmalloc_array(FTRACE_RETFUNC_DEPTH,
7058 sizeof(struct ftrace_ret_stack),
7059 GFP_KERNEL);
7060 if (!ret_stack)
7061 return;
7062 per_cpu(idle_ret_stack, cpu) = ret_stack;
7063 }
7064 graph_init_task(t, ret_stack);
7065 }
7066}
7067
7068/* Allocate a return stack for newly created task */
7069void ftrace_graph_init_task(struct task_struct *t)
7070{
7071 /* Make sure we do not use the parent ret_stack */
7072 t->ret_stack = NULL;
7073 t->curr_ret_stack = -1;
7074 t->curr_ret_depth = -1;
7075
7076 if (ftrace_graph_active) {
7077 struct ftrace_ret_stack *ret_stack;
7078
7079 ret_stack = kmalloc_array(FTRACE_RETFUNC_DEPTH,
7080 sizeof(struct ftrace_ret_stack),
7081 GFP_KERNEL);
7082 if (!ret_stack)
7083 return;
7084 graph_init_task(t, ret_stack);
7085 }
7086}
7087
7088void ftrace_graph_exit_task(struct task_struct *t)
7089{
7090 struct ftrace_ret_stack *ret_stack = t->ret_stack;
7091
7092 t->ret_stack = NULL;
7093 /* NULL must become visible to IRQs before we free it: */
7094 barrier();
7095
7096 kfree(ret_stack);
7097}
7098#endif
diff --git a/kernel/trace/ftrace_internal.h b/kernel/trace/ftrace_internal.h
new file mode 100644
index 000000000000..0515a2096f90
--- /dev/null
+++ b/kernel/trace/ftrace_internal.h
@@ -0,0 +1,75 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _LINUX_KERNEL_FTRACE_INTERNAL_H
3#define _LINUX_KERNEL_FTRACE_INTERNAL_H
4
5#ifdef CONFIG_FUNCTION_TRACER
6
7/*
8 * Traverse the ftrace_global_list, invoking all entries. The reason that we
9 * can use rcu_dereference_raw_notrace() is that elements removed from this list
10 * are simply leaked, so there is no need to interact with a grace-period
11 * mechanism. The rcu_dereference_raw_notrace() calls are needed to handle
12 * concurrent insertions into the ftrace_global_list.
13 *
14 * Silly Alpha and silly pointer-speculation compiler optimizations!
15 */
16#define do_for_each_ftrace_op(op, list) \
17 op = rcu_dereference_raw_notrace(list); \
18 do
19
20/*
21 * Optimized for just a single item in the list (as that is the normal case).
22 */
23#define while_for_each_ftrace_op(op) \
24 while (likely(op = rcu_dereference_raw_notrace((op)->next)) && \
25 unlikely((op) != &ftrace_list_end))
26
27extern struct ftrace_ops __rcu *ftrace_ops_list;
28extern struct ftrace_ops ftrace_list_end;
29extern struct mutex ftrace_lock;
30extern struct ftrace_ops global_ops;
31
32#ifdef CONFIG_DYNAMIC_FTRACE
33
34int ftrace_startup(struct ftrace_ops *ops, int command);
35int ftrace_shutdown(struct ftrace_ops *ops, int command);
36int ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs);
37
38#else /* !CONFIG_DYNAMIC_FTRACE */
39
40int __register_ftrace_function(struct ftrace_ops *ops);
41int __unregister_ftrace_function(struct ftrace_ops *ops);
42/* Keep as macros so we do not need to define the commands */
43# define ftrace_startup(ops, command) \
44 ({ \
45 int ___ret = __register_ftrace_function(ops); \
46 if (!___ret) \
47 (ops)->flags |= FTRACE_OPS_FL_ENABLED; \
48 ___ret; \
49 })
50# define ftrace_shutdown(ops, command) \
51 ({ \
52 int ___ret = __unregister_ftrace_function(ops); \
53 if (!___ret) \
54 (ops)->flags &= ~FTRACE_OPS_FL_ENABLED; \
55 ___ret; \
56 })
57static inline int
58ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs)
59{
60 return 1;
61}
62#endif /* CONFIG_DYNAMIC_FTRACE */
63
64#ifdef CONFIG_FUNCTION_GRAPH_TRACER
65extern int ftrace_graph_active;
66void update_function_graph_func(void);
67#else /* !CONFIG_FUNCTION_GRAPH_TRACER */
68# define ftrace_graph_active 0
69static inline void update_function_graph_func(void) { }
70#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
71
72#else /* !CONFIG_FUNCTION_TRACER */
73#endif /* CONFIG_FUNCTION_TRACER */
74
75#endif
diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c
index 4f3247a53259..06e864a334bb 100644
--- a/kernel/trace/ring_buffer.c
+++ b/kernel/trace/ring_buffer.c
@@ -487,6 +487,10 @@ struct ring_buffer_per_cpu {
487 local_t dropped_events; 487 local_t dropped_events;
488 local_t committing; 488 local_t committing;
489 local_t commits; 489 local_t commits;
490 local_t pages_touched;
491 local_t pages_read;
492 long last_pages_touch;
493 size_t shortest_full;
490 unsigned long read; 494 unsigned long read;
491 unsigned long read_bytes; 495 unsigned long read_bytes;
492 u64 write_stamp; 496 u64 write_stamp;
@@ -529,6 +533,41 @@ struct ring_buffer_iter {
529 u64 read_stamp; 533 u64 read_stamp;
530}; 534};
531 535
536/**
537 * ring_buffer_nr_pages - get the number of buffer pages in the ring buffer
538 * @buffer: The ring_buffer to get the number of pages from
539 * @cpu: The cpu of the ring_buffer to get the number of pages from
540 *
541 * Returns the number of pages used by a per_cpu buffer of the ring buffer.
542 */
543size_t ring_buffer_nr_pages(struct ring_buffer *buffer, int cpu)
544{
545 return buffer->buffers[cpu]->nr_pages;
546}
547
548/**
549 * ring_buffer_nr_pages_dirty - get the number of used pages in the ring buffer
550 * @buffer: The ring_buffer to get the number of pages from
551 * @cpu: The cpu of the ring_buffer to get the number of pages from
552 *
553 * Returns the number of pages that have content in the ring buffer.
554 */
555size_t ring_buffer_nr_dirty_pages(struct ring_buffer *buffer, int cpu)
556{
557 size_t read;
558 size_t cnt;
559
560 read = local_read(&buffer->buffers[cpu]->pages_read);
561 cnt = local_read(&buffer->buffers[cpu]->pages_touched);
562 /* The reader can read an empty page, but not more than that */
563 if (cnt < read) {
564 WARN_ON_ONCE(read > cnt + 1);
565 return 0;
566 }
567
568 return cnt - read;
569}
570
532/* 571/*
533 * rb_wake_up_waiters - wake up tasks waiting for ring buffer input 572 * rb_wake_up_waiters - wake up tasks waiting for ring buffer input
534 * 573 *
@@ -556,7 +595,7 @@ static void rb_wake_up_waiters(struct irq_work *work)
556 * as data is added to any of the @buffer's cpu buffers. Otherwise 595 * as data is added to any of the @buffer's cpu buffers. Otherwise
557 * it will wait for data to be added to a specific cpu buffer. 596 * it will wait for data to be added to a specific cpu buffer.
558 */ 597 */
559int ring_buffer_wait(struct ring_buffer *buffer, int cpu, bool full) 598int ring_buffer_wait(struct ring_buffer *buffer, int cpu, int full)
560{ 599{
561 struct ring_buffer_per_cpu *uninitialized_var(cpu_buffer); 600 struct ring_buffer_per_cpu *uninitialized_var(cpu_buffer);
562 DEFINE_WAIT(wait); 601 DEFINE_WAIT(wait);
@@ -571,7 +610,7 @@ int ring_buffer_wait(struct ring_buffer *buffer, int cpu, bool full)
571 if (cpu == RING_BUFFER_ALL_CPUS) { 610 if (cpu == RING_BUFFER_ALL_CPUS) {
572 work = &buffer->irq_work; 611 work = &buffer->irq_work;
573 /* Full only makes sense on per cpu reads */ 612 /* Full only makes sense on per cpu reads */
574 full = false; 613 full = 0;
575 } else { 614 } else {
576 if (!cpumask_test_cpu(cpu, buffer->cpumask)) 615 if (!cpumask_test_cpu(cpu, buffer->cpumask))
577 return -ENODEV; 616 return -ENODEV;
@@ -623,15 +662,22 @@ int ring_buffer_wait(struct ring_buffer *buffer, int cpu, bool full)
623 !ring_buffer_empty_cpu(buffer, cpu)) { 662 !ring_buffer_empty_cpu(buffer, cpu)) {
624 unsigned long flags; 663 unsigned long flags;
625 bool pagebusy; 664 bool pagebusy;
665 size_t nr_pages;
666 size_t dirty;
626 667
627 if (!full) 668 if (!full)
628 break; 669 break;
629 670
630 raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); 671 raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
631 pagebusy = cpu_buffer->reader_page == cpu_buffer->commit_page; 672 pagebusy = cpu_buffer->reader_page == cpu_buffer->commit_page;
673 nr_pages = cpu_buffer->nr_pages;
674 dirty = ring_buffer_nr_dirty_pages(buffer, cpu);
675 if (!cpu_buffer->shortest_full ||
676 cpu_buffer->shortest_full < full)
677 cpu_buffer->shortest_full = full;
632 raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); 678 raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
633 679 if (!pagebusy &&
634 if (!pagebusy) 680 (!nr_pages || (dirty * 100) > full * nr_pages))
635 break; 681 break;
636 } 682 }
637 683
@@ -1054,6 +1100,7 @@ static void rb_tail_page_update(struct ring_buffer_per_cpu *cpu_buffer,
1054 old_write = local_add_return(RB_WRITE_INTCNT, &next_page->write); 1100 old_write = local_add_return(RB_WRITE_INTCNT, &next_page->write);
1055 old_entries = local_add_return(RB_WRITE_INTCNT, &next_page->entries); 1101 old_entries = local_add_return(RB_WRITE_INTCNT, &next_page->entries);
1056 1102
1103 local_inc(&cpu_buffer->pages_touched);
1057 /* 1104 /*
1058 * Just make sure we have seen our old_write and synchronize 1105 * Just make sure we have seen our old_write and synchronize
1059 * with any interrupts that come in. 1106 * with any interrupts that come in.
@@ -2586,7 +2633,9 @@ static void rb_commit(struct ring_buffer_per_cpu *cpu_buffer,
2586static __always_inline void 2633static __always_inline void
2587rb_wakeups(struct ring_buffer *buffer, struct ring_buffer_per_cpu *cpu_buffer) 2634rb_wakeups(struct ring_buffer *buffer, struct ring_buffer_per_cpu *cpu_buffer)
2588{ 2635{
2589 bool pagebusy; 2636 size_t nr_pages;
2637 size_t dirty;
2638 size_t full;
2590 2639
2591 if (buffer->irq_work.waiters_pending) { 2640 if (buffer->irq_work.waiters_pending) {
2592 buffer->irq_work.waiters_pending = false; 2641 buffer->irq_work.waiters_pending = false;
@@ -2600,14 +2649,27 @@ rb_wakeups(struct ring_buffer *buffer, struct ring_buffer_per_cpu *cpu_buffer)
2600 irq_work_queue(&cpu_buffer->irq_work.work); 2649 irq_work_queue(&cpu_buffer->irq_work.work);
2601 } 2650 }
2602 2651
2603 pagebusy = cpu_buffer->reader_page == cpu_buffer->commit_page; 2652 if (cpu_buffer->last_pages_touch == local_read(&cpu_buffer->pages_touched))
2653 return;
2604 2654
2605 if (!pagebusy && cpu_buffer->irq_work.full_waiters_pending) { 2655 if (cpu_buffer->reader_page == cpu_buffer->commit_page)
2606 cpu_buffer->irq_work.wakeup_full = true; 2656 return;
2607 cpu_buffer->irq_work.full_waiters_pending = false; 2657
2608 /* irq_work_queue() supplies it's own memory barriers */ 2658 if (!cpu_buffer->irq_work.full_waiters_pending)
2609 irq_work_queue(&cpu_buffer->irq_work.work); 2659 return;
2610 } 2660
2661 cpu_buffer->last_pages_touch = local_read(&cpu_buffer->pages_touched);
2662
2663 full = cpu_buffer->shortest_full;
2664 nr_pages = cpu_buffer->nr_pages;
2665 dirty = ring_buffer_nr_dirty_pages(buffer, cpu_buffer->cpu);
2666 if (full && nr_pages && (dirty * 100) <= full * nr_pages)
2667 return;
2668
2669 cpu_buffer->irq_work.wakeup_full = true;
2670 cpu_buffer->irq_work.full_waiters_pending = false;
2671 /* irq_work_queue() supplies it's own memory barriers */
2672 irq_work_queue(&cpu_buffer->irq_work.work);
2611} 2673}
2612 2674
2613/* 2675/*
@@ -3732,13 +3794,15 @@ rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer)
3732 goto spin; 3794 goto spin;
3733 3795
3734 /* 3796 /*
3735 * Yeah! We succeeded in replacing the page. 3797 * Yay! We succeeded in replacing the page.
3736 * 3798 *
3737 * Now make the new head point back to the reader page. 3799 * Now make the new head point back to the reader page.
3738 */ 3800 */
3739 rb_list_head(reader->list.next)->prev = &cpu_buffer->reader_page->list; 3801 rb_list_head(reader->list.next)->prev = &cpu_buffer->reader_page->list;
3740 rb_inc_page(cpu_buffer, &cpu_buffer->head_page); 3802 rb_inc_page(cpu_buffer, &cpu_buffer->head_page);
3741 3803
3804 local_inc(&cpu_buffer->pages_read);
3805
3742 /* Finally update the reader page to the new head */ 3806 /* Finally update the reader page to the new head */
3743 cpu_buffer->reader_page = reader; 3807 cpu_buffer->reader_page = reader;
3744 cpu_buffer->reader_page->read = 0; 3808 cpu_buffer->reader_page->read = 0;
@@ -4334,6 +4398,10 @@ rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer)
4334 local_set(&cpu_buffer->entries, 0); 4398 local_set(&cpu_buffer->entries, 0);
4335 local_set(&cpu_buffer->committing, 0); 4399 local_set(&cpu_buffer->committing, 0);
4336 local_set(&cpu_buffer->commits, 0); 4400 local_set(&cpu_buffer->commits, 0);
4401 local_set(&cpu_buffer->pages_touched, 0);
4402 local_set(&cpu_buffer->pages_read, 0);
4403 cpu_buffer->last_pages_touch = 0;
4404 cpu_buffer->shortest_full = 0;
4337 cpu_buffer->read = 0; 4405 cpu_buffer->read = 0;
4338 cpu_buffer->read_bytes = 0; 4406 cpu_buffer->read_bytes = 0;
4339 4407
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index 51612b4a603f..c521b7347482 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -1431,7 +1431,7 @@ update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
1431} 1431}
1432#endif /* CONFIG_TRACER_MAX_TRACE */ 1432#endif /* CONFIG_TRACER_MAX_TRACE */
1433 1433
1434static int wait_on_pipe(struct trace_iterator *iter, bool full) 1434static int wait_on_pipe(struct trace_iterator *iter, int full)
1435{ 1435{
1436 /* Iterators are static, they should be filled or empty */ 1436 /* Iterators are static, they should be filled or empty */
1437 if (trace_buffer_iter(iter, iter->cpu_file)) 1437 if (trace_buffer_iter(iter, iter->cpu_file))
@@ -2452,7 +2452,7 @@ static inline void ftrace_exports_disable(void)
2452 static_branch_disable(&ftrace_exports_enabled); 2452 static_branch_disable(&ftrace_exports_enabled);
2453} 2453}
2454 2454
2455void ftrace_exports(struct ring_buffer_event *event) 2455static void ftrace_exports(struct ring_buffer_event *event)
2456{ 2456{
2457 struct trace_export *export; 2457 struct trace_export *export;
2458 2458
@@ -4408,13 +4408,15 @@ static int trace_set_options(struct trace_array *tr, char *option)
4408 int neg = 0; 4408 int neg = 0;
4409 int ret; 4409 int ret;
4410 size_t orig_len = strlen(option); 4410 size_t orig_len = strlen(option);
4411 int len;
4411 4412
4412 cmp = strstrip(option); 4413 cmp = strstrip(option);
4413 4414
4414 if (strncmp(cmp, "no", 2) == 0) { 4415 len = str_has_prefix(cmp, "no");
4416 if (len)
4415 neg = 1; 4417 neg = 1;
4416 cmp += 2; 4418
4417 } 4419 cmp += len;
4418 4420
4419 mutex_lock(&trace_types_lock); 4421 mutex_lock(&trace_types_lock);
4420 4422
@@ -4604,6 +4606,10 @@ static const char readme_msg[] =
4604 "\t\t\t traces\n" 4606 "\t\t\t traces\n"
4605#endif 4607#endif
4606#endif /* CONFIG_STACK_TRACER */ 4608#endif /* CONFIG_STACK_TRACER */
4609#ifdef CONFIG_DYNAMIC_EVENTS
4610 " dynamic_events\t\t- Add/remove/show the generic dynamic events\n"
4611 "\t\t\t Write into this file to define/undefine new trace events.\n"
4612#endif
4607#ifdef CONFIG_KPROBE_EVENTS 4613#ifdef CONFIG_KPROBE_EVENTS
4608 " kprobe_events\t\t- Add/remove/show the kernel dynamic events\n" 4614 " kprobe_events\t\t- Add/remove/show the kernel dynamic events\n"
4609 "\t\t\t Write into this file to define/undefine new trace events.\n" 4615 "\t\t\t Write into this file to define/undefine new trace events.\n"
@@ -4616,6 +4622,9 @@ static const char readme_msg[] =
4616 "\t accepts: event-definitions (one definition per line)\n" 4622 "\t accepts: event-definitions (one definition per line)\n"
4617 "\t Format: p[:[<group>/]<event>] <place> [<args>]\n" 4623 "\t Format: p[:[<group>/]<event>] <place> [<args>]\n"
4618 "\t r[maxactive][:[<group>/]<event>] <place> [<args>]\n" 4624 "\t r[maxactive][:[<group>/]<event>] <place> [<args>]\n"
4625#ifdef CONFIG_HIST_TRIGGERS
4626 "\t s:[synthetic/]<event> <field> [<field>]\n"
4627#endif
4619 "\t -:[<group>/]<event>\n" 4628 "\t -:[<group>/]<event>\n"
4620#ifdef CONFIG_KPROBE_EVENTS 4629#ifdef CONFIG_KPROBE_EVENTS
4621 "\t place: [<module>:]<symbol>[+<offset>]|<memaddr>\n" 4630 "\t place: [<module>:]<symbol>[+<offset>]|<memaddr>\n"
@@ -4634,6 +4643,11 @@ static const char readme_msg[] =
4634 "\t type: s8/16/32/64, u8/16/32/64, x8/16/32/64, string, symbol,\n" 4643 "\t type: s8/16/32/64, u8/16/32/64, x8/16/32/64, string, symbol,\n"
4635 "\t b<bit-width>@<bit-offset>/<container-size>,\n" 4644 "\t b<bit-width>@<bit-offset>/<container-size>,\n"
4636 "\t <type>\\[<array-size>\\]\n" 4645 "\t <type>\\[<array-size>\\]\n"
4646#ifdef CONFIG_HIST_TRIGGERS
4647 "\t field: <stype> <name>;\n"
4648 "\t stype: u8/u16/u32/u64, s8/s16/s32/s64, pid_t,\n"
4649 "\t [unsigned] char/int/long\n"
4650#endif
4637#endif 4651#endif
4638 " events/\t\t- Directory containing all trace event subsystems:\n" 4652 " events/\t\t- Directory containing all trace event subsystems:\n"
4639 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n" 4653 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
@@ -5693,7 +5707,7 @@ static int tracing_wait_pipe(struct file *filp)
5693 5707
5694 mutex_unlock(&iter->mutex); 5708 mutex_unlock(&iter->mutex);
5695 5709
5696 ret = wait_on_pipe(iter, false); 5710 ret = wait_on_pipe(iter, 0);
5697 5711
5698 mutex_lock(&iter->mutex); 5712 mutex_lock(&iter->mutex);
5699 5713
@@ -6751,7 +6765,7 @@ tracing_buffers_read(struct file *filp, char __user *ubuf,
6751 if ((filp->f_flags & O_NONBLOCK)) 6765 if ((filp->f_flags & O_NONBLOCK))
6752 return -EAGAIN; 6766 return -EAGAIN;
6753 6767
6754 ret = wait_on_pipe(iter, false); 6768 ret = wait_on_pipe(iter, 0);
6755 if (ret) 6769 if (ret)
6756 return ret; 6770 return ret;
6757 6771
@@ -6948,7 +6962,7 @@ tracing_buffers_splice_read(struct file *file, loff_t *ppos,
6948 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK)) 6962 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK))
6949 goto out; 6963 goto out;
6950 6964
6951 ret = wait_on_pipe(iter, true); 6965 ret = wait_on_pipe(iter, iter->tr->buffer_percent);
6952 if (ret) 6966 if (ret)
6953 goto out; 6967 goto out;
6954 6968
@@ -7662,6 +7676,53 @@ static const struct file_operations rb_simple_fops = {
7662 .llseek = default_llseek, 7676 .llseek = default_llseek,
7663}; 7677};
7664 7678
7679static ssize_t
7680buffer_percent_read(struct file *filp, char __user *ubuf,
7681 size_t cnt, loff_t *ppos)
7682{
7683 struct trace_array *tr = filp->private_data;
7684 char buf[64];
7685 int r;
7686
7687 r = tr->buffer_percent;
7688 r = sprintf(buf, "%d\n", r);
7689
7690 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
7691}
7692
7693static ssize_t
7694buffer_percent_write(struct file *filp, const char __user *ubuf,
7695 size_t cnt, loff_t *ppos)
7696{
7697 struct trace_array *tr = filp->private_data;
7698 unsigned long val;
7699 int ret;
7700
7701 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
7702 if (ret)
7703 return ret;
7704
7705 if (val > 100)
7706 return -EINVAL;
7707
7708 if (!val)
7709 val = 1;
7710
7711 tr->buffer_percent = val;
7712
7713 (*ppos)++;
7714
7715 return cnt;
7716}
7717
7718static const struct file_operations buffer_percent_fops = {
7719 .open = tracing_open_generic_tr,
7720 .read = buffer_percent_read,
7721 .write = buffer_percent_write,
7722 .release = tracing_release_generic_tr,
7723 .llseek = default_llseek,
7724};
7725
7665struct dentry *trace_instance_dir; 7726struct dentry *trace_instance_dir;
7666 7727
7667static void 7728static void
@@ -7970,6 +8031,11 @@ init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer)
7970 trace_create_file("timestamp_mode", 0444, d_tracer, tr, 8031 trace_create_file("timestamp_mode", 0444, d_tracer, tr,
7971 &trace_time_stamp_mode_fops); 8032 &trace_time_stamp_mode_fops);
7972 8033
8034 tr->buffer_percent = 50;
8035
8036 trace_create_file("buffer_percent", 0444, d_tracer,
8037 tr, &buffer_percent_fops);
8038
7973 create_trace_options_dir(tr); 8039 create_trace_options_dir(tr);
7974 8040
7975#if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER) 8041#if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h
index 447bd96ee658..08900828d282 100644
--- a/kernel/trace/trace.h
+++ b/kernel/trace/trace.h
@@ -247,6 +247,7 @@ struct trace_array {
247 int clock_id; 247 int clock_id;
248 int nr_topts; 248 int nr_topts;
249 bool clear_trace; 249 bool clear_trace;
250 int buffer_percent;
250 struct tracer *current_trace; 251 struct tracer *current_trace;
251 unsigned int trace_flags; 252 unsigned int trace_flags;
252 unsigned char trace_flags_index[TRACE_FLAGS_MAX_SIZE]; 253 unsigned char trace_flags_index[TRACE_FLAGS_MAX_SIZE];
@@ -534,6 +535,13 @@ enum {
534 535
535 TRACE_GRAPH_DEPTH_START_BIT, 536 TRACE_GRAPH_DEPTH_START_BIT,
536 TRACE_GRAPH_DEPTH_END_BIT, 537 TRACE_GRAPH_DEPTH_END_BIT,
538
539 /*
540 * To implement set_graph_notrace, if this bit is set, we ignore
541 * function graph tracing of called functions, until the return
542 * function is called to clear it.
543 */
544 TRACE_GRAPH_NOTRACE_BIT,
537}; 545};
538 546
539#define trace_recursion_set(bit) do { (current)->trace_recursion |= (1<<(bit)); } while (0) 547#define trace_recursion_set(bit) do { (current)->trace_recursion |= (1<<(bit)); } while (0)
@@ -855,7 +863,12 @@ static __always_inline bool ftrace_hash_empty(struct ftrace_hash *hash)
855#define TRACE_GRAPH_PRINT_FILL_MASK (0x3 << TRACE_GRAPH_PRINT_FILL_SHIFT) 863#define TRACE_GRAPH_PRINT_FILL_MASK (0x3 << TRACE_GRAPH_PRINT_FILL_SHIFT)
856 864
857extern void ftrace_graph_sleep_time_control(bool enable); 865extern void ftrace_graph_sleep_time_control(bool enable);
866
867#ifdef CONFIG_FUNCTION_PROFILER
858extern void ftrace_graph_graph_time_control(bool enable); 868extern void ftrace_graph_graph_time_control(bool enable);
869#else
870static inline void ftrace_graph_graph_time_control(bool enable) { }
871#endif
859 872
860extern enum print_line_t 873extern enum print_line_t
861print_graph_function_flags(struct trace_iterator *iter, u32 flags); 874print_graph_function_flags(struct trace_iterator *iter, u32 flags);
diff --git a/kernel/trace/trace_dynevent.c b/kernel/trace/trace_dynevent.c
new file mode 100644
index 000000000000..dd1f43588d70
--- /dev/null
+++ b/kernel/trace/trace_dynevent.c
@@ -0,0 +1,217 @@
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Generic dynamic event control interface
4 *
5 * Copyright (C) 2018 Masami Hiramatsu <mhiramat@kernel.org>
6 */
7
8#include <linux/debugfs.h>
9#include <linux/kernel.h>
10#include <linux/list.h>
11#include <linux/mm.h>
12#include <linux/mutex.h>
13#include <linux/tracefs.h>
14
15#include "trace.h"
16#include "trace_dynevent.h"
17
18static DEFINE_MUTEX(dyn_event_ops_mutex);
19static LIST_HEAD(dyn_event_ops_list);
20
21int dyn_event_register(struct dyn_event_operations *ops)
22{
23 if (!ops || !ops->create || !ops->show || !ops->is_busy ||
24 !ops->free || !ops->match)
25 return -EINVAL;
26
27 INIT_LIST_HEAD(&ops->list);
28 mutex_lock(&dyn_event_ops_mutex);
29 list_add_tail(&ops->list, &dyn_event_ops_list);
30 mutex_unlock(&dyn_event_ops_mutex);
31 return 0;
32}
33
34int dyn_event_release(int argc, char **argv, struct dyn_event_operations *type)
35{
36 struct dyn_event *pos, *n;
37 char *system = NULL, *event, *p;
38 int ret = -ENOENT;
39
40 if (argv[0][0] == '-') {
41 if (argv[0][1] != ':')
42 return -EINVAL;
43 event = &argv[0][2];
44 } else {
45 event = strchr(argv[0], ':');
46 if (!event)
47 return -EINVAL;
48 event++;
49 }
50
51 p = strchr(event, '/');
52 if (p) {
53 system = event;
54 event = p + 1;
55 *p = '\0';
56 }
57 if (event[0] == '\0')
58 return -EINVAL;
59
60 mutex_lock(&event_mutex);
61 for_each_dyn_event_safe(pos, n) {
62 if (type && type != pos->ops)
63 continue;
64 if (pos->ops->match(system, event, pos)) {
65 ret = pos->ops->free(pos);
66 break;
67 }
68 }
69 mutex_unlock(&event_mutex);
70
71 return ret;
72}
73
74static int create_dyn_event(int argc, char **argv)
75{
76 struct dyn_event_operations *ops;
77 int ret;
78
79 if (argv[0][0] == '-' || argv[0][0] == '!')
80 return dyn_event_release(argc, argv, NULL);
81
82 mutex_lock(&dyn_event_ops_mutex);
83 list_for_each_entry(ops, &dyn_event_ops_list, list) {
84 ret = ops->create(argc, (const char **)argv);
85 if (!ret || ret != -ECANCELED)
86 break;
87 }
88 mutex_unlock(&dyn_event_ops_mutex);
89 if (ret == -ECANCELED)
90 ret = -EINVAL;
91
92 return ret;
93}
94
95/* Protected by event_mutex */
96LIST_HEAD(dyn_event_list);
97
98void *dyn_event_seq_start(struct seq_file *m, loff_t *pos)
99{
100 mutex_lock(&event_mutex);
101 return seq_list_start(&dyn_event_list, *pos);
102}
103
104void *dyn_event_seq_next(struct seq_file *m, void *v, loff_t *pos)
105{
106 return seq_list_next(v, &dyn_event_list, pos);
107}
108
109void dyn_event_seq_stop(struct seq_file *m, void *v)
110{
111 mutex_unlock(&event_mutex);
112}
113
114static int dyn_event_seq_show(struct seq_file *m, void *v)
115{
116 struct dyn_event *ev = v;
117
118 if (ev && ev->ops)
119 return ev->ops->show(m, ev);
120
121 return 0;
122}
123
124static const struct seq_operations dyn_event_seq_op = {
125 .start = dyn_event_seq_start,
126 .next = dyn_event_seq_next,
127 .stop = dyn_event_seq_stop,
128 .show = dyn_event_seq_show
129};
130
131/*
132 * dyn_events_release_all - Release all specific events
133 * @type: the dyn_event_operations * which filters releasing events
134 *
135 * This releases all events which ->ops matches @type. If @type is NULL,
136 * all events are released.
137 * Return -EBUSY if any of them are in use, and return other errors when
138 * it failed to free the given event. Except for -EBUSY, event releasing
139 * process will be aborted at that point and there may be some other
140 * releasable events on the list.
141 */
142int dyn_events_release_all(struct dyn_event_operations *type)
143{
144 struct dyn_event *ev, *tmp;
145 int ret = 0;
146
147 mutex_lock(&event_mutex);
148 for_each_dyn_event(ev) {
149 if (type && ev->ops != type)
150 continue;
151 if (ev->ops->is_busy(ev)) {
152 ret = -EBUSY;
153 goto out;
154 }
155 }
156 for_each_dyn_event_safe(ev, tmp) {
157 if (type && ev->ops != type)
158 continue;
159 ret = ev->ops->free(ev);
160 if (ret)
161 break;
162 }
163out:
164 mutex_unlock(&event_mutex);
165
166 return ret;
167}
168
169static int dyn_event_open(struct inode *inode, struct file *file)
170{
171 int ret;
172
173 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
174 ret = dyn_events_release_all(NULL);
175 if (ret < 0)
176 return ret;
177 }
178
179 return seq_open(file, &dyn_event_seq_op);
180}
181
182static ssize_t dyn_event_write(struct file *file, const char __user *buffer,
183 size_t count, loff_t *ppos)
184{
185 return trace_parse_run_command(file, buffer, count, ppos,
186 create_dyn_event);
187}
188
189static const struct file_operations dynamic_events_ops = {
190 .owner = THIS_MODULE,
191 .open = dyn_event_open,
192 .read = seq_read,
193 .llseek = seq_lseek,
194 .release = seq_release,
195 .write = dyn_event_write,
196};
197
198/* Make a tracefs interface for controlling dynamic events */
199static __init int init_dynamic_event(void)
200{
201 struct dentry *d_tracer;
202 struct dentry *entry;
203
204 d_tracer = tracing_init_dentry();
205 if (IS_ERR(d_tracer))
206 return 0;
207
208 entry = tracefs_create_file("dynamic_events", 0644, d_tracer,
209 NULL, &dynamic_events_ops);
210
211 /* Event list interface */
212 if (!entry)
213 pr_warn("Could not create tracefs 'dynamic_events' entry\n");
214
215 return 0;
216}
217fs_initcall(init_dynamic_event);
diff --git a/kernel/trace/trace_dynevent.h b/kernel/trace/trace_dynevent.h
new file mode 100644
index 000000000000..8c334064e4d6
--- /dev/null
+++ b/kernel/trace/trace_dynevent.h
@@ -0,0 +1,119 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Common header file for generic dynamic events.
4 */
5
6#ifndef _TRACE_DYNEVENT_H
7#define _TRACE_DYNEVENT_H
8
9#include <linux/kernel.h>
10#include <linux/list.h>
11#include <linux/mutex.h>
12#include <linux/seq_file.h>
13
14#include "trace.h"
15
16struct dyn_event;
17
18/**
19 * struct dyn_event_operations - Methods for each type of dynamic events
20 *
21 * These methods must be set for each type, since there is no default method.
22 * Before using this for dyn_event_init(), it must be registered by
23 * dyn_event_register().
24 *
25 * @create: Parse and create event method. This is invoked when user passes
26 * a event definition to dynamic_events interface. This must not destruct
27 * the arguments and return -ECANCELED if given arguments doesn't match its
28 * command prefix.
29 * @show: Showing method. This is invoked when user reads the event definitions
30 * via dynamic_events interface.
31 * @is_busy: Check whether given event is busy so that it can not be deleted.
32 * Return true if it is busy, otherwides false.
33 * @free: Delete the given event. Return 0 if success, otherwides error.
34 * @match: Check whether given event and system name match this event.
35 * Return true if it matches, otherwides false.
36 *
37 * Except for @create, these methods are called under holding event_mutex.
38 */
39struct dyn_event_operations {
40 struct list_head list;
41 int (*create)(int argc, const char *argv[]);
42 int (*show)(struct seq_file *m, struct dyn_event *ev);
43 bool (*is_busy)(struct dyn_event *ev);
44 int (*free)(struct dyn_event *ev);
45 bool (*match)(const char *system, const char *event,
46 struct dyn_event *ev);
47};
48
49/* Register new dyn_event type -- must be called at first */
50int dyn_event_register(struct dyn_event_operations *ops);
51
52/**
53 * struct dyn_event - Dynamic event list header
54 *
55 * The dyn_event structure encapsulates a list and a pointer to the operators
56 * for making a global list of dynamic events.
57 * User must includes this in each event structure, so that those events can
58 * be added/removed via dynamic_events interface.
59 */
60struct dyn_event {
61 struct list_head list;
62 struct dyn_event_operations *ops;
63};
64
65extern struct list_head dyn_event_list;
66
67static inline
68int dyn_event_init(struct dyn_event *ev, struct dyn_event_operations *ops)
69{
70 if (!ev || !ops)
71 return -EINVAL;
72
73 INIT_LIST_HEAD(&ev->list);
74 ev->ops = ops;
75 return 0;
76}
77
78static inline int dyn_event_add(struct dyn_event *ev)
79{
80 lockdep_assert_held(&event_mutex);
81
82 if (!ev || !ev->ops)
83 return -EINVAL;
84
85 list_add_tail(&ev->list, &dyn_event_list);
86 return 0;
87}
88
89static inline void dyn_event_remove(struct dyn_event *ev)
90{
91 lockdep_assert_held(&event_mutex);
92 list_del_init(&ev->list);
93}
94
95void *dyn_event_seq_start(struct seq_file *m, loff_t *pos);
96void *dyn_event_seq_next(struct seq_file *m, void *v, loff_t *pos);
97void dyn_event_seq_stop(struct seq_file *m, void *v);
98int dyn_events_release_all(struct dyn_event_operations *type);
99int dyn_event_release(int argc, char **argv, struct dyn_event_operations *type);
100
101/*
102 * for_each_dyn_event - iterate over the dyn_event list
103 * @pos: the struct dyn_event * to use as a loop cursor
104 *
105 * This is just a basement of for_each macro. Wrap this for
106 * each actual event structure with ops filtering.
107 */
108#define for_each_dyn_event(pos) \
109 list_for_each_entry(pos, &dyn_event_list, list)
110
111/*
112 * for_each_dyn_event - iterate over the dyn_event list safely
113 * @pos: the struct dyn_event * to use as a loop cursor
114 * @n: the struct dyn_event * to use as temporary storage
115 */
116#define for_each_dyn_event_safe(pos, n) \
117 list_for_each_entry_safe(pos, n, &dyn_event_list, list)
118
119#endif
diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c
index f94be0c2827b..5b3b0c3c8a47 100644
--- a/kernel/trace/trace_events.c
+++ b/kernel/trace/trace_events.c
@@ -1251,7 +1251,7 @@ static int f_show(struct seq_file *m, void *v)
1251 */ 1251 */
1252 array_descriptor = strchr(field->type, '['); 1252 array_descriptor = strchr(field->type, '[');
1253 1253
1254 if (!strncmp(field->type, "__data_loc", 10)) 1254 if (str_has_prefix(field->type, "__data_loc"))
1255 array_descriptor = NULL; 1255 array_descriptor = NULL;
1256 1256
1257 if (!array_descriptor) 1257 if (!array_descriptor)
@@ -2309,7 +2309,8 @@ static void __add_event_to_tracers(struct trace_event_call *call);
2309int trace_add_event_call(struct trace_event_call *call) 2309int trace_add_event_call(struct trace_event_call *call)
2310{ 2310{
2311 int ret; 2311 int ret;
2312 mutex_lock(&event_mutex); 2312 lockdep_assert_held(&event_mutex);
2313
2313 mutex_lock(&trace_types_lock); 2314 mutex_lock(&trace_types_lock);
2314 2315
2315 ret = __register_event(call, NULL); 2316 ret = __register_event(call, NULL);
@@ -2317,7 +2318,6 @@ int trace_add_event_call(struct trace_event_call *call)
2317 __add_event_to_tracers(call); 2318 __add_event_to_tracers(call);
2318 2319
2319 mutex_unlock(&trace_types_lock); 2320 mutex_unlock(&trace_types_lock);
2320 mutex_unlock(&event_mutex);
2321 return ret; 2321 return ret;
2322} 2322}
2323 2323
@@ -2371,13 +2371,13 @@ int trace_remove_event_call(struct trace_event_call *call)
2371{ 2371{
2372 int ret; 2372 int ret;
2373 2373
2374 mutex_lock(&event_mutex); 2374 lockdep_assert_held(&event_mutex);
2375
2375 mutex_lock(&trace_types_lock); 2376 mutex_lock(&trace_types_lock);
2376 down_write(&trace_event_sem); 2377 down_write(&trace_event_sem);
2377 ret = probe_remove_event_call(call); 2378 ret = probe_remove_event_call(call);
2378 up_write(&trace_event_sem); 2379 up_write(&trace_event_sem);
2379 mutex_unlock(&trace_types_lock); 2380 mutex_unlock(&trace_types_lock);
2380 mutex_unlock(&event_mutex);
2381 2381
2382 return ret; 2382 return ret;
2383} 2383}
diff --git a/kernel/trace/trace_events_hist.c b/kernel/trace/trace_events_hist.c
index eb908ef2ecec..449d90cfa151 100644
--- a/kernel/trace/trace_events_hist.c
+++ b/kernel/trace/trace_events_hist.c
@@ -15,6 +15,7 @@
15 15
16#include "tracing_map.h" 16#include "tracing_map.h"
17#include "trace.h" 17#include "trace.h"
18#include "trace_dynevent.h"
18 19
19#define SYNTH_SYSTEM "synthetic" 20#define SYNTH_SYSTEM "synthetic"
20#define SYNTH_FIELDS_MAX 16 21#define SYNTH_FIELDS_MAX 16
@@ -39,6 +40,16 @@ enum field_op_id {
39 FIELD_OP_UNARY_MINUS, 40 FIELD_OP_UNARY_MINUS,
40}; 41};
41 42
43/*
44 * A hist_var (histogram variable) contains variable information for
45 * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
46 * flag set. A hist_var has a variable name e.g. ts0, and is
47 * associated with a given histogram trigger, as specified by
48 * hist_data. The hist_var idx is the unique index assigned to the
49 * variable by the hist trigger's tracing_map. The idx is what is
50 * used to set a variable's value and, by a variable reference, to
51 * retrieve it.
52 */
42struct hist_var { 53struct hist_var {
43 char *name; 54 char *name;
44 struct hist_trigger_data *hist_data; 55 struct hist_trigger_data *hist_data;
@@ -55,12 +66,29 @@ struct hist_field {
55 const char *type; 66 const char *type;
56 struct hist_field *operands[HIST_FIELD_OPERANDS_MAX]; 67 struct hist_field *operands[HIST_FIELD_OPERANDS_MAX];
57 struct hist_trigger_data *hist_data; 68 struct hist_trigger_data *hist_data;
69
70 /*
71 * Variable fields contain variable-specific info in var.
72 */
58 struct hist_var var; 73 struct hist_var var;
59 enum field_op_id operator; 74 enum field_op_id operator;
60 char *system; 75 char *system;
61 char *event_name; 76 char *event_name;
77
78 /*
79 * The name field is used for EXPR and VAR_REF fields. VAR
80 * fields contain the variable name in var.name.
81 */
62 char *name; 82 char *name;
63 unsigned int var_idx; 83
84 /*
85 * When a histogram trigger is hit, if it has any references
86 * to variables, the values of those variables are collected
87 * into a var_ref_vals array by resolve_var_refs(). The
88 * current value of each variable is read from the tracing_map
89 * using the hist field's hist_var.idx and entered into the
90 * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
91 */
64 unsigned int var_ref_idx; 92 unsigned int var_ref_idx;
65 bool read_once; 93 bool read_once;
66}; 94};
@@ -279,8 +307,6 @@ struct hist_trigger_data {
279 struct action_data *actions[HIST_ACTIONS_MAX]; 307 struct action_data *actions[HIST_ACTIONS_MAX];
280 unsigned int n_actions; 308 unsigned int n_actions;
281 309
282 struct hist_field *synth_var_refs[SYNTH_FIELDS_MAX];
283 unsigned int n_synth_var_refs;
284 struct field_var *field_vars[SYNTH_FIELDS_MAX]; 310 struct field_var *field_vars[SYNTH_FIELDS_MAX];
285 unsigned int n_field_vars; 311 unsigned int n_field_vars;
286 unsigned int n_field_var_str; 312 unsigned int n_field_var_str;
@@ -292,6 +318,21 @@ struct hist_trigger_data {
292 unsigned int n_max_var_str; 318 unsigned int n_max_var_str;
293}; 319};
294 320
321static int synth_event_create(int argc, const char **argv);
322static int synth_event_show(struct seq_file *m, struct dyn_event *ev);
323static int synth_event_release(struct dyn_event *ev);
324static bool synth_event_is_busy(struct dyn_event *ev);
325static bool synth_event_match(const char *system, const char *event,
326 struct dyn_event *ev);
327
328static struct dyn_event_operations synth_event_ops = {
329 .create = synth_event_create,
330 .show = synth_event_show,
331 .is_busy = synth_event_is_busy,
332 .free = synth_event_release,
333 .match = synth_event_match,
334};
335
295struct synth_field { 336struct synth_field {
296 char *type; 337 char *type;
297 char *name; 338 char *name;
@@ -301,7 +342,7 @@ struct synth_field {
301}; 342};
302 343
303struct synth_event { 344struct synth_event {
304 struct list_head list; 345 struct dyn_event devent;
305 int ref; 346 int ref;
306 char *name; 347 char *name;
307 struct synth_field **fields; 348 struct synth_field **fields;
@@ -312,6 +353,32 @@ struct synth_event {
312 struct tracepoint *tp; 353 struct tracepoint *tp;
313}; 354};
314 355
356static bool is_synth_event(struct dyn_event *ev)
357{
358 return ev->ops == &synth_event_ops;
359}
360
361static struct synth_event *to_synth_event(struct dyn_event *ev)
362{
363 return container_of(ev, struct synth_event, devent);
364}
365
366static bool synth_event_is_busy(struct dyn_event *ev)
367{
368 struct synth_event *event = to_synth_event(ev);
369
370 return event->ref != 0;
371}
372
373static bool synth_event_match(const char *system, const char *event,
374 struct dyn_event *ev)
375{
376 struct synth_event *sev = to_synth_event(ev);
377
378 return strcmp(sev->name, event) == 0 &&
379 (!system || strcmp(system, SYNTH_SYSTEM) == 0);
380}
381
315struct action_data; 382struct action_data;
316 383
317typedef void (*action_fn_t) (struct hist_trigger_data *hist_data, 384typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
@@ -326,6 +393,14 @@ struct action_data {
326 393
327 union { 394 union {
328 struct { 395 struct {
396 /*
397 * When a histogram trigger is hit, the values of any
398 * references to variables, including variables being passed
399 * as parameters to synthetic events, are collected into a
400 * var_ref_vals array. This var_ref_idx is the index of the
401 * first param in the array to be passed to the synthetic
402 * event invocation.
403 */
329 unsigned int var_ref_idx; 404 unsigned int var_ref_idx;
330 char *match_event; 405 char *match_event;
331 char *match_event_system; 406 char *match_event_system;
@@ -402,9 +477,6 @@ static bool have_hist_err(void)
402 return false; 477 return false;
403} 478}
404 479
405static LIST_HEAD(synth_event_list);
406static DEFINE_MUTEX(synth_event_mutex);
407
408struct synth_trace_event { 480struct synth_trace_event {
409 struct trace_entry ent; 481 struct trace_entry ent;
410 u64 fields[]; 482 u64 fields[];
@@ -446,7 +518,7 @@ static int synth_event_define_fields(struct trace_event_call *call)
446 518
447static bool synth_field_signed(char *type) 519static bool synth_field_signed(char *type)
448{ 520{
449 if (strncmp(type, "u", 1) == 0) 521 if (str_has_prefix(type, "u"))
450 return false; 522 return false;
451 523
452 return true; 524 return true;
@@ -469,7 +541,7 @@ static int synth_field_string_size(char *type)
469 start = strstr(type, "char["); 541 start = strstr(type, "char[");
470 if (start == NULL) 542 if (start == NULL)
471 return -EINVAL; 543 return -EINVAL;
472 start += strlen("char["); 544 start += sizeof("char[") - 1;
473 545
474 end = strchr(type, ']'); 546 end = strchr(type, ']');
475 if (!end || end < start) 547 if (!end || end < start)
@@ -738,14 +810,12 @@ static void free_synth_field(struct synth_field *field)
738 kfree(field); 810 kfree(field);
739} 811}
740 812
741static struct synth_field *parse_synth_field(int argc, char **argv, 813static struct synth_field *parse_synth_field(int argc, const char **argv,
742 int *consumed) 814 int *consumed)
743{ 815{
744 struct synth_field *field; 816 struct synth_field *field;
745 const char *prefix = NULL; 817 const char *prefix = NULL, *field_type = argv[0], *field_name, *array;
746 char *field_type = argv[0], *field_name;
747 int len, ret = 0; 818 int len, ret = 0;
748 char *array;
749 819
750 if (field_type[0] == ';') 820 if (field_type[0] == ';')
751 field_type++; 821 field_type++;
@@ -762,20 +832,31 @@ static struct synth_field *parse_synth_field(int argc, char **argv,
762 *consumed = 2; 832 *consumed = 2;
763 } 833 }
764 834
765 len = strlen(field_name);
766 if (field_name[len - 1] == ';')
767 field_name[len - 1] = '\0';
768
769 field = kzalloc(sizeof(*field), GFP_KERNEL); 835 field = kzalloc(sizeof(*field), GFP_KERNEL);
770 if (!field) 836 if (!field)
771 return ERR_PTR(-ENOMEM); 837 return ERR_PTR(-ENOMEM);
772 838
773 len = strlen(field_type) + 1; 839 len = strlen(field_name);
774 array = strchr(field_name, '['); 840 array = strchr(field_name, '[');
775 if (array) 841 if (array)
842 len -= strlen(array);
843 else if (field_name[len - 1] == ';')
844 len--;
845
846 field->name = kmemdup_nul(field_name, len, GFP_KERNEL);
847 if (!field->name) {
848 ret = -ENOMEM;
849 goto free;
850 }
851
852 if (field_type[0] == ';')
853 field_type++;
854 len = strlen(field_type) + 1;
855 if (array)
776 len += strlen(array); 856 len += strlen(array);
777 if (prefix) 857 if (prefix)
778 len += strlen(prefix); 858 len += strlen(prefix);
859
779 field->type = kzalloc(len, GFP_KERNEL); 860 field->type = kzalloc(len, GFP_KERNEL);
780 if (!field->type) { 861 if (!field->type) {
781 ret = -ENOMEM; 862 ret = -ENOMEM;
@@ -786,7 +867,8 @@ static struct synth_field *parse_synth_field(int argc, char **argv,
786 strcat(field->type, field_type); 867 strcat(field->type, field_type);
787 if (array) { 868 if (array) {
788 strcat(field->type, array); 869 strcat(field->type, array);
789 *array = '\0'; 870 if (field->type[len - 1] == ';')
871 field->type[len - 1] = '\0';
790 } 872 }
791 873
792 field->size = synth_field_size(field->type); 874 field->size = synth_field_size(field->type);
@@ -800,11 +882,6 @@ static struct synth_field *parse_synth_field(int argc, char **argv,
800 882
801 field->is_signed = synth_field_signed(field->type); 883 field->is_signed = synth_field_signed(field->type);
802 884
803 field->name = kstrdup(field_name, GFP_KERNEL);
804 if (!field->name) {
805 ret = -ENOMEM;
806 goto free;
807 }
808 out: 885 out:
809 return field; 886 return field;
810 free: 887 free:
@@ -868,9 +945,13 @@ static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
868 945
869static struct synth_event *find_synth_event(const char *name) 946static struct synth_event *find_synth_event(const char *name)
870{ 947{
948 struct dyn_event *pos;
871 struct synth_event *event; 949 struct synth_event *event;
872 950
873 list_for_each_entry(event, &synth_event_list, list) { 951 for_each_dyn_event(pos) {
952 if (!is_synth_event(pos))
953 continue;
954 event = to_synth_event(pos);
874 if (strcmp(event->name, name) == 0) 955 if (strcmp(event->name, name) == 0)
875 return event; 956 return event;
876 } 957 }
@@ -959,7 +1040,7 @@ static void free_synth_event(struct synth_event *event)
959 kfree(event); 1040 kfree(event);
960} 1041}
961 1042
962static struct synth_event *alloc_synth_event(char *event_name, int n_fields, 1043static struct synth_event *alloc_synth_event(const char *name, int n_fields,
963 struct synth_field **fields) 1044 struct synth_field **fields)
964{ 1045{
965 struct synth_event *event; 1046 struct synth_event *event;
@@ -971,7 +1052,7 @@ static struct synth_event *alloc_synth_event(char *event_name, int n_fields,
971 goto out; 1052 goto out;
972 } 1053 }
973 1054
974 event->name = kstrdup(event_name, GFP_KERNEL); 1055 event->name = kstrdup(name, GFP_KERNEL);
975 if (!event->name) { 1056 if (!event->name) {
976 kfree(event); 1057 kfree(event);
977 event = ERR_PTR(-ENOMEM); 1058 event = ERR_PTR(-ENOMEM);
@@ -985,6 +1066,8 @@ static struct synth_event *alloc_synth_event(char *event_name, int n_fields,
985 goto out; 1066 goto out;
986 } 1067 }
987 1068
1069 dyn_event_init(&event->devent, &synth_event_ops);
1070
988 for (i = 0; i < n_fields; i++) 1071 for (i = 0; i < n_fields; i++)
989 event->fields[i] = fields[i]; 1072 event->fields[i] = fields[i];
990 1073
@@ -1008,29 +1091,11 @@ struct hist_var_data {
1008 struct hist_trigger_data *hist_data; 1091 struct hist_trigger_data *hist_data;
1009}; 1092};
1010 1093
1011static void add_or_delete_synth_event(struct synth_event *event, int delete) 1094static int __create_synth_event(int argc, const char *name, const char **argv)
1012{
1013 if (delete)
1014 free_synth_event(event);
1015 else {
1016 mutex_lock(&synth_event_mutex);
1017 if (!find_synth_event(event->name))
1018 list_add(&event->list, &synth_event_list);
1019 else
1020 free_synth_event(event);
1021 mutex_unlock(&synth_event_mutex);
1022 }
1023}
1024
1025static int create_synth_event(int argc, char **argv)
1026{ 1095{
1027 struct synth_field *field, *fields[SYNTH_FIELDS_MAX]; 1096 struct synth_field *field, *fields[SYNTH_FIELDS_MAX];
1028 struct synth_event *event = NULL; 1097 struct synth_event *event = NULL;
1029 bool delete_event = false;
1030 int i, consumed = 0, n_fields = 0, ret = 0; 1098 int i, consumed = 0, n_fields = 0, ret = 0;
1031 char *name;
1032
1033 mutex_lock(&synth_event_mutex);
1034 1099
1035 /* 1100 /*
1036 * Argument syntax: 1101 * Argument syntax:
@@ -1038,42 +1103,19 @@ static int create_synth_event(int argc, char **argv)
1038 * - Remove synthetic event: !<event_name> field[;field] ... 1103 * - Remove synthetic event: !<event_name> field[;field] ...
1039 * where 'field' = type field_name 1104 * where 'field' = type field_name
1040 */ 1105 */
1041 if (argc < 1) {
1042 ret = -EINVAL;
1043 goto out;
1044 }
1045 1106
1046 name = argv[0]; 1107 if (name[0] == '\0' || argc < 1)
1047 if (name[0] == '!') { 1108 return -EINVAL;
1048 delete_event = true; 1109
1049 name++; 1110 mutex_lock(&event_mutex);
1050 }
1051 1111
1052 event = find_synth_event(name); 1112 event = find_synth_event(name);
1053 if (event) { 1113 if (event) {
1054 if (delete_event) {
1055 if (event->ref) {
1056 event = NULL;
1057 ret = -EBUSY;
1058 goto out;
1059 }
1060 list_del(&event->list);
1061 goto out;
1062 }
1063 event = NULL;
1064 ret = -EEXIST; 1114 ret = -EEXIST;
1065 goto out; 1115 goto out;
1066 } else if (delete_event) {
1067 ret = -ENOENT;
1068 goto out;
1069 }
1070
1071 if (argc < 2) {
1072 ret = -EINVAL;
1073 goto out;
1074 } 1116 }
1075 1117
1076 for (i = 1; i < argc - 1; i++) { 1118 for (i = 0; i < argc - 1; i++) {
1077 if (strcmp(argv[i], ";") == 0) 1119 if (strcmp(argv[i], ";") == 0)
1078 continue; 1120 continue;
1079 if (n_fields == SYNTH_FIELDS_MAX) { 1121 if (n_fields == SYNTH_FIELDS_MAX) {
@@ -1101,83 +1143,91 @@ static int create_synth_event(int argc, char **argv)
1101 event = NULL; 1143 event = NULL;
1102 goto err; 1144 goto err;
1103 } 1145 }
1146 ret = register_synth_event(event);
1147 if (!ret)
1148 dyn_event_add(&event->devent);
1149 else
1150 free_synth_event(event);
1104 out: 1151 out:
1105 mutex_unlock(&synth_event_mutex); 1152 mutex_unlock(&event_mutex);
1106
1107 if (event) {
1108 if (delete_event) {
1109 ret = unregister_synth_event(event);
1110 add_or_delete_synth_event(event, !ret);
1111 } else {
1112 ret = register_synth_event(event);
1113 add_or_delete_synth_event(event, ret);
1114 }
1115 }
1116 1153
1117 return ret; 1154 return ret;
1118 err: 1155 err:
1119 mutex_unlock(&synth_event_mutex);
1120
1121 for (i = 0; i < n_fields; i++) 1156 for (i = 0; i < n_fields; i++)
1122 free_synth_field(fields[i]); 1157 free_synth_field(fields[i]);
1123 free_synth_event(event);
1124 1158
1125 return ret; 1159 goto out;
1126} 1160}
1127 1161
1128static int release_all_synth_events(void) 1162static int create_or_delete_synth_event(int argc, char **argv)
1129{ 1163{
1130 struct list_head release_events; 1164 const char *name = argv[0];
1131 struct synth_event *event, *e; 1165 struct synth_event *event = NULL;
1132 int ret = 0; 1166 int ret;
1133
1134 INIT_LIST_HEAD(&release_events);
1135
1136 mutex_lock(&synth_event_mutex);
1137 1167
1138 list_for_each_entry(event, &synth_event_list, list) { 1168 /* trace_run_command() ensures argc != 0 */
1139 if (event->ref) { 1169 if (name[0] == '!') {
1140 mutex_unlock(&synth_event_mutex); 1170 mutex_lock(&event_mutex);
1141 return -EBUSY; 1171 event = find_synth_event(name + 1);
1142 } 1172 if (event) {
1173 if (event->ref)
1174 ret = -EBUSY;
1175 else {
1176 ret = unregister_synth_event(event);
1177 if (!ret) {
1178 dyn_event_remove(&event->devent);
1179 free_synth_event(event);
1180 }
1181 }
1182 } else
1183 ret = -ENOENT;
1184 mutex_unlock(&event_mutex);
1185 return ret;
1143 } 1186 }
1144 1187
1145 list_splice_init(&event->list, &release_events); 1188 ret = __create_synth_event(argc - 1, name, (const char **)argv + 1);
1189 return ret == -ECANCELED ? -EINVAL : ret;
1190}
1146 1191
1147 mutex_unlock(&synth_event_mutex); 1192static int synth_event_create(int argc, const char **argv)
1193{
1194 const char *name = argv[0];
1195 int len;
1148 1196
1149 list_for_each_entry_safe(event, e, &release_events, list) { 1197 if (name[0] != 's' || name[1] != ':')
1150 list_del(&event->list); 1198 return -ECANCELED;
1199 name += 2;
1151 1200
1152 ret = unregister_synth_event(event); 1201 /* This interface accepts group name prefix */
1153 add_or_delete_synth_event(event, !ret); 1202 if (strchr(name, '/')) {
1203 len = sizeof(SYNTH_SYSTEM "/") - 1;
1204 if (strncmp(name, SYNTH_SYSTEM "/", len))
1205 return -EINVAL;
1206 name += len;
1154 } 1207 }
1155 1208 return __create_synth_event(argc - 1, name, argv + 1);
1156 return ret;
1157} 1209}
1158 1210
1159 1211static int synth_event_release(struct dyn_event *ev)
1160static void *synth_events_seq_start(struct seq_file *m, loff_t *pos)
1161{ 1212{
1162 mutex_lock(&synth_event_mutex); 1213 struct synth_event *event = to_synth_event(ev);
1214 int ret;
1163 1215
1164 return seq_list_start(&synth_event_list, *pos); 1216 if (event->ref)
1165} 1217 return -EBUSY;
1166 1218
1167static void *synth_events_seq_next(struct seq_file *m, void *v, loff_t *pos) 1219 ret = unregister_synth_event(event);
1168{ 1220 if (ret)
1169 return seq_list_next(v, &synth_event_list, pos); 1221 return ret;
1170}
1171 1222
1172static void synth_events_seq_stop(struct seq_file *m, void *v) 1223 dyn_event_remove(ev);
1173{ 1224 free_synth_event(event);
1174 mutex_unlock(&synth_event_mutex); 1225 return 0;
1175} 1226}
1176 1227
1177static int synth_events_seq_show(struct seq_file *m, void *v) 1228static int __synth_event_show(struct seq_file *m, struct synth_event *event)
1178{ 1229{
1179 struct synth_field *field; 1230 struct synth_field *field;
1180 struct synth_event *event = v;
1181 unsigned int i; 1231 unsigned int i;
1182 1232
1183 seq_printf(m, "%s\t", event->name); 1233 seq_printf(m, "%s\t", event->name);
@@ -1195,11 +1245,30 @@ static int synth_events_seq_show(struct seq_file *m, void *v)
1195 return 0; 1245 return 0;
1196} 1246}
1197 1247
1248static int synth_event_show(struct seq_file *m, struct dyn_event *ev)
1249{
1250 struct synth_event *event = to_synth_event(ev);
1251
1252 seq_printf(m, "s:%s/", event->class.system);
1253
1254 return __synth_event_show(m, event);
1255}
1256
1257static int synth_events_seq_show(struct seq_file *m, void *v)
1258{
1259 struct dyn_event *ev = v;
1260
1261 if (!is_synth_event(ev))
1262 return 0;
1263
1264 return __synth_event_show(m, to_synth_event(ev));
1265}
1266
1198static const struct seq_operations synth_events_seq_op = { 1267static const struct seq_operations synth_events_seq_op = {
1199 .start = synth_events_seq_start, 1268 .start = dyn_event_seq_start,
1200 .next = synth_events_seq_next, 1269 .next = dyn_event_seq_next,
1201 .stop = synth_events_seq_stop, 1270 .stop = dyn_event_seq_stop,
1202 .show = synth_events_seq_show 1271 .show = synth_events_seq_show,
1203}; 1272};
1204 1273
1205static int synth_events_open(struct inode *inode, struct file *file) 1274static int synth_events_open(struct inode *inode, struct file *file)
@@ -1207,7 +1276,7 @@ static int synth_events_open(struct inode *inode, struct file *file)
1207 int ret; 1276 int ret;
1208 1277
1209 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) { 1278 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
1210 ret = release_all_synth_events(); 1279 ret = dyn_events_release_all(&synth_event_ops);
1211 if (ret < 0) 1280 if (ret < 0)
1212 return ret; 1281 return ret;
1213 } 1282 }
@@ -1220,7 +1289,7 @@ static ssize_t synth_events_write(struct file *file,
1220 size_t count, loff_t *ppos) 1289 size_t count, loff_t *ppos)
1221{ 1290{
1222 return trace_parse_run_command(file, buffer, count, ppos, 1291 return trace_parse_run_command(file, buffer, count, ppos,
1223 create_synth_event); 1292 create_or_delete_synth_event);
1224} 1293}
1225 1294
1226static const struct file_operations synth_events_fops = { 1295static const struct file_operations synth_events_fops = {
@@ -1257,82 +1326,73 @@ static u64 hist_field_cpu(struct hist_field *hist_field,
1257 return cpu; 1326 return cpu;
1258} 1327}
1259 1328
1329/**
1330 * check_field_for_var_ref - Check if a VAR_REF field references a variable
1331 * @hist_field: The VAR_REF field to check
1332 * @var_data: The hist trigger that owns the variable
1333 * @var_idx: The trigger variable identifier
1334 *
1335 * Check the given VAR_REF field to see whether or not it references
1336 * the given variable associated with the given trigger.
1337 *
1338 * Return: The VAR_REF field if it does reference the variable, NULL if not
1339 */
1260static struct hist_field * 1340static struct hist_field *
1261check_field_for_var_ref(struct hist_field *hist_field, 1341check_field_for_var_ref(struct hist_field *hist_field,
1262 struct hist_trigger_data *var_data, 1342 struct hist_trigger_data *var_data,
1263 unsigned int var_idx) 1343 unsigned int var_idx)
1264{ 1344{
1265 struct hist_field *found = NULL; 1345 WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
1266
1267 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF) {
1268 if (hist_field->var.idx == var_idx &&
1269 hist_field->var.hist_data == var_data) {
1270 found = hist_field;
1271 }
1272 }
1273 1346
1274 return found; 1347 if (hist_field && hist_field->var.idx == var_idx &&
1275} 1348 hist_field->var.hist_data == var_data)
1276 1349 return hist_field;
1277static struct hist_field *
1278check_field_for_var_refs(struct hist_trigger_data *hist_data,
1279 struct hist_field *hist_field,
1280 struct hist_trigger_data *var_data,
1281 unsigned int var_idx,
1282 unsigned int level)
1283{
1284 struct hist_field *found = NULL;
1285 unsigned int i;
1286
1287 if (level > 3)
1288 return found;
1289
1290 if (!hist_field)
1291 return found;
1292
1293 found = check_field_for_var_ref(hist_field, var_data, var_idx);
1294 if (found)
1295 return found;
1296
1297 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
1298 struct hist_field *operand;
1299
1300 operand = hist_field->operands[i];
1301 found = check_field_for_var_refs(hist_data, operand, var_data,
1302 var_idx, level + 1);
1303 if (found)
1304 return found;
1305 }
1306 1350
1307 return found; 1351 return NULL;
1308} 1352}
1309 1353
1354/**
1355 * find_var_ref - Check if a trigger has a reference to a trigger variable
1356 * @hist_data: The hist trigger that might have a reference to the variable
1357 * @var_data: The hist trigger that owns the variable
1358 * @var_idx: The trigger variable identifier
1359 *
1360 * Check the list of var_refs[] on the first hist trigger to see
1361 * whether any of them are references to the variable on the second
1362 * trigger.
1363 *
1364 * Return: The VAR_REF field referencing the variable if so, NULL if not
1365 */
1310static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data, 1366static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
1311 struct hist_trigger_data *var_data, 1367 struct hist_trigger_data *var_data,
1312 unsigned int var_idx) 1368 unsigned int var_idx)
1313{ 1369{
1314 struct hist_field *hist_field, *found = NULL; 1370 struct hist_field *hist_field;
1315 unsigned int i; 1371 unsigned int i;
1316 1372
1317 for_each_hist_field(i, hist_data) { 1373 for (i = 0; i < hist_data->n_var_refs; i++) {
1318 hist_field = hist_data->fields[i]; 1374 hist_field = hist_data->var_refs[i];
1319 found = check_field_for_var_refs(hist_data, hist_field, 1375 if (check_field_for_var_ref(hist_field, var_data, var_idx))
1320 var_data, var_idx, 0); 1376 return hist_field;
1321 if (found)
1322 return found;
1323 }
1324
1325 for (i = 0; i < hist_data->n_synth_var_refs; i++) {
1326 hist_field = hist_data->synth_var_refs[i];
1327 found = check_field_for_var_refs(hist_data, hist_field,
1328 var_data, var_idx, 0);
1329 if (found)
1330 return found;
1331 } 1377 }
1332 1378
1333 return found; 1379 return NULL;
1334} 1380}
1335 1381
1382/**
1383 * find_any_var_ref - Check if there is a reference to a given trigger variable
1384 * @hist_data: The hist trigger
1385 * @var_idx: The trigger variable identifier
1386 *
1387 * Check to see whether the given variable is currently referenced by
1388 * any other trigger.
1389 *
1390 * The trigger the variable is defined on is explicitly excluded - the
1391 * assumption being that a self-reference doesn't prevent a trigger
1392 * from being removed.
1393 *
1394 * Return: The VAR_REF field referencing the variable if so, NULL if not
1395 */
1336static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data, 1396static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
1337 unsigned int var_idx) 1397 unsigned int var_idx)
1338{ 1398{
@@ -1351,6 +1411,19 @@ static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
1351 return found; 1411 return found;
1352} 1412}
1353 1413
1414/**
1415 * check_var_refs - Check if there is a reference to any of trigger's variables
1416 * @hist_data: The hist trigger
1417 *
1418 * A trigger can define one or more variables. If any one of them is
1419 * currently referenced by any other trigger, this function will
1420 * determine that.
1421
1422 * Typically used to determine whether or not a trigger can be removed
1423 * - if there are any references to a trigger's variables, it cannot.
1424 *
1425 * Return: True if there is a reference to any of trigger's variables
1426 */
1354static bool check_var_refs(struct hist_trigger_data *hist_data) 1427static bool check_var_refs(struct hist_trigger_data *hist_data)
1355{ 1428{
1356 struct hist_field *field; 1429 struct hist_field *field;
@@ -1808,8 +1881,8 @@ static int parse_action(char *str, struct hist_trigger_attrs *attrs)
1808 if (attrs->n_actions >= HIST_ACTIONS_MAX) 1881 if (attrs->n_actions >= HIST_ACTIONS_MAX)
1809 return ret; 1882 return ret;
1810 1883
1811 if ((strncmp(str, "onmatch(", strlen("onmatch(")) == 0) || 1884 if ((str_has_prefix(str, "onmatch(")) ||
1812 (strncmp(str, "onmax(", strlen("onmax(")) == 0)) { 1885 (str_has_prefix(str, "onmax("))) {
1813 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL); 1886 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
1814 if (!attrs->action_str[attrs->n_actions]) { 1887 if (!attrs->action_str[attrs->n_actions]) {
1815 ret = -ENOMEM; 1888 ret = -ENOMEM;
@@ -1826,34 +1899,34 @@ static int parse_assignment(char *str, struct hist_trigger_attrs *attrs)
1826{ 1899{
1827 int ret = 0; 1900 int ret = 0;
1828 1901
1829 if ((strncmp(str, "key=", strlen("key=")) == 0) || 1902 if ((str_has_prefix(str, "key=")) ||
1830 (strncmp(str, "keys=", strlen("keys=")) == 0)) { 1903 (str_has_prefix(str, "keys="))) {
1831 attrs->keys_str = kstrdup(str, GFP_KERNEL); 1904 attrs->keys_str = kstrdup(str, GFP_KERNEL);
1832 if (!attrs->keys_str) { 1905 if (!attrs->keys_str) {
1833 ret = -ENOMEM; 1906 ret = -ENOMEM;
1834 goto out; 1907 goto out;
1835 } 1908 }
1836 } else if ((strncmp(str, "val=", strlen("val=")) == 0) || 1909 } else if ((str_has_prefix(str, "val=")) ||
1837 (strncmp(str, "vals=", strlen("vals=")) == 0) || 1910 (str_has_prefix(str, "vals=")) ||
1838 (strncmp(str, "values=", strlen("values=")) == 0)) { 1911 (str_has_prefix(str, "values="))) {
1839 attrs->vals_str = kstrdup(str, GFP_KERNEL); 1912 attrs->vals_str = kstrdup(str, GFP_KERNEL);
1840 if (!attrs->vals_str) { 1913 if (!attrs->vals_str) {
1841 ret = -ENOMEM; 1914 ret = -ENOMEM;
1842 goto out; 1915 goto out;
1843 } 1916 }
1844 } else if (strncmp(str, "sort=", strlen("sort=")) == 0) { 1917 } else if (str_has_prefix(str, "sort=")) {
1845 attrs->sort_key_str = kstrdup(str, GFP_KERNEL); 1918 attrs->sort_key_str = kstrdup(str, GFP_KERNEL);
1846 if (!attrs->sort_key_str) { 1919 if (!attrs->sort_key_str) {
1847 ret = -ENOMEM; 1920 ret = -ENOMEM;
1848 goto out; 1921 goto out;
1849 } 1922 }
1850 } else if (strncmp(str, "name=", strlen("name=")) == 0) { 1923 } else if (str_has_prefix(str, "name=")) {
1851 attrs->name = kstrdup(str, GFP_KERNEL); 1924 attrs->name = kstrdup(str, GFP_KERNEL);
1852 if (!attrs->name) { 1925 if (!attrs->name) {
1853 ret = -ENOMEM; 1926 ret = -ENOMEM;
1854 goto out; 1927 goto out;
1855 } 1928 }
1856 } else if (strncmp(str, "clock=", strlen("clock=")) == 0) { 1929 } else if (str_has_prefix(str, "clock=")) {
1857 strsep(&str, "="); 1930 strsep(&str, "=");
1858 if (!str) { 1931 if (!str) {
1859 ret = -EINVAL; 1932 ret = -EINVAL;
@@ -1866,7 +1939,7 @@ static int parse_assignment(char *str, struct hist_trigger_attrs *attrs)
1866 ret = -ENOMEM; 1939 ret = -ENOMEM;
1867 goto out; 1940 goto out;
1868 } 1941 }
1869 } else if (strncmp(str, "size=", strlen("size=")) == 0) { 1942 } else if (str_has_prefix(str, "size=")) {
1870 int map_bits = parse_map_size(str); 1943 int map_bits = parse_map_size(str);
1871 1944
1872 if (map_bits < 0) { 1945 if (map_bits < 0) {
@@ -2151,6 +2224,15 @@ static int contains_operator(char *str)
2151 return field_op; 2224 return field_op;
2152} 2225}
2153 2226
2227static void __destroy_hist_field(struct hist_field *hist_field)
2228{
2229 kfree(hist_field->var.name);
2230 kfree(hist_field->name);
2231 kfree(hist_field->type);
2232
2233 kfree(hist_field);
2234}
2235
2154static void destroy_hist_field(struct hist_field *hist_field, 2236static void destroy_hist_field(struct hist_field *hist_field,
2155 unsigned int level) 2237 unsigned int level)
2156{ 2238{
@@ -2162,14 +2244,13 @@ static void destroy_hist_field(struct hist_field *hist_field,
2162 if (!hist_field) 2244 if (!hist_field)
2163 return; 2245 return;
2164 2246
2247 if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
2248 return; /* var refs will be destroyed separately */
2249
2165 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) 2250 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
2166 destroy_hist_field(hist_field->operands[i], level + 1); 2251 destroy_hist_field(hist_field->operands[i], level + 1);
2167 2252
2168 kfree(hist_field->var.name); 2253 __destroy_hist_field(hist_field);
2169 kfree(hist_field->name);
2170 kfree(hist_field->type);
2171
2172 kfree(hist_field);
2173} 2254}
2174 2255
2175static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data, 2256static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
@@ -2296,6 +2377,12 @@ static void destroy_hist_fields(struct hist_trigger_data *hist_data)
2296 hist_data->fields[i] = NULL; 2377 hist_data->fields[i] = NULL;
2297 } 2378 }
2298 } 2379 }
2380
2381 for (i = 0; i < hist_data->n_var_refs; i++) {
2382 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
2383 __destroy_hist_field(hist_data->var_refs[i]);
2384 hist_data->var_refs[i] = NULL;
2385 }
2299} 2386}
2300 2387
2301static int init_var_ref(struct hist_field *ref_field, 2388static int init_var_ref(struct hist_field *ref_field,
@@ -2354,7 +2441,23 @@ static int init_var_ref(struct hist_field *ref_field,
2354 goto out; 2441 goto out;
2355} 2442}
2356 2443
2357static struct hist_field *create_var_ref(struct hist_field *var_field, 2444/**
2445 * create_var_ref - Create a variable reference and attach it to trigger
2446 * @hist_data: The trigger that will be referencing the variable
2447 * @var_field: The VAR field to create a reference to
2448 * @system: The optional system string
2449 * @event_name: The optional event_name string
2450 *
2451 * Given a variable hist_field, create a VAR_REF hist_field that
2452 * represents a reference to it.
2453 *
2454 * This function also adds the reference to the trigger that
2455 * now references the variable.
2456 *
2457 * Return: The VAR_REF field if successful, NULL if not
2458 */
2459static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
2460 struct hist_field *var_field,
2358 char *system, char *event_name) 2461 char *system, char *event_name)
2359{ 2462{
2360 unsigned long flags = HIST_FIELD_FL_VAR_REF; 2463 unsigned long flags = HIST_FIELD_FL_VAR_REF;
@@ -2366,6 +2469,9 @@ static struct hist_field *create_var_ref(struct hist_field *var_field,
2366 destroy_hist_field(ref_field, 0); 2469 destroy_hist_field(ref_field, 0);
2367 return NULL; 2470 return NULL;
2368 } 2471 }
2472
2473 hist_data->var_refs[hist_data->n_var_refs] = ref_field;
2474 ref_field->var_ref_idx = hist_data->n_var_refs++;
2369 } 2475 }
2370 2476
2371 return ref_field; 2477 return ref_field;
@@ -2439,7 +2545,8 @@ static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
2439 2545
2440 var_field = find_event_var(hist_data, system, event_name, var_name); 2546 var_field = find_event_var(hist_data, system, event_name, var_name);
2441 if (var_field) 2547 if (var_field)
2442 ref_field = create_var_ref(var_field, system, event_name); 2548 ref_field = create_var_ref(hist_data, var_field,
2549 system, event_name);
2443 2550
2444 if (!ref_field) 2551 if (!ref_field)
2445 hist_err_event("Couldn't find variable: $", 2552 hist_err_event("Couldn't find variable: $",
@@ -2557,8 +2664,6 @@ static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2557 if (!s) { 2664 if (!s) {
2558 hist_field = parse_var_ref(hist_data, ref_system, ref_event, ref_var); 2665 hist_field = parse_var_ref(hist_data, ref_system, ref_event, ref_var);
2559 if (hist_field) { 2666 if (hist_field) {
2560 hist_data->var_refs[hist_data->n_var_refs] = hist_field;
2561 hist_field->var_ref_idx = hist_data->n_var_refs++;
2562 if (var_name) { 2667 if (var_name) {
2563 hist_field = create_alias(hist_data, hist_field, var_name); 2668 hist_field = create_alias(hist_data, hist_field, var_name);
2564 if (!hist_field) { 2669 if (!hist_field) {
@@ -3332,7 +3437,6 @@ static int onmax_create(struct hist_trigger_data *hist_data,
3332 unsigned int var_ref_idx = hist_data->n_var_refs; 3437 unsigned int var_ref_idx = hist_data->n_var_refs;
3333 struct field_var *field_var; 3438 struct field_var *field_var;
3334 char *onmax_var_str, *param; 3439 char *onmax_var_str, *param;
3335 unsigned long flags;
3336 unsigned int i; 3440 unsigned int i;
3337 int ret = 0; 3441 int ret = 0;
3338 3442
@@ -3349,18 +3453,10 @@ static int onmax_create(struct hist_trigger_data *hist_data,
3349 return -EINVAL; 3453 return -EINVAL;
3350 } 3454 }
3351 3455
3352 flags = HIST_FIELD_FL_VAR_REF; 3456 ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3353 ref_field = create_hist_field(hist_data, NULL, flags, NULL);
3354 if (!ref_field) 3457 if (!ref_field)
3355 return -ENOMEM; 3458 return -ENOMEM;
3356 3459
3357 if (init_var_ref(ref_field, var_field, NULL, NULL)) {
3358 destroy_hist_field(ref_field, 0);
3359 ret = -ENOMEM;
3360 goto out;
3361 }
3362 hist_data->var_refs[hist_data->n_var_refs] = ref_field;
3363 ref_field->var_ref_idx = hist_data->n_var_refs++;
3364 data->onmax.var = ref_field; 3460 data->onmax.var = ref_field;
3365 3461
3366 data->fn = onmax_save; 3462 data->fn = onmax_save;
@@ -3462,7 +3558,7 @@ static struct action_data *onmax_parse(char *str)
3462 if (!onmax_fn_name || !str) 3558 if (!onmax_fn_name || !str)
3463 goto free; 3559 goto free;
3464 3560
3465 if (strncmp(onmax_fn_name, "save", strlen("save")) == 0) { 3561 if (str_has_prefix(onmax_fn_name, "save")) {
3466 char *params = strsep(&str, ")"); 3562 char *params = strsep(&str, ")");
3467 3563
3468 if (!params) { 3564 if (!params) {
@@ -3493,7 +3589,7 @@ static void onmatch_destroy(struct action_data *data)
3493{ 3589{
3494 unsigned int i; 3590 unsigned int i;
3495 3591
3496 mutex_lock(&synth_event_mutex); 3592 lockdep_assert_held(&event_mutex);
3497 3593
3498 kfree(data->onmatch.match_event); 3594 kfree(data->onmatch.match_event);
3499 kfree(data->onmatch.match_event_system); 3595 kfree(data->onmatch.match_event_system);
@@ -3506,8 +3602,6 @@ static void onmatch_destroy(struct action_data *data)
3506 data->onmatch.synth_event->ref--; 3602 data->onmatch.synth_event->ref--;
3507 3603
3508 kfree(data); 3604 kfree(data);
3509
3510 mutex_unlock(&synth_event_mutex);
3511} 3605}
3512 3606
3513static void destroy_field_var(struct field_var *field_var) 3607static void destroy_field_var(struct field_var *field_var)
@@ -3539,23 +3633,6 @@ static void save_field_var(struct hist_trigger_data *hist_data,
3539} 3633}
3540 3634
3541 3635
3542static void destroy_synth_var_refs(struct hist_trigger_data *hist_data)
3543{
3544 unsigned int i;
3545
3546 for (i = 0; i < hist_data->n_synth_var_refs; i++)
3547 destroy_hist_field(hist_data->synth_var_refs[i], 0);
3548}
3549
3550static void save_synth_var_ref(struct hist_trigger_data *hist_data,
3551 struct hist_field *var_ref)
3552{
3553 hist_data->synth_var_refs[hist_data->n_synth_var_refs++] = var_ref;
3554
3555 hist_data->var_refs[hist_data->n_var_refs] = var_ref;
3556 var_ref->var_ref_idx = hist_data->n_var_refs++;
3557}
3558
3559static int check_synth_field(struct synth_event *event, 3636static int check_synth_field(struct synth_event *event,
3560 struct hist_field *hist_field, 3637 struct hist_field *hist_field,
3561 unsigned int field_pos) 3638 unsigned int field_pos)
@@ -3658,15 +3735,14 @@ static int onmatch_create(struct hist_trigger_data *hist_data,
3658 struct synth_event *event; 3735 struct synth_event *event;
3659 int ret = 0; 3736 int ret = 0;
3660 3737
3661 mutex_lock(&synth_event_mutex); 3738 lockdep_assert_held(&event_mutex);
3739
3662 event = find_synth_event(data->onmatch.synth_event_name); 3740 event = find_synth_event(data->onmatch.synth_event_name);
3663 if (!event) { 3741 if (!event) {
3664 hist_err("onmatch: Couldn't find synthetic event: ", data->onmatch.synth_event_name); 3742 hist_err("onmatch: Couldn't find synthetic event: ", data->onmatch.synth_event_name);
3665 mutex_unlock(&synth_event_mutex);
3666 return -EINVAL; 3743 return -EINVAL;
3667 } 3744 }
3668 event->ref++; 3745 event->ref++;
3669 mutex_unlock(&synth_event_mutex);
3670 3746
3671 var_ref_idx = hist_data->n_var_refs; 3747 var_ref_idx = hist_data->n_var_refs;
3672 3748
@@ -3708,14 +3784,14 @@ static int onmatch_create(struct hist_trigger_data *hist_data,
3708 } 3784 }
3709 3785
3710 if (check_synth_field(event, hist_field, field_pos) == 0) { 3786 if (check_synth_field(event, hist_field, field_pos) == 0) {
3711 var_ref = create_var_ref(hist_field, system, event_name); 3787 var_ref = create_var_ref(hist_data, hist_field,
3788 system, event_name);
3712 if (!var_ref) { 3789 if (!var_ref) {
3713 kfree(p); 3790 kfree(p);
3714 ret = -ENOMEM; 3791 ret = -ENOMEM;
3715 goto err; 3792 goto err;
3716 } 3793 }
3717 3794
3718 save_synth_var_ref(hist_data, var_ref);
3719 field_pos++; 3795 field_pos++;
3720 kfree(p); 3796 kfree(p);
3721 continue; 3797 continue;
@@ -3740,9 +3816,7 @@ static int onmatch_create(struct hist_trigger_data *hist_data,
3740 out: 3816 out:
3741 return ret; 3817 return ret;
3742 err: 3818 err:
3743 mutex_lock(&synth_event_mutex);
3744 event->ref--; 3819 event->ref--;
3745 mutex_unlock(&synth_event_mutex);
3746 3820
3747 goto out; 3821 goto out;
3748} 3822}
@@ -4268,12 +4342,13 @@ static int parse_actions(struct hist_trigger_data *hist_data)
4268 unsigned int i; 4342 unsigned int i;
4269 int ret = 0; 4343 int ret = 0;
4270 char *str; 4344 char *str;
4345 int len;
4271 4346
4272 for (i = 0; i < hist_data->attrs->n_actions; i++) { 4347 for (i = 0; i < hist_data->attrs->n_actions; i++) {
4273 str = hist_data->attrs->action_str[i]; 4348 str = hist_data->attrs->action_str[i];
4274 4349
4275 if (strncmp(str, "onmatch(", strlen("onmatch(")) == 0) { 4350 if ((len = str_has_prefix(str, "onmatch("))) {
4276 char *action_str = str + strlen("onmatch("); 4351 char *action_str = str + len;
4277 4352
4278 data = onmatch_parse(tr, action_str); 4353 data = onmatch_parse(tr, action_str);
4279 if (IS_ERR(data)) { 4354 if (IS_ERR(data)) {
@@ -4281,8 +4356,8 @@ static int parse_actions(struct hist_trigger_data *hist_data)
4281 break; 4356 break;
4282 } 4357 }
4283 data->fn = action_trace; 4358 data->fn = action_trace;
4284 } else if (strncmp(str, "onmax(", strlen("onmax(")) == 0) { 4359 } else if ((len = str_has_prefix(str, "onmax("))) {
4285 char *action_str = str + strlen("onmax("); 4360 char *action_str = str + len;
4286 4361
4287 data = onmax_parse(action_str); 4362 data = onmax_parse(action_str);
4288 if (IS_ERR(data)) { 4363 if (IS_ERR(data)) {
@@ -4461,7 +4536,6 @@ static void destroy_hist_data(struct hist_trigger_data *hist_data)
4461 destroy_actions(hist_data); 4536 destroy_actions(hist_data);
4462 destroy_field_vars(hist_data); 4537 destroy_field_vars(hist_data);
4463 destroy_field_var_hists(hist_data); 4538 destroy_field_var_hists(hist_data);
4464 destroy_synth_var_refs(hist_data);
4465 4539
4466 kfree(hist_data); 4540 kfree(hist_data);
4467} 4541}
@@ -5450,6 +5524,8 @@ static void hist_unreg_all(struct trace_event_file *file)
5450 struct synth_event *se; 5524 struct synth_event *se;
5451 const char *se_name; 5525 const char *se_name;
5452 5526
5527 lockdep_assert_held(&event_mutex);
5528
5453 if (hist_file_check_refs(file)) 5529 if (hist_file_check_refs(file))
5454 return; 5530 return;
5455 5531
@@ -5459,12 +5535,10 @@ static void hist_unreg_all(struct trace_event_file *file)
5459 list_del_rcu(&test->list); 5535 list_del_rcu(&test->list);
5460 trace_event_trigger_enable_disable(file, 0); 5536 trace_event_trigger_enable_disable(file, 0);
5461 5537
5462 mutex_lock(&synth_event_mutex);
5463 se_name = trace_event_name(file->event_call); 5538 se_name = trace_event_name(file->event_call);
5464 se = find_synth_event(se_name); 5539 se = find_synth_event(se_name);
5465 if (se) 5540 if (se)
5466 se->ref--; 5541 se->ref--;
5467 mutex_unlock(&synth_event_mutex);
5468 5542
5469 update_cond_flag(file); 5543 update_cond_flag(file);
5470 if (hist_data->enable_timestamps) 5544 if (hist_data->enable_timestamps)
@@ -5490,6 +5564,8 @@ static int event_hist_trigger_func(struct event_command *cmd_ops,
5490 char *trigger, *p; 5564 char *trigger, *p;
5491 int ret = 0; 5565 int ret = 0;
5492 5566
5567 lockdep_assert_held(&event_mutex);
5568
5493 if (glob && strlen(glob)) { 5569 if (glob && strlen(glob)) {
5494 last_cmd_set(param); 5570 last_cmd_set(param);
5495 hist_err_clear(); 5571 hist_err_clear();
@@ -5516,9 +5592,9 @@ static int event_hist_trigger_func(struct event_command *cmd_ops,
5516 p++; 5592 p++;
5517 continue; 5593 continue;
5518 } 5594 }
5519 if (p >= param + strlen(param) - strlen("if") - 1) 5595 if (p >= param + strlen(param) - (sizeof("if") - 1) - 1)
5520 return -EINVAL; 5596 return -EINVAL;
5521 if (*(p + strlen("if")) != ' ' && *(p + strlen("if")) != '\t') { 5597 if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') {
5522 p++; 5598 p++;
5523 continue; 5599 continue;
5524 } 5600 }
@@ -5580,14 +5656,10 @@ static int event_hist_trigger_func(struct event_command *cmd_ops,
5580 } 5656 }
5581 5657
5582 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file); 5658 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
5583
5584 mutex_lock(&synth_event_mutex);
5585 se_name = trace_event_name(file->event_call); 5659 se_name = trace_event_name(file->event_call);
5586 se = find_synth_event(se_name); 5660 se = find_synth_event(se_name);
5587 if (se) 5661 if (se)
5588 se->ref--; 5662 se->ref--;
5589 mutex_unlock(&synth_event_mutex);
5590
5591 ret = 0; 5663 ret = 0;
5592 goto out_free; 5664 goto out_free;
5593 } 5665 }
@@ -5623,13 +5695,10 @@ enable:
5623 if (ret) 5695 if (ret)
5624 goto out_unreg; 5696 goto out_unreg;
5625 5697
5626 mutex_lock(&synth_event_mutex);
5627 se_name = trace_event_name(file->event_call); 5698 se_name = trace_event_name(file->event_call);
5628 se = find_synth_event(se_name); 5699 se = find_synth_event(se_name);
5629 if (se) 5700 if (se)
5630 se->ref++; 5701 se->ref++;
5631 mutex_unlock(&synth_event_mutex);
5632
5633 /* Just return zero, not the number of registered triggers */ 5702 /* Just return zero, not the number of registered triggers */
5634 ret = 0; 5703 ret = 0;
5635 out: 5704 out:
@@ -5812,6 +5881,12 @@ static __init int trace_events_hist_init(void)
5812 struct dentry *d_tracer; 5881 struct dentry *d_tracer;
5813 int err = 0; 5882 int err = 0;
5814 5883
5884 err = dyn_event_register(&synth_event_ops);
5885 if (err) {
5886 pr_warn("Could not register synth_event_ops\n");
5887 return err;
5888 }
5889
5815 d_tracer = tracing_init_dentry(); 5890 d_tracer = tracing_init_dentry();
5816 if (IS_ERR(d_tracer)) { 5891 if (IS_ERR(d_tracer)) {
5817 err = PTR_ERR(d_tracer); 5892 err = PTR_ERR(d_tracer);
diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c
index 086af4f5c3e8..c2af1560e856 100644
--- a/kernel/trace/trace_functions_graph.c
+++ b/kernel/trace/trace_functions_graph.c
@@ -16,33 +16,6 @@
16#include "trace.h" 16#include "trace.h"
17#include "trace_output.h" 17#include "trace_output.h"
18 18
19static bool kill_ftrace_graph;
20
21/**
22 * ftrace_graph_is_dead - returns true if ftrace_graph_stop() was called
23 *
24 * ftrace_graph_stop() is called when a severe error is detected in
25 * the function graph tracing. This function is called by the critical
26 * paths of function graph to keep those paths from doing any more harm.
27 */
28bool ftrace_graph_is_dead(void)
29{
30 return kill_ftrace_graph;
31}
32
33/**
34 * ftrace_graph_stop - set to permanently disable function graph tracincg
35 *
36 * In case of an error int function graph tracing, this is called
37 * to try to keep function graph tracing from causing any more harm.
38 * Usually this is pretty severe and this is called to try to at least
39 * get a warning out to the user.
40 */
41void ftrace_graph_stop(void)
42{
43 kill_ftrace_graph = true;
44}
45
46/* When set, irq functions will be ignored */ 19/* When set, irq functions will be ignored */
47static int ftrace_graph_skip_irqs; 20static int ftrace_graph_skip_irqs;
48 21
@@ -87,8 +60,12 @@ static struct tracer_opt trace_opts[] = {
87 { TRACER_OPT(funcgraph-tail, TRACE_GRAPH_PRINT_TAIL) }, 60 { TRACER_OPT(funcgraph-tail, TRACE_GRAPH_PRINT_TAIL) },
88 /* Include sleep time (scheduled out) between entry and return */ 61 /* Include sleep time (scheduled out) between entry and return */
89 { TRACER_OPT(sleep-time, TRACE_GRAPH_SLEEP_TIME) }, 62 { TRACER_OPT(sleep-time, TRACE_GRAPH_SLEEP_TIME) },
63
64#ifdef CONFIG_FUNCTION_PROFILER
90 /* Include time within nested functions */ 65 /* Include time within nested functions */
91 { TRACER_OPT(graph-time, TRACE_GRAPH_GRAPH_TIME) }, 66 { TRACER_OPT(graph-time, TRACE_GRAPH_GRAPH_TIME) },
67#endif
68
92 { } /* Empty entry */ 69 { } /* Empty entry */
93}; 70};
94 71
@@ -117,258 +94,6 @@ static void
117print_graph_duration(struct trace_array *tr, unsigned long long duration, 94print_graph_duration(struct trace_array *tr, unsigned long long duration,
118 struct trace_seq *s, u32 flags); 95 struct trace_seq *s, u32 flags);
119 96
120/* Add a function return address to the trace stack on thread info.*/
121static int
122ftrace_push_return_trace(unsigned long ret, unsigned long func,
123 unsigned long frame_pointer, unsigned long *retp)
124{
125 unsigned long long calltime;
126 int index;
127
128 if (unlikely(ftrace_graph_is_dead()))
129 return -EBUSY;
130
131 if (!current->ret_stack)
132 return -EBUSY;
133
134 /*
135 * We must make sure the ret_stack is tested before we read
136 * anything else.
137 */
138 smp_rmb();
139
140 /* The return trace stack is full */
141 if (current->curr_ret_stack == FTRACE_RETFUNC_DEPTH - 1) {
142 atomic_inc(&current->trace_overrun);
143 return -EBUSY;
144 }
145
146 /*
147 * The curr_ret_stack is an index to ftrace return stack of
148 * current task. Its value should be in [0, FTRACE_RETFUNC_
149 * DEPTH) when the function graph tracer is used. To support
150 * filtering out specific functions, it makes the index
151 * negative by subtracting huge value (FTRACE_NOTRACE_DEPTH)
152 * so when it sees a negative index the ftrace will ignore
153 * the record. And the index gets recovered when returning
154 * from the filtered function by adding the FTRACE_NOTRACE_
155 * DEPTH and then it'll continue to record functions normally.
156 *
157 * The curr_ret_stack is initialized to -1 and get increased
158 * in this function. So it can be less than -1 only if it was
159 * filtered out via ftrace_graph_notrace_addr() which can be
160 * set from set_graph_notrace file in tracefs by user.
161 */
162 if (current->curr_ret_stack < -1)
163 return -EBUSY;
164
165 calltime = trace_clock_local();
166
167 index = ++current->curr_ret_stack;
168 if (ftrace_graph_notrace_addr(func))
169 current->curr_ret_stack -= FTRACE_NOTRACE_DEPTH;
170 barrier();
171 current->ret_stack[index].ret = ret;
172 current->ret_stack[index].func = func;
173 current->ret_stack[index].calltime = calltime;
174#ifdef HAVE_FUNCTION_GRAPH_FP_TEST
175 current->ret_stack[index].fp = frame_pointer;
176#endif
177#ifdef HAVE_FUNCTION_GRAPH_RET_ADDR_PTR
178 current->ret_stack[index].retp = retp;
179#endif
180 return 0;
181}
182
183int function_graph_enter(unsigned long ret, unsigned long func,
184 unsigned long frame_pointer, unsigned long *retp)
185{
186 struct ftrace_graph_ent trace;
187
188 trace.func = func;
189 trace.depth = ++current->curr_ret_depth;
190
191 if (ftrace_push_return_trace(ret, func,
192 frame_pointer, retp))
193 goto out;
194
195 /* Only trace if the calling function expects to */
196 if (!ftrace_graph_entry(&trace))
197 goto out_ret;
198
199 return 0;
200 out_ret:
201 current->curr_ret_stack--;
202 out:
203 current->curr_ret_depth--;
204 return -EBUSY;
205}
206
207/* Retrieve a function return address to the trace stack on thread info.*/
208static void
209ftrace_pop_return_trace(struct ftrace_graph_ret *trace, unsigned long *ret,
210 unsigned long frame_pointer)
211{
212 int index;
213
214 index = current->curr_ret_stack;
215
216 /*
217 * A negative index here means that it's just returned from a
218 * notrace'd function. Recover index to get an original
219 * return address. See ftrace_push_return_trace().
220 *
221 * TODO: Need to check whether the stack gets corrupted.
222 */
223 if (index < 0)
224 index += FTRACE_NOTRACE_DEPTH;
225
226 if (unlikely(index < 0 || index >= FTRACE_RETFUNC_DEPTH)) {
227 ftrace_graph_stop();
228 WARN_ON(1);
229 /* Might as well panic, otherwise we have no where to go */
230 *ret = (unsigned long)panic;
231 return;
232 }
233
234#ifdef HAVE_FUNCTION_GRAPH_FP_TEST
235 /*
236 * The arch may choose to record the frame pointer used
237 * and check it here to make sure that it is what we expect it
238 * to be. If gcc does not set the place holder of the return
239 * address in the frame pointer, and does a copy instead, then
240 * the function graph trace will fail. This test detects this
241 * case.
242 *
243 * Currently, x86_32 with optimize for size (-Os) makes the latest
244 * gcc do the above.
245 *
246 * Note, -mfentry does not use frame pointers, and this test
247 * is not needed if CC_USING_FENTRY is set.
248 */
249 if (unlikely(current->ret_stack[index].fp != frame_pointer)) {
250 ftrace_graph_stop();
251 WARN(1, "Bad frame pointer: expected %lx, received %lx\n"
252 " from func %ps return to %lx\n",
253 current->ret_stack[index].fp,
254 frame_pointer,
255 (void *)current->ret_stack[index].func,
256 current->ret_stack[index].ret);
257 *ret = (unsigned long)panic;
258 return;
259 }
260#endif
261
262 *ret = current->ret_stack[index].ret;
263 trace->func = current->ret_stack[index].func;
264 trace->calltime = current->ret_stack[index].calltime;
265 trace->overrun = atomic_read(&current->trace_overrun);
266 trace->depth = current->curr_ret_depth--;
267 /*
268 * We still want to trace interrupts coming in if
269 * max_depth is set to 1. Make sure the decrement is
270 * seen before ftrace_graph_return.
271 */
272 barrier();
273}
274
275/*
276 * Send the trace to the ring-buffer.
277 * @return the original return address.
278 */
279unsigned long ftrace_return_to_handler(unsigned long frame_pointer)
280{
281 struct ftrace_graph_ret trace;
282 unsigned long ret;
283
284 ftrace_pop_return_trace(&trace, &ret, frame_pointer);
285 trace.rettime = trace_clock_local();
286 ftrace_graph_return(&trace);
287 /*
288 * The ftrace_graph_return() may still access the current
289 * ret_stack structure, we need to make sure the update of
290 * curr_ret_stack is after that.
291 */
292 barrier();
293 current->curr_ret_stack--;
294 /*
295 * The curr_ret_stack can be less than -1 only if it was
296 * filtered out and it's about to return from the function.
297 * Recover the index and continue to trace normal functions.
298 */
299 if (current->curr_ret_stack < -1) {
300 current->curr_ret_stack += FTRACE_NOTRACE_DEPTH;
301 return ret;
302 }
303
304 if (unlikely(!ret)) {
305 ftrace_graph_stop();
306 WARN_ON(1);
307 /* Might as well panic. What else to do? */
308 ret = (unsigned long)panic;
309 }
310
311 return ret;
312}
313
314/**
315 * ftrace_graph_ret_addr - convert a potentially modified stack return address
316 * to its original value
317 *
318 * This function can be called by stack unwinding code to convert a found stack
319 * return address ('ret') to its original value, in case the function graph
320 * tracer has modified it to be 'return_to_handler'. If the address hasn't
321 * been modified, the unchanged value of 'ret' is returned.
322 *
323 * 'idx' is a state variable which should be initialized by the caller to zero
324 * before the first call.
325 *
326 * 'retp' is a pointer to the return address on the stack. It's ignored if
327 * the arch doesn't have HAVE_FUNCTION_GRAPH_RET_ADDR_PTR defined.
328 */
329#ifdef HAVE_FUNCTION_GRAPH_RET_ADDR_PTR
330unsigned long ftrace_graph_ret_addr(struct task_struct *task, int *idx,
331 unsigned long ret, unsigned long *retp)
332{
333 int index = task->curr_ret_stack;
334 int i;
335
336 if (ret != (unsigned long)return_to_handler)
337 return ret;
338
339 if (index < -1)
340 index += FTRACE_NOTRACE_DEPTH;
341
342 if (index < 0)
343 return ret;
344
345 for (i = 0; i <= index; i++)
346 if (task->ret_stack[i].retp == retp)
347 return task->ret_stack[i].ret;
348
349 return ret;
350}
351#else /* !HAVE_FUNCTION_GRAPH_RET_ADDR_PTR */
352unsigned long ftrace_graph_ret_addr(struct task_struct *task, int *idx,
353 unsigned long ret, unsigned long *retp)
354{
355 int task_idx;
356
357 if (ret != (unsigned long)return_to_handler)
358 return ret;
359
360 task_idx = task->curr_ret_stack;
361
362 if (!task->ret_stack || task_idx < *idx)
363 return ret;
364
365 task_idx -= *idx;
366 (*idx)++;
367
368 return task->ret_stack[task_idx].ret;
369}
370#endif /* HAVE_FUNCTION_GRAPH_RET_ADDR_PTR */
371
372int __trace_graph_entry(struct trace_array *tr, 97int __trace_graph_entry(struct trace_array *tr,
373 struct ftrace_graph_ent *trace, 98 struct ftrace_graph_ent *trace,
374 unsigned long flags, 99 unsigned long flags,
@@ -409,6 +134,18 @@ int trace_graph_entry(struct ftrace_graph_ent *trace)
409 int cpu; 134 int cpu;
410 int pc; 135 int pc;
411 136
137 if (trace_recursion_test(TRACE_GRAPH_NOTRACE_BIT))
138 return 0;
139
140 if (ftrace_graph_notrace_addr(trace->func)) {
141 trace_recursion_set(TRACE_GRAPH_NOTRACE_BIT);
142 /*
143 * Need to return 1 to have the return called
144 * that will clear the NOTRACE bit.
145 */
146 return 1;
147 }
148
412 if (!ftrace_trace_task(tr)) 149 if (!ftrace_trace_task(tr))
413 return 0; 150 return 0;
414 151
@@ -511,6 +248,11 @@ void trace_graph_return(struct ftrace_graph_ret *trace)
511 248
512 ftrace_graph_addr_finish(trace); 249 ftrace_graph_addr_finish(trace);
513 250
251 if (trace_recursion_test(TRACE_GRAPH_NOTRACE_BIT)) {
252 trace_recursion_clear(TRACE_GRAPH_NOTRACE_BIT);
253 return;
254 }
255
514 local_irq_save(flags); 256 local_irq_save(flags);
515 cpu = raw_smp_processor_id(); 257 cpu = raw_smp_processor_id();
516 data = per_cpu_ptr(tr->trace_buffer.data, cpu); 258 data = per_cpu_ptr(tr->trace_buffer.data, cpu);
@@ -536,6 +278,11 @@ static void trace_graph_thresh_return(struct ftrace_graph_ret *trace)
536{ 278{
537 ftrace_graph_addr_finish(trace); 279 ftrace_graph_addr_finish(trace);
538 280
281 if (trace_recursion_test(TRACE_GRAPH_NOTRACE_BIT)) {
282 trace_recursion_clear(TRACE_GRAPH_NOTRACE_BIT);
283 return;
284 }
285
539 if (tracing_thresh && 286 if (tracing_thresh &&
540 (trace->rettime - trace->calltime < tracing_thresh)) 287 (trace->rettime - trace->calltime < tracing_thresh))
541 return; 288 return;
@@ -543,17 +290,25 @@ static void trace_graph_thresh_return(struct ftrace_graph_ret *trace)
543 trace_graph_return(trace); 290 trace_graph_return(trace);
544} 291}
545 292
293static struct fgraph_ops funcgraph_thresh_ops = {
294 .entryfunc = &trace_graph_entry,
295 .retfunc = &trace_graph_thresh_return,
296};
297
298static struct fgraph_ops funcgraph_ops = {
299 .entryfunc = &trace_graph_entry,
300 .retfunc = &trace_graph_return,
301};
302
546static int graph_trace_init(struct trace_array *tr) 303static int graph_trace_init(struct trace_array *tr)
547{ 304{
548 int ret; 305 int ret;
549 306
550 set_graph_array(tr); 307 set_graph_array(tr);
551 if (tracing_thresh) 308 if (tracing_thresh)
552 ret = register_ftrace_graph(&trace_graph_thresh_return, 309 ret = register_ftrace_graph(&funcgraph_thresh_ops);
553 &trace_graph_entry);
554 else 310 else
555 ret = register_ftrace_graph(&trace_graph_return, 311 ret = register_ftrace_graph(&funcgraph_ops);
556 &trace_graph_entry);
557 if (ret) 312 if (ret)
558 return ret; 313 return ret;
559 tracing_start_cmdline_record(); 314 tracing_start_cmdline_record();
@@ -564,7 +319,10 @@ static int graph_trace_init(struct trace_array *tr)
564static void graph_trace_reset(struct trace_array *tr) 319static void graph_trace_reset(struct trace_array *tr)
565{ 320{
566 tracing_stop_cmdline_record(); 321 tracing_stop_cmdline_record();
567 unregister_ftrace_graph(); 322 if (tracing_thresh)
323 unregister_ftrace_graph(&funcgraph_thresh_ops);
324 else
325 unregister_ftrace_graph(&funcgraph_ops);
568} 326}
569 327
570static int graph_trace_update_thresh(struct trace_array *tr) 328static int graph_trace_update_thresh(struct trace_array *tr)
@@ -874,10 +632,6 @@ print_graph_entry_leaf(struct trace_iterator *iter,
874 632
875 cpu_data = per_cpu_ptr(data->cpu_data, cpu); 633 cpu_data = per_cpu_ptr(data->cpu_data, cpu);
876 634
877 /* If a graph tracer ignored set_graph_notrace */
878 if (call->depth < -1)
879 call->depth += FTRACE_NOTRACE_DEPTH;
880
881 /* 635 /*
882 * Comments display at + 1 to depth. Since 636 * Comments display at + 1 to depth. Since
883 * this is a leaf function, keep the comments 637 * this is a leaf function, keep the comments
@@ -920,10 +674,6 @@ print_graph_entry_nested(struct trace_iterator *iter,
920 struct fgraph_cpu_data *cpu_data; 674 struct fgraph_cpu_data *cpu_data;
921 int cpu = iter->cpu; 675 int cpu = iter->cpu;
922 676
923 /* If a graph tracer ignored set_graph_notrace */
924 if (call->depth < -1)
925 call->depth += FTRACE_NOTRACE_DEPTH;
926
927 cpu_data = per_cpu_ptr(data->cpu_data, cpu); 677 cpu_data = per_cpu_ptr(data->cpu_data, cpu);
928 cpu_data->depth = call->depth; 678 cpu_data->depth = call->depth;
929 679
diff --git a/kernel/trace/trace_irqsoff.c b/kernel/trace/trace_irqsoff.c
index 98ea6d28df15..d3294721f119 100644
--- a/kernel/trace/trace_irqsoff.c
+++ b/kernel/trace/trace_irqsoff.c
@@ -218,6 +218,11 @@ static void irqsoff_graph_return(struct ftrace_graph_ret *trace)
218 atomic_dec(&data->disabled); 218 atomic_dec(&data->disabled);
219} 219}
220 220
221static struct fgraph_ops fgraph_ops = {
222 .entryfunc = &irqsoff_graph_entry,
223 .retfunc = &irqsoff_graph_return,
224};
225
221static void irqsoff_trace_open(struct trace_iterator *iter) 226static void irqsoff_trace_open(struct trace_iterator *iter)
222{ 227{
223 if (is_graph(iter->tr)) 228 if (is_graph(iter->tr))
@@ -272,13 +277,6 @@ __trace_function(struct trace_array *tr,
272#else 277#else
273#define __trace_function trace_function 278#define __trace_function trace_function
274 279
275#ifdef CONFIG_FUNCTION_TRACER
276static int irqsoff_graph_entry(struct ftrace_graph_ent *trace)
277{
278 return -1;
279}
280#endif
281
282static enum print_line_t irqsoff_print_line(struct trace_iterator *iter) 280static enum print_line_t irqsoff_print_line(struct trace_iterator *iter)
283{ 281{
284 return TRACE_TYPE_UNHANDLED; 282 return TRACE_TYPE_UNHANDLED;
@@ -288,7 +286,6 @@ static void irqsoff_trace_open(struct trace_iterator *iter) { }
288static void irqsoff_trace_close(struct trace_iterator *iter) { } 286static void irqsoff_trace_close(struct trace_iterator *iter) { }
289 287
290#ifdef CONFIG_FUNCTION_TRACER 288#ifdef CONFIG_FUNCTION_TRACER
291static void irqsoff_graph_return(struct ftrace_graph_ret *trace) { }
292static void irqsoff_print_header(struct seq_file *s) 289static void irqsoff_print_header(struct seq_file *s)
293{ 290{
294 trace_default_header(s); 291 trace_default_header(s);
@@ -468,8 +465,7 @@ static int register_irqsoff_function(struct trace_array *tr, int graph, int set)
468 return 0; 465 return 0;
469 466
470 if (graph) 467 if (graph)
471 ret = register_ftrace_graph(&irqsoff_graph_return, 468 ret = register_ftrace_graph(&fgraph_ops);
472 &irqsoff_graph_entry);
473 else 469 else
474 ret = register_ftrace_function(tr->ops); 470 ret = register_ftrace_function(tr->ops);
475 471
@@ -485,7 +481,7 @@ static void unregister_irqsoff_function(struct trace_array *tr, int graph)
485 return; 481 return;
486 482
487 if (graph) 483 if (graph)
488 unregister_ftrace_graph(); 484 unregister_ftrace_graph(&fgraph_ops);
489 else 485 else
490 unregister_ftrace_function(tr->ops); 486 unregister_ftrace_function(tr->ops);
491 487
diff --git a/kernel/trace/trace_kprobe.c b/kernel/trace/trace_kprobe.c
index adc153ab51c0..5c19b8c41c7e 100644
--- a/kernel/trace/trace_kprobe.c
+++ b/kernel/trace/trace_kprobe.c
@@ -12,6 +12,7 @@
12#include <linux/rculist.h> 12#include <linux/rculist.h>
13#include <linux/error-injection.h> 13#include <linux/error-injection.h>
14 14
15#include "trace_dynevent.h"
15#include "trace_kprobe_selftest.h" 16#include "trace_kprobe_selftest.h"
16#include "trace_probe.h" 17#include "trace_probe.h"
17#include "trace_probe_tmpl.h" 18#include "trace_probe_tmpl.h"
@@ -19,17 +20,51 @@
19#define KPROBE_EVENT_SYSTEM "kprobes" 20#define KPROBE_EVENT_SYSTEM "kprobes"
20#define KRETPROBE_MAXACTIVE_MAX 4096 21#define KRETPROBE_MAXACTIVE_MAX 4096
21 22
23static int trace_kprobe_create(int argc, const char **argv);
24static int trace_kprobe_show(struct seq_file *m, struct dyn_event *ev);
25static int trace_kprobe_release(struct dyn_event *ev);
26static bool trace_kprobe_is_busy(struct dyn_event *ev);
27static bool trace_kprobe_match(const char *system, const char *event,
28 struct dyn_event *ev);
29
30static struct dyn_event_operations trace_kprobe_ops = {
31 .create = trace_kprobe_create,
32 .show = trace_kprobe_show,
33 .is_busy = trace_kprobe_is_busy,
34 .free = trace_kprobe_release,
35 .match = trace_kprobe_match,
36};
37
22/** 38/**
23 * Kprobe event core functions 39 * Kprobe event core functions
24 */ 40 */
25struct trace_kprobe { 41struct trace_kprobe {
26 struct list_head list; 42 struct dyn_event devent;
27 struct kretprobe rp; /* Use rp.kp for kprobe use */ 43 struct kretprobe rp; /* Use rp.kp for kprobe use */
28 unsigned long __percpu *nhit; 44 unsigned long __percpu *nhit;
29 const char *symbol; /* symbol name */ 45 const char *symbol; /* symbol name */
30 struct trace_probe tp; 46 struct trace_probe tp;
31}; 47};
32 48
49static bool is_trace_kprobe(struct dyn_event *ev)
50{
51 return ev->ops == &trace_kprobe_ops;
52}
53
54static struct trace_kprobe *to_trace_kprobe(struct dyn_event *ev)
55{
56 return container_of(ev, struct trace_kprobe, devent);
57}
58
59/**
60 * for_each_trace_kprobe - iterate over the trace_kprobe list
61 * @pos: the struct trace_kprobe * for each entry
62 * @dpos: the struct dyn_event * to use as a loop cursor
63 */
64#define for_each_trace_kprobe(pos, dpos) \
65 for_each_dyn_event(dpos) \
66 if (is_trace_kprobe(dpos) && (pos = to_trace_kprobe(dpos)))
67
33#define SIZEOF_TRACE_KPROBE(n) \ 68#define SIZEOF_TRACE_KPROBE(n) \
34 (offsetof(struct trace_kprobe, tp.args) + \ 69 (offsetof(struct trace_kprobe, tp.args) + \
35 (sizeof(struct probe_arg) * (n))) 70 (sizeof(struct probe_arg) * (n)))
@@ -81,6 +116,22 @@ static nokprobe_inline bool trace_kprobe_module_exist(struct trace_kprobe *tk)
81 return ret; 116 return ret;
82} 117}
83 118
119static bool trace_kprobe_is_busy(struct dyn_event *ev)
120{
121 struct trace_kprobe *tk = to_trace_kprobe(ev);
122
123 return trace_probe_is_enabled(&tk->tp);
124}
125
126static bool trace_kprobe_match(const char *system, const char *event,
127 struct dyn_event *ev)
128{
129 struct trace_kprobe *tk = to_trace_kprobe(ev);
130
131 return strcmp(trace_event_name(&tk->tp.call), event) == 0 &&
132 (!system || strcmp(tk->tp.call.class->system, system) == 0);
133}
134
84static nokprobe_inline unsigned long trace_kprobe_nhit(struct trace_kprobe *tk) 135static nokprobe_inline unsigned long trace_kprobe_nhit(struct trace_kprobe *tk)
85{ 136{
86 unsigned long nhit = 0; 137 unsigned long nhit = 0;
@@ -128,9 +179,6 @@ bool trace_kprobe_error_injectable(struct trace_event_call *call)
128static int register_kprobe_event(struct trace_kprobe *tk); 179static int register_kprobe_event(struct trace_kprobe *tk);
129static int unregister_kprobe_event(struct trace_kprobe *tk); 180static int unregister_kprobe_event(struct trace_kprobe *tk);
130 181
131static DEFINE_MUTEX(probe_lock);
132static LIST_HEAD(probe_list);
133
134static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs); 182static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs);
135static int kretprobe_dispatcher(struct kretprobe_instance *ri, 183static int kretprobe_dispatcher(struct kretprobe_instance *ri,
136 struct pt_regs *regs); 184 struct pt_regs *regs);
@@ -192,7 +240,7 @@ static struct trace_kprobe *alloc_trace_kprobe(const char *group,
192 if (!tk->tp.class.system) 240 if (!tk->tp.class.system)
193 goto error; 241 goto error;
194 242
195 INIT_LIST_HEAD(&tk->list); 243 dyn_event_init(&tk->devent, &trace_kprobe_ops);
196 INIT_LIST_HEAD(&tk->tp.files); 244 INIT_LIST_HEAD(&tk->tp.files);
197 return tk; 245 return tk;
198error: 246error:
@@ -207,6 +255,9 @@ static void free_trace_kprobe(struct trace_kprobe *tk)
207{ 255{
208 int i; 256 int i;
209 257
258 if (!tk)
259 return;
260
210 for (i = 0; i < tk->tp.nr_args; i++) 261 for (i = 0; i < tk->tp.nr_args; i++)
211 traceprobe_free_probe_arg(&tk->tp.args[i]); 262 traceprobe_free_probe_arg(&tk->tp.args[i]);
212 263
@@ -220,9 +271,10 @@ static void free_trace_kprobe(struct trace_kprobe *tk)
220static struct trace_kprobe *find_trace_kprobe(const char *event, 271static struct trace_kprobe *find_trace_kprobe(const char *event,
221 const char *group) 272 const char *group)
222{ 273{
274 struct dyn_event *pos;
223 struct trace_kprobe *tk; 275 struct trace_kprobe *tk;
224 276
225 list_for_each_entry(tk, &probe_list, list) 277 for_each_trace_kprobe(tk, pos)
226 if (strcmp(trace_event_name(&tk->tp.call), event) == 0 && 278 if (strcmp(trace_event_name(&tk->tp.call), event) == 0 &&
227 strcmp(tk->tp.call.class->system, group) == 0) 279 strcmp(tk->tp.call.class->system, group) == 0)
228 return tk; 280 return tk;
@@ -321,7 +373,7 @@ disable_trace_kprobe(struct trace_kprobe *tk, struct trace_event_file *file)
321 * created with perf_event_open. We don't need to wait for these 373 * created with perf_event_open. We don't need to wait for these
322 * trace_kprobes 374 * trace_kprobes
323 */ 375 */
324 if (list_empty(&tk->list)) 376 if (list_empty(&tk->devent.list))
325 wait = 0; 377 wait = 0;
326 out: 378 out:
327 if (wait) { 379 if (wait) {
@@ -419,7 +471,7 @@ static void __unregister_trace_kprobe(struct trace_kprobe *tk)
419 } 471 }
420} 472}
421 473
422/* Unregister a trace_probe and probe_event: call with locking probe_lock */ 474/* Unregister a trace_probe and probe_event */
423static int unregister_trace_kprobe(struct trace_kprobe *tk) 475static int unregister_trace_kprobe(struct trace_kprobe *tk)
424{ 476{
425 /* Enabled event can not be unregistered */ 477 /* Enabled event can not be unregistered */
@@ -431,7 +483,7 @@ static int unregister_trace_kprobe(struct trace_kprobe *tk)
431 return -EBUSY; 483 return -EBUSY;
432 484
433 __unregister_trace_kprobe(tk); 485 __unregister_trace_kprobe(tk);
434 list_del(&tk->list); 486 dyn_event_remove(&tk->devent);
435 487
436 return 0; 488 return 0;
437} 489}
@@ -442,7 +494,7 @@ static int register_trace_kprobe(struct trace_kprobe *tk)
442 struct trace_kprobe *old_tk; 494 struct trace_kprobe *old_tk;
443 int ret; 495 int ret;
444 496
445 mutex_lock(&probe_lock); 497 mutex_lock(&event_mutex);
446 498
447 /* Delete old (same name) event if exist */ 499 /* Delete old (same name) event if exist */
448 old_tk = find_trace_kprobe(trace_event_name(&tk->tp.call), 500 old_tk = find_trace_kprobe(trace_event_name(&tk->tp.call),
@@ -471,10 +523,10 @@ static int register_trace_kprobe(struct trace_kprobe *tk)
471 if (ret < 0) 523 if (ret < 0)
472 unregister_kprobe_event(tk); 524 unregister_kprobe_event(tk);
473 else 525 else
474 list_add_tail(&tk->list, &probe_list); 526 dyn_event_add(&tk->devent);
475 527
476end: 528end:
477 mutex_unlock(&probe_lock); 529 mutex_unlock(&event_mutex);
478 return ret; 530 return ret;
479} 531}
480 532
@@ -483,6 +535,7 @@ static int trace_kprobe_module_callback(struct notifier_block *nb,
483 unsigned long val, void *data) 535 unsigned long val, void *data)
484{ 536{
485 struct module *mod = data; 537 struct module *mod = data;
538 struct dyn_event *pos;
486 struct trace_kprobe *tk; 539 struct trace_kprobe *tk;
487 int ret; 540 int ret;
488 541
@@ -490,8 +543,8 @@ static int trace_kprobe_module_callback(struct notifier_block *nb,
490 return NOTIFY_DONE; 543 return NOTIFY_DONE;
491 544
492 /* Update probes on coming module */ 545 /* Update probes on coming module */
493 mutex_lock(&probe_lock); 546 mutex_lock(&event_mutex);
494 list_for_each_entry(tk, &probe_list, list) { 547 for_each_trace_kprobe(tk, pos) {
495 if (trace_kprobe_within_module(tk, mod)) { 548 if (trace_kprobe_within_module(tk, mod)) {
496 /* Don't need to check busy - this should have gone. */ 549 /* Don't need to check busy - this should have gone. */
497 __unregister_trace_kprobe(tk); 550 __unregister_trace_kprobe(tk);
@@ -502,7 +555,7 @@ static int trace_kprobe_module_callback(struct notifier_block *nb,
502 mod->name, ret); 555 mod->name, ret);
503 } 556 }
504 } 557 }
505 mutex_unlock(&probe_lock); 558 mutex_unlock(&event_mutex);
506 559
507 return NOTIFY_DONE; 560 return NOTIFY_DONE;
508} 561}
@@ -520,7 +573,7 @@ static inline void sanitize_event_name(char *name)
520 *name = '_'; 573 *name = '_';
521} 574}
522 575
523static int create_trace_kprobe(int argc, char **argv) 576static int trace_kprobe_create(int argc, const char *argv[])
524{ 577{
525 /* 578 /*
526 * Argument syntax: 579 * Argument syntax:
@@ -544,37 +597,37 @@ static int create_trace_kprobe(int argc, char **argv)
544 * FETCHARG:TYPE : use TYPE instead of unsigned long. 597 * FETCHARG:TYPE : use TYPE instead of unsigned long.
545 */ 598 */
546 struct trace_kprobe *tk; 599 struct trace_kprobe *tk;
547 int i, ret = 0; 600 int i, len, ret = 0;
548 bool is_return = false, is_delete = false; 601 bool is_return = false;
549 char *symbol = NULL, *event = NULL, *group = NULL; 602 char *symbol = NULL, *tmp = NULL;
603 const char *event = NULL, *group = KPROBE_EVENT_SYSTEM;
550 int maxactive = 0; 604 int maxactive = 0;
551 char *arg;
552 long offset = 0; 605 long offset = 0;
553 void *addr = NULL; 606 void *addr = NULL;
554 char buf[MAX_EVENT_NAME_LEN]; 607 char buf[MAX_EVENT_NAME_LEN];
555 unsigned int flags = TPARG_FL_KERNEL; 608 unsigned int flags = TPARG_FL_KERNEL;
556 609
557 /* argc must be >= 1 */ 610 /* argc must be >= 1 */
558 if (argv[0][0] == 'p') 611 if (argv[0][0] == 'r') {
559 is_return = false;
560 else if (argv[0][0] == 'r') {
561 is_return = true; 612 is_return = true;
562 flags |= TPARG_FL_RETURN; 613 flags |= TPARG_FL_RETURN;
563 } else if (argv[0][0] == '-') 614 } else if (argv[0][0] != 'p' || argc < 2)
564 is_delete = true; 615 return -ECANCELED;
565 else {
566 pr_info("Probe definition must be started with 'p', 'r' or"
567 " '-'.\n");
568 return -EINVAL;
569 }
570 616
571 event = strchr(&argv[0][1], ':'); 617 event = strchr(&argv[0][1], ':');
572 if (event) { 618 if (event)
573 event[0] = '\0';
574 event++; 619 event++;
575 } 620
576 if (is_return && isdigit(argv[0][1])) { 621 if (is_return && isdigit(argv[0][1])) {
577 ret = kstrtouint(&argv[0][1], 0, &maxactive); 622 if (event)
623 len = event - &argv[0][1] - 1;
624 else
625 len = strlen(&argv[0][1]);
626 if (len > MAX_EVENT_NAME_LEN - 1)
627 return -E2BIG;
628 memcpy(buf, &argv[0][1], len);
629 buf[len] = '\0';
630 ret = kstrtouint(buf, 0, &maxactive);
578 if (ret) { 631 if (ret) {
579 pr_info("Failed to parse maxactive.\n"); 632 pr_info("Failed to parse maxactive.\n");
580 return ret; 633 return ret;
@@ -589,74 +642,37 @@ static int create_trace_kprobe(int argc, char **argv)
589 } 642 }
590 } 643 }
591 644
592 if (event) {
593 char *slash;
594
595 slash = strchr(event, '/');
596 if (slash) {
597 group = event;
598 event = slash + 1;
599 slash[0] = '\0';
600 if (strlen(group) == 0) {
601 pr_info("Group name is not specified\n");
602 return -EINVAL;
603 }
604 }
605 if (strlen(event) == 0) {
606 pr_info("Event name is not specified\n");
607 return -EINVAL;
608 }
609 }
610 if (!group)
611 group = KPROBE_EVENT_SYSTEM;
612
613 if (is_delete) {
614 if (!event) {
615 pr_info("Delete command needs an event name.\n");
616 return -EINVAL;
617 }
618 mutex_lock(&probe_lock);
619 tk = find_trace_kprobe(event, group);
620 if (!tk) {
621 mutex_unlock(&probe_lock);
622 pr_info("Event %s/%s doesn't exist.\n", group, event);
623 return -ENOENT;
624 }
625 /* delete an event */
626 ret = unregister_trace_kprobe(tk);
627 if (ret == 0)
628 free_trace_kprobe(tk);
629 mutex_unlock(&probe_lock);
630 return ret;
631 }
632
633 if (argc < 2) {
634 pr_info("Probe point is not specified.\n");
635 return -EINVAL;
636 }
637
638 /* try to parse an address. if that fails, try to read the 645 /* try to parse an address. if that fails, try to read the
639 * input as a symbol. */ 646 * input as a symbol. */
640 if (kstrtoul(argv[1], 0, (unsigned long *)&addr)) { 647 if (kstrtoul(argv[1], 0, (unsigned long *)&addr)) {
648 /* Check whether uprobe event specified */
649 if (strchr(argv[1], '/') && strchr(argv[1], ':'))
650 return -ECANCELED;
641 /* a symbol specified */ 651 /* a symbol specified */
642 symbol = argv[1]; 652 symbol = kstrdup(argv[1], GFP_KERNEL);
653 if (!symbol)
654 return -ENOMEM;
643 /* TODO: support .init module functions */ 655 /* TODO: support .init module functions */
644 ret = traceprobe_split_symbol_offset(symbol, &offset); 656 ret = traceprobe_split_symbol_offset(symbol, &offset);
645 if (ret || offset < 0 || offset > UINT_MAX) { 657 if (ret || offset < 0 || offset > UINT_MAX) {
646 pr_info("Failed to parse either an address or a symbol.\n"); 658 pr_info("Failed to parse either an address or a symbol.\n");
647 return ret; 659 goto out;
648 } 660 }
649 if (kprobe_on_func_entry(NULL, symbol, offset)) 661 if (kprobe_on_func_entry(NULL, symbol, offset))
650 flags |= TPARG_FL_FENTRY; 662 flags |= TPARG_FL_FENTRY;
651 if (offset && is_return && !(flags & TPARG_FL_FENTRY)) { 663 if (offset && is_return && !(flags & TPARG_FL_FENTRY)) {
652 pr_info("Given offset is not valid for return probe.\n"); 664 pr_info("Given offset is not valid for return probe.\n");
653 return -EINVAL; 665 ret = -EINVAL;
666 goto out;
654 } 667 }
655 } 668 }
656 argc -= 2; argv += 2; 669 argc -= 2; argv += 2;
657 670
658 /* setup a probe */ 671 if (event) {
659 if (!event) { 672 ret = traceprobe_parse_event_name(&event, &group, buf);
673 if (ret)
674 goto out;
675 } else {
660 /* Make a new event name */ 676 /* Make a new event name */
661 if (symbol) 677 if (symbol)
662 snprintf(buf, MAX_EVENT_NAME_LEN, "%c_%s_%ld", 678 snprintf(buf, MAX_EVENT_NAME_LEN, "%c_%s_%ld",
@@ -667,121 +683,67 @@ static int create_trace_kprobe(int argc, char **argv)
667 sanitize_event_name(buf); 683 sanitize_event_name(buf);
668 event = buf; 684 event = buf;
669 } 685 }
686
687 /* setup a probe */
670 tk = alloc_trace_kprobe(group, event, addr, symbol, offset, maxactive, 688 tk = alloc_trace_kprobe(group, event, addr, symbol, offset, maxactive,
671 argc, is_return); 689 argc, is_return);
672 if (IS_ERR(tk)) { 690 if (IS_ERR(tk)) {
673 pr_info("Failed to allocate trace_probe.(%d)\n", 691 pr_info("Failed to allocate trace_probe.(%d)\n",
674 (int)PTR_ERR(tk)); 692 (int)PTR_ERR(tk));
675 return PTR_ERR(tk); 693 ret = PTR_ERR(tk);
694 goto out;
676 } 695 }
677 696
678 /* parse arguments */ 697 /* parse arguments */
679 ret = 0;
680 for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) { 698 for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) {
681 struct probe_arg *parg = &tk->tp.args[i]; 699 tmp = kstrdup(argv[i], GFP_KERNEL);
682 700 if (!tmp) {
683 /* Increment count for freeing args in error case */
684 tk->tp.nr_args++;
685
686 /* Parse argument name */
687 arg = strchr(argv[i], '=');
688 if (arg) {
689 *arg++ = '\0';
690 parg->name = kstrdup(argv[i], GFP_KERNEL);
691 } else {
692 arg = argv[i];
693 /* If argument name is omitted, set "argN" */
694 snprintf(buf, MAX_EVENT_NAME_LEN, "arg%d", i + 1);
695 parg->name = kstrdup(buf, GFP_KERNEL);
696 }
697
698 if (!parg->name) {
699 pr_info("Failed to allocate argument[%d] name.\n", i);
700 ret = -ENOMEM; 701 ret = -ENOMEM;
701 goto error; 702 goto error;
702 } 703 }
703 704
704 if (!is_good_name(parg->name)) { 705 ret = traceprobe_parse_probe_arg(&tk->tp, i, tmp, flags);
705 pr_info("Invalid argument[%d] name: %s\n", 706 kfree(tmp);
706 i, parg->name); 707 if (ret)
707 ret = -EINVAL;
708 goto error;
709 }
710
711 if (traceprobe_conflict_field_name(parg->name,
712 tk->tp.args, i)) {
713 pr_info("Argument[%d] name '%s' conflicts with "
714 "another field.\n", i, argv[i]);
715 ret = -EINVAL;
716 goto error;
717 }
718
719 /* Parse fetch argument */
720 ret = traceprobe_parse_probe_arg(arg, &tk->tp.size, parg,
721 flags);
722 if (ret) {
723 pr_info("Parse error at argument[%d]. (%d)\n", i, ret);
724 goto error; 708 goto error;
725 }
726 } 709 }
727 710
728 ret = register_trace_kprobe(tk); 711 ret = register_trace_kprobe(tk);
729 if (ret) 712 if (ret)
730 goto error; 713 goto error;
731 return 0; 714out:
715 kfree(symbol);
716 return ret;
732 717
733error: 718error:
734 free_trace_kprobe(tk); 719 free_trace_kprobe(tk);
735 return ret; 720 goto out;
736} 721}
737 722
738static int release_all_trace_kprobes(void) 723static int create_or_delete_trace_kprobe(int argc, char **argv)
739{ 724{
740 struct trace_kprobe *tk; 725 int ret;
741 int ret = 0;
742
743 mutex_lock(&probe_lock);
744 /* Ensure no probe is in use. */
745 list_for_each_entry(tk, &probe_list, list)
746 if (trace_probe_is_enabled(&tk->tp)) {
747 ret = -EBUSY;
748 goto end;
749 }
750 /* TODO: Use batch unregistration */
751 while (!list_empty(&probe_list)) {
752 tk = list_entry(probe_list.next, struct trace_kprobe, list);
753 ret = unregister_trace_kprobe(tk);
754 if (ret)
755 goto end;
756 free_trace_kprobe(tk);
757 }
758
759end:
760 mutex_unlock(&probe_lock);
761 726
762 return ret; 727 if (argv[0][0] == '-')
763} 728 return dyn_event_release(argc, argv, &trace_kprobe_ops);
764 729
765/* Probes listing interfaces */ 730 ret = trace_kprobe_create(argc, (const char **)argv);
766static void *probes_seq_start(struct seq_file *m, loff_t *pos) 731 return ret == -ECANCELED ? -EINVAL : ret;
767{
768 mutex_lock(&probe_lock);
769 return seq_list_start(&probe_list, *pos);
770} 732}
771 733
772static void *probes_seq_next(struct seq_file *m, void *v, loff_t *pos) 734static int trace_kprobe_release(struct dyn_event *ev)
773{ 735{
774 return seq_list_next(v, &probe_list, pos); 736 struct trace_kprobe *tk = to_trace_kprobe(ev);
775} 737 int ret = unregister_trace_kprobe(tk);
776 738
777static void probes_seq_stop(struct seq_file *m, void *v) 739 if (!ret)
778{ 740 free_trace_kprobe(tk);
779 mutex_unlock(&probe_lock); 741 return ret;
780} 742}
781 743
782static int probes_seq_show(struct seq_file *m, void *v) 744static int trace_kprobe_show(struct seq_file *m, struct dyn_event *ev)
783{ 745{
784 struct trace_kprobe *tk = v; 746 struct trace_kprobe *tk = to_trace_kprobe(ev);
785 int i; 747 int i;
786 748
787 seq_putc(m, trace_kprobe_is_return(tk) ? 'r' : 'p'); 749 seq_putc(m, trace_kprobe_is_return(tk) ? 'r' : 'p');
@@ -803,10 +765,20 @@ static int probes_seq_show(struct seq_file *m, void *v)
803 return 0; 765 return 0;
804} 766}
805 767
768static int probes_seq_show(struct seq_file *m, void *v)
769{
770 struct dyn_event *ev = v;
771
772 if (!is_trace_kprobe(ev))
773 return 0;
774
775 return trace_kprobe_show(m, ev);
776}
777
806static const struct seq_operations probes_seq_op = { 778static const struct seq_operations probes_seq_op = {
807 .start = probes_seq_start, 779 .start = dyn_event_seq_start,
808 .next = probes_seq_next, 780 .next = dyn_event_seq_next,
809 .stop = probes_seq_stop, 781 .stop = dyn_event_seq_stop,
810 .show = probes_seq_show 782 .show = probes_seq_show
811}; 783};
812 784
@@ -815,7 +787,7 @@ static int probes_open(struct inode *inode, struct file *file)
815 int ret; 787 int ret;
816 788
817 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) { 789 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
818 ret = release_all_trace_kprobes(); 790 ret = dyn_events_release_all(&trace_kprobe_ops);
819 if (ret < 0) 791 if (ret < 0)
820 return ret; 792 return ret;
821 } 793 }
@@ -827,7 +799,7 @@ static ssize_t probes_write(struct file *file, const char __user *buffer,
827 size_t count, loff_t *ppos) 799 size_t count, loff_t *ppos)
828{ 800{
829 return trace_parse_run_command(file, buffer, count, ppos, 801 return trace_parse_run_command(file, buffer, count, ppos,
830 create_trace_kprobe); 802 create_or_delete_trace_kprobe);
831} 803}
832 804
833static const struct file_operations kprobe_events_ops = { 805static const struct file_operations kprobe_events_ops = {
@@ -842,8 +814,13 @@ static const struct file_operations kprobe_events_ops = {
842/* Probes profiling interfaces */ 814/* Probes profiling interfaces */
843static int probes_profile_seq_show(struct seq_file *m, void *v) 815static int probes_profile_seq_show(struct seq_file *m, void *v)
844{ 816{
845 struct trace_kprobe *tk = v; 817 struct dyn_event *ev = v;
818 struct trace_kprobe *tk;
819
820 if (!is_trace_kprobe(ev))
821 return 0;
846 822
823 tk = to_trace_kprobe(ev);
847 seq_printf(m, " %-44s %15lu %15lu\n", 824 seq_printf(m, " %-44s %15lu %15lu\n",
848 trace_event_name(&tk->tp.call), 825 trace_event_name(&tk->tp.call),
849 trace_kprobe_nhit(tk), 826 trace_kprobe_nhit(tk),
@@ -853,9 +830,9 @@ static int probes_profile_seq_show(struct seq_file *m, void *v)
853} 830}
854 831
855static const struct seq_operations profile_seq_op = { 832static const struct seq_operations profile_seq_op = {
856 .start = probes_seq_start, 833 .start = dyn_event_seq_start,
857 .next = probes_seq_next, 834 .next = dyn_event_seq_next,
858 .stop = probes_seq_stop, 835 .stop = dyn_event_seq_stop,
859 .show = probes_profile_seq_show 836 .show = probes_profile_seq_show
860}; 837};
861 838
@@ -1408,7 +1385,7 @@ create_local_trace_kprobe(char *func, void *addr, unsigned long offs,
1408 char *event; 1385 char *event;
1409 1386
1410 /* 1387 /*
1411 * local trace_kprobes are not added to probe_list, so they are never 1388 * local trace_kprobes are not added to dyn_event, so they are never
1412 * searched in find_trace_kprobe(). Therefore, there is no concern of 1389 * searched in find_trace_kprobe(). Therefore, there is no concern of
1413 * duplicated name here. 1390 * duplicated name here.
1414 */ 1391 */
@@ -1466,6 +1443,11 @@ static __init int init_kprobe_trace(void)
1466{ 1443{
1467 struct dentry *d_tracer; 1444 struct dentry *d_tracer;
1468 struct dentry *entry; 1445 struct dentry *entry;
1446 int ret;
1447
1448 ret = dyn_event_register(&trace_kprobe_ops);
1449 if (ret)
1450 return ret;
1469 1451
1470 if (register_module_notifier(&trace_kprobe_module_nb)) 1452 if (register_module_notifier(&trace_kprobe_module_nb))
1471 return -EINVAL; 1453 return -EINVAL;
@@ -1523,9 +1505,8 @@ static __init int kprobe_trace_self_tests_init(void)
1523 1505
1524 pr_info("Testing kprobe tracing: "); 1506 pr_info("Testing kprobe tracing: ");
1525 1507
1526 ret = trace_run_command("p:testprobe kprobe_trace_selftest_target " 1508 ret = trace_run_command("p:testprobe kprobe_trace_selftest_target $stack $stack0 +0($stack)",
1527 "$stack $stack0 +0($stack)", 1509 create_or_delete_trace_kprobe);
1528 create_trace_kprobe);
1529 if (WARN_ON_ONCE(ret)) { 1510 if (WARN_ON_ONCE(ret)) {
1530 pr_warn("error on probing function entry.\n"); 1511 pr_warn("error on probing function entry.\n");
1531 warn++; 1512 warn++;
@@ -1545,8 +1526,8 @@ static __init int kprobe_trace_self_tests_init(void)
1545 } 1526 }
1546 } 1527 }
1547 1528
1548 ret = trace_run_command("r:testprobe2 kprobe_trace_selftest_target " 1529 ret = trace_run_command("r:testprobe2 kprobe_trace_selftest_target $retval",
1549 "$retval", create_trace_kprobe); 1530 create_or_delete_trace_kprobe);
1550 if (WARN_ON_ONCE(ret)) { 1531 if (WARN_ON_ONCE(ret)) {
1551 pr_warn("error on probing function return.\n"); 1532 pr_warn("error on probing function return.\n");
1552 warn++; 1533 warn++;
@@ -1616,20 +1597,24 @@ static __init int kprobe_trace_self_tests_init(void)
1616 disable_trace_kprobe(tk, file); 1597 disable_trace_kprobe(tk, file);
1617 } 1598 }
1618 1599
1619 ret = trace_run_command("-:testprobe", create_trace_kprobe); 1600 ret = trace_run_command("-:testprobe", create_or_delete_trace_kprobe);
1620 if (WARN_ON_ONCE(ret)) { 1601 if (WARN_ON_ONCE(ret)) {
1621 pr_warn("error on deleting a probe.\n"); 1602 pr_warn("error on deleting a probe.\n");
1622 warn++; 1603 warn++;
1623 } 1604 }
1624 1605
1625 ret = trace_run_command("-:testprobe2", create_trace_kprobe); 1606 ret = trace_run_command("-:testprobe2", create_or_delete_trace_kprobe);
1626 if (WARN_ON_ONCE(ret)) { 1607 if (WARN_ON_ONCE(ret)) {
1627 pr_warn("error on deleting a probe.\n"); 1608 pr_warn("error on deleting a probe.\n");
1628 warn++; 1609 warn++;
1629 } 1610 }
1630 1611
1631end: 1612end:
1632 release_all_trace_kprobes(); 1613 ret = dyn_events_release_all(&trace_kprobe_ops);
1614 if (WARN_ON_ONCE(ret)) {
1615 pr_warn("error on cleaning up probes.\n");
1616 warn++;
1617 }
1633 /* 1618 /*
1634 * Wait for the optimizer work to finish. Otherwise it might fiddle 1619 * Wait for the optimizer work to finish. Otherwise it might fiddle
1635 * with probes in already freed __init text. 1620 * with probes in already freed __init text.
diff --git a/kernel/trace/trace_output.c b/kernel/trace/trace_output.c
index 6e6cc64faa38..54373d93e251 100644
--- a/kernel/trace/trace_output.c
+++ b/kernel/trace/trace_output.c
@@ -339,43 +339,24 @@ static inline const char *kretprobed(const char *name)
339#endif /* CONFIG_KRETPROBES */ 339#endif /* CONFIG_KRETPROBES */
340 340
341static void 341static void
342seq_print_sym_short(struct trace_seq *s, const char *fmt, unsigned long address) 342seq_print_sym(struct trace_seq *s, unsigned long address, bool offset)
343{ 343{
344 char str[KSYM_SYMBOL_LEN];
345#ifdef CONFIG_KALLSYMS 344#ifdef CONFIG_KALLSYMS
346 const char *name;
347
348 kallsyms_lookup(address, NULL, NULL, NULL, str);
349
350 name = kretprobed(str);
351
352 if (name && strlen(name)) {
353 trace_seq_printf(s, fmt, name);
354 return;
355 }
356#endif
357 snprintf(str, KSYM_SYMBOL_LEN, "0x%08lx", address);
358 trace_seq_printf(s, fmt, str);
359}
360
361static void
362seq_print_sym_offset(struct trace_seq *s, const char *fmt,
363 unsigned long address)
364{
365 char str[KSYM_SYMBOL_LEN]; 345 char str[KSYM_SYMBOL_LEN];
366#ifdef CONFIG_KALLSYMS
367 const char *name; 346 const char *name;
368 347
369 sprint_symbol(str, address); 348 if (offset)
349 sprint_symbol(str, address);
350 else
351 kallsyms_lookup(address, NULL, NULL, NULL, str);
370 name = kretprobed(str); 352 name = kretprobed(str);
371 353
372 if (name && strlen(name)) { 354 if (name && strlen(name)) {
373 trace_seq_printf(s, fmt, name); 355 trace_seq_puts(s, name);
374 return; 356 return;
375 } 357 }
376#endif 358#endif
377 snprintf(str, KSYM_SYMBOL_LEN, "0x%08lx", address); 359 trace_seq_printf(s, "0x%08lx", address);
378 trace_seq_printf(s, fmt, str);
379} 360}
380 361
381#ifndef CONFIG_64BIT 362#ifndef CONFIG_64BIT
@@ -424,10 +405,7 @@ seq_print_ip_sym(struct trace_seq *s, unsigned long ip, unsigned long sym_flags)
424 goto out; 405 goto out;
425 } 406 }
426 407
427 if (sym_flags & TRACE_ITER_SYM_OFFSET) 408 seq_print_sym(s, ip, sym_flags & TRACE_ITER_SYM_OFFSET);
428 seq_print_sym_offset(s, "%s", ip);
429 else
430 seq_print_sym_short(s, "%s", ip);
431 409
432 if (sym_flags & TRACE_ITER_SYM_ADDR) 410 if (sym_flags & TRACE_ITER_SYM_ADDR)
433 trace_seq_printf(s, " <" IP_FMT ">", ip); 411 trace_seq_printf(s, " <" IP_FMT ">", ip);
diff --git a/kernel/trace/trace_probe.c b/kernel/trace/trace_probe.c
index bd30e9398d2a..9962cb5da8ac 100644
--- a/kernel/trace/trace_probe.c
+++ b/kernel/trace/trace_probe.c
@@ -154,24 +154,52 @@ int traceprobe_split_symbol_offset(char *symbol, long *offset)
154 return 0; 154 return 0;
155} 155}
156 156
157/* @buf must has MAX_EVENT_NAME_LEN size */
158int traceprobe_parse_event_name(const char **pevent, const char **pgroup,
159 char *buf)
160{
161 const char *slash, *event = *pevent;
162
163 slash = strchr(event, '/');
164 if (slash) {
165 if (slash == event) {
166 pr_info("Group name is not specified\n");
167 return -EINVAL;
168 }
169 if (slash - event + 1 > MAX_EVENT_NAME_LEN) {
170 pr_info("Group name is too long\n");
171 return -E2BIG;
172 }
173 strlcpy(buf, event, slash - event + 1);
174 *pgroup = buf;
175 *pevent = slash + 1;
176 }
177 if (strlen(event) == 0) {
178 pr_info("Event name is not specified\n");
179 return -EINVAL;
180 }
181 return 0;
182}
183
157#define PARAM_MAX_STACK (THREAD_SIZE / sizeof(unsigned long)) 184#define PARAM_MAX_STACK (THREAD_SIZE / sizeof(unsigned long))
158 185
159static int parse_probe_vars(char *arg, const struct fetch_type *t, 186static int parse_probe_vars(char *arg, const struct fetch_type *t,
160 struct fetch_insn *code, unsigned int flags) 187 struct fetch_insn *code, unsigned int flags)
161{ 188{
162 int ret = 0;
163 unsigned long param; 189 unsigned long param;
190 int ret = 0;
191 int len;
164 192
165 if (strcmp(arg, "retval") == 0) { 193 if (strcmp(arg, "retval") == 0) {
166 if (flags & TPARG_FL_RETURN) 194 if (flags & TPARG_FL_RETURN)
167 code->op = FETCH_OP_RETVAL; 195 code->op = FETCH_OP_RETVAL;
168 else 196 else
169 ret = -EINVAL; 197 ret = -EINVAL;
170 } else if (strncmp(arg, "stack", 5) == 0) { 198 } else if ((len = str_has_prefix(arg, "stack"))) {
171 if (arg[5] == '\0') { 199 if (arg[len] == '\0') {
172 code->op = FETCH_OP_STACKP; 200 code->op = FETCH_OP_STACKP;
173 } else if (isdigit(arg[5])) { 201 } else if (isdigit(arg[len])) {
174 ret = kstrtoul(arg + 5, 10, &param); 202 ret = kstrtoul(arg + len, 10, &param);
175 if (ret || ((flags & TPARG_FL_KERNEL) && 203 if (ret || ((flags & TPARG_FL_KERNEL) &&
176 param > PARAM_MAX_STACK)) 204 param > PARAM_MAX_STACK))
177 ret = -EINVAL; 205 ret = -EINVAL;
@@ -186,10 +214,10 @@ static int parse_probe_vars(char *arg, const struct fetch_type *t,
186#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API 214#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
187 } else if (((flags & TPARG_FL_MASK) == 215 } else if (((flags & TPARG_FL_MASK) ==
188 (TPARG_FL_KERNEL | TPARG_FL_FENTRY)) && 216 (TPARG_FL_KERNEL | TPARG_FL_FENTRY)) &&
189 strncmp(arg, "arg", 3) == 0) { 217 (len = str_has_prefix(arg, "arg"))) {
190 if (!isdigit(arg[3])) 218 if (!isdigit(arg[len]))
191 return -EINVAL; 219 return -EINVAL;
192 ret = kstrtoul(arg + 3, 10, &param); 220 ret = kstrtoul(arg + len, 10, &param);
193 if (ret || !param || param > PARAM_MAX_STACK) 221 if (ret || !param || param > PARAM_MAX_STACK)
194 return -EINVAL; 222 return -EINVAL;
195 code->op = FETCH_OP_ARG; 223 code->op = FETCH_OP_ARG;
@@ -348,7 +376,7 @@ static int __parse_bitfield_probe_arg(const char *bf,
348} 376}
349 377
350/* String length checking wrapper */ 378/* String length checking wrapper */
351int traceprobe_parse_probe_arg(char *arg, ssize_t *size, 379static int traceprobe_parse_probe_arg_body(char *arg, ssize_t *size,
352 struct probe_arg *parg, unsigned int flags) 380 struct probe_arg *parg, unsigned int flags)
353{ 381{
354 struct fetch_insn *code, *scode, *tmp = NULL; 382 struct fetch_insn *code, *scode, *tmp = NULL;
@@ -491,8 +519,8 @@ fail:
491} 519}
492 520
493/* Return 1 if name is reserved or already used by another argument */ 521/* Return 1 if name is reserved or already used by another argument */
494int traceprobe_conflict_field_name(const char *name, 522static int traceprobe_conflict_field_name(const char *name,
495 struct probe_arg *args, int narg) 523 struct probe_arg *args, int narg)
496{ 524{
497 int i; 525 int i;
498 526
@@ -507,6 +535,47 @@ int traceprobe_conflict_field_name(const char *name,
507 return 0; 535 return 0;
508} 536}
509 537
538int traceprobe_parse_probe_arg(struct trace_probe *tp, int i, char *arg,
539 unsigned int flags)
540{
541 struct probe_arg *parg = &tp->args[i];
542 char *body;
543 int ret;
544
545 /* Increment count for freeing args in error case */
546 tp->nr_args++;
547
548 body = strchr(arg, '=');
549 if (body) {
550 parg->name = kmemdup_nul(arg, body - arg, GFP_KERNEL);
551 body++;
552 } else {
553 /* If argument name is omitted, set "argN" */
554 parg->name = kasprintf(GFP_KERNEL, "arg%d", i + 1);
555 body = arg;
556 }
557 if (!parg->name)
558 return -ENOMEM;
559
560 if (!is_good_name(parg->name)) {
561 pr_info("Invalid argument[%d] name: %s\n",
562 i, parg->name);
563 return -EINVAL;
564 }
565
566 if (traceprobe_conflict_field_name(parg->name, tp->args, i)) {
567 pr_info("Argument[%d]: '%s' conflicts with another field.\n",
568 i, parg->name);
569 return -EINVAL;
570 }
571
572 /* Parse fetch argument */
573 ret = traceprobe_parse_probe_arg_body(body, &tp->size, parg, flags);
574 if (ret)
575 pr_info("Parse error at argument[%d]. (%d)\n", i, ret);
576 return ret;
577}
578
510void traceprobe_free_probe_arg(struct probe_arg *arg) 579void traceprobe_free_probe_arg(struct probe_arg *arg)
511{ 580{
512 struct fetch_insn *code = arg->code; 581 struct fetch_insn *code = arg->code;
diff --git a/kernel/trace/trace_probe.h b/kernel/trace/trace_probe.h
index 974afc1a3e73..8a63f8bc01bc 100644
--- a/kernel/trace/trace_probe.h
+++ b/kernel/trace/trace_probe.h
@@ -272,16 +272,15 @@ find_event_file_link(struct trace_probe *tp, struct trace_event_file *file)
272#define TPARG_FL_FENTRY BIT(2) 272#define TPARG_FL_FENTRY BIT(2)
273#define TPARG_FL_MASK GENMASK(2, 0) 273#define TPARG_FL_MASK GENMASK(2, 0)
274 274
275extern int traceprobe_parse_probe_arg(char *arg, ssize_t *size, 275extern int traceprobe_parse_probe_arg(struct trace_probe *tp, int i,
276 struct probe_arg *parg, unsigned int flags); 276 char *arg, unsigned int flags);
277
278extern int traceprobe_conflict_field_name(const char *name,
279 struct probe_arg *args, int narg);
280 277
281extern int traceprobe_update_arg(struct probe_arg *arg); 278extern int traceprobe_update_arg(struct probe_arg *arg);
282extern void traceprobe_free_probe_arg(struct probe_arg *arg); 279extern void traceprobe_free_probe_arg(struct probe_arg *arg);
283 280
284extern int traceprobe_split_symbol_offset(char *symbol, long *offset); 281extern int traceprobe_split_symbol_offset(char *symbol, long *offset);
282extern int traceprobe_parse_event_name(const char **pevent,
283 const char **pgroup, char *buf);
285 284
286extern int traceprobe_set_print_fmt(struct trace_probe *tp, bool is_return); 285extern int traceprobe_set_print_fmt(struct trace_probe *tp, bool is_return);
287 286
diff --git a/kernel/trace/trace_sched_wakeup.c b/kernel/trace/trace_sched_wakeup.c
index 7d04b9890755..4ea7e6845efb 100644
--- a/kernel/trace/trace_sched_wakeup.c
+++ b/kernel/trace/trace_sched_wakeup.c
@@ -35,26 +35,19 @@ static arch_spinlock_t wakeup_lock =
35 35
36static void wakeup_reset(struct trace_array *tr); 36static void wakeup_reset(struct trace_array *tr);
37static void __wakeup_reset(struct trace_array *tr); 37static void __wakeup_reset(struct trace_array *tr);
38static int start_func_tracer(struct trace_array *tr, int graph);
39static void stop_func_tracer(struct trace_array *tr, int graph);
38 40
39static int save_flags; 41static int save_flags;
40 42
41#ifdef CONFIG_FUNCTION_GRAPH_TRACER 43#ifdef CONFIG_FUNCTION_GRAPH_TRACER
42static int wakeup_display_graph(struct trace_array *tr, int set);
43# define is_graph(tr) ((tr)->trace_flags & TRACE_ITER_DISPLAY_GRAPH) 44# define is_graph(tr) ((tr)->trace_flags & TRACE_ITER_DISPLAY_GRAPH)
44#else 45#else
45static inline int wakeup_display_graph(struct trace_array *tr, int set)
46{
47 return 0;
48}
49# define is_graph(tr) false 46# define is_graph(tr) false
50#endif 47#endif
51 48
52
53#ifdef CONFIG_FUNCTION_TRACER 49#ifdef CONFIG_FUNCTION_TRACER
54 50
55static int wakeup_graph_entry(struct ftrace_graph_ent *trace);
56static void wakeup_graph_return(struct ftrace_graph_ret *trace);
57
58static bool function_enabled; 51static bool function_enabled;
59 52
60/* 53/*
@@ -104,122 +97,8 @@ out_enable:
104 return 0; 97 return 0;
105} 98}
106 99
107/*
108 * wakeup uses its own tracer function to keep the overhead down:
109 */
110static void
111wakeup_tracer_call(unsigned long ip, unsigned long parent_ip,
112 struct ftrace_ops *op, struct pt_regs *pt_regs)
113{
114 struct trace_array *tr = wakeup_trace;
115 struct trace_array_cpu *data;
116 unsigned long flags;
117 int pc;
118
119 if (!func_prolog_preempt_disable(tr, &data, &pc))
120 return;
121
122 local_irq_save(flags);
123 trace_function(tr, ip, parent_ip, flags, pc);
124 local_irq_restore(flags);
125
126 atomic_dec(&data->disabled);
127 preempt_enable_notrace();
128}
129
130static int register_wakeup_function(struct trace_array *tr, int graph, int set)
131{
132 int ret;
133
134 /* 'set' is set if TRACE_ITER_FUNCTION is about to be set */
135 if (function_enabled || (!set && !(tr->trace_flags & TRACE_ITER_FUNCTION)))
136 return 0;
137
138 if (graph)
139 ret = register_ftrace_graph(&wakeup_graph_return,
140 &wakeup_graph_entry);
141 else
142 ret = register_ftrace_function(tr->ops);
143
144 if (!ret)
145 function_enabled = true;
146
147 return ret;
148}
149
150static void unregister_wakeup_function(struct trace_array *tr, int graph)
151{
152 if (!function_enabled)
153 return;
154
155 if (graph)
156 unregister_ftrace_graph();
157 else
158 unregister_ftrace_function(tr->ops);
159
160 function_enabled = false;
161}
162
163static int wakeup_function_set(struct trace_array *tr, u32 mask, int set)
164{
165 if (!(mask & TRACE_ITER_FUNCTION))
166 return 0;
167
168 if (set)
169 register_wakeup_function(tr, is_graph(tr), 1);
170 else
171 unregister_wakeup_function(tr, is_graph(tr));
172 return 1;
173}
174#else
175static int register_wakeup_function(struct trace_array *tr, int graph, int set)
176{
177 return 0;
178}
179static void unregister_wakeup_function(struct trace_array *tr, int graph) { }
180static int wakeup_function_set(struct trace_array *tr, u32 mask, int set)
181{
182 return 0;
183}
184#endif /* CONFIG_FUNCTION_TRACER */
185
186static int wakeup_flag_changed(struct trace_array *tr, u32 mask, int set)
187{
188 struct tracer *tracer = tr->current_trace;
189
190 if (wakeup_function_set(tr, mask, set))
191 return 0;
192
193#ifdef CONFIG_FUNCTION_GRAPH_TRACER 100#ifdef CONFIG_FUNCTION_GRAPH_TRACER
194 if (mask & TRACE_ITER_DISPLAY_GRAPH)
195 return wakeup_display_graph(tr, set);
196#endif
197 101
198 return trace_keep_overwrite(tracer, mask, set);
199}
200
201static int start_func_tracer(struct trace_array *tr, int graph)
202{
203 int ret;
204
205 ret = register_wakeup_function(tr, graph, 0);
206
207 if (!ret && tracing_is_enabled())
208 tracer_enabled = 1;
209 else
210 tracer_enabled = 0;
211
212 return ret;
213}
214
215static void stop_func_tracer(struct trace_array *tr, int graph)
216{
217 tracer_enabled = 0;
218
219 unregister_wakeup_function(tr, graph);
220}
221
222#ifdef CONFIG_FUNCTION_GRAPH_TRACER
223static int wakeup_display_graph(struct trace_array *tr, int set) 102static int wakeup_display_graph(struct trace_array *tr, int set)
224{ 103{
225 if (!(is_graph(tr) ^ set)) 104 if (!(is_graph(tr) ^ set))
@@ -283,6 +162,11 @@ static void wakeup_graph_return(struct ftrace_graph_ret *trace)
283 return; 162 return;
284} 163}
285 164
165static struct fgraph_ops fgraph_wakeup_ops = {
166 .entryfunc = &wakeup_graph_entry,
167 .retfunc = &wakeup_graph_return,
168};
169
286static void wakeup_trace_open(struct trace_iterator *iter) 170static void wakeup_trace_open(struct trace_iterator *iter)
287{ 171{
288 if (is_graph(iter->tr)) 172 if (is_graph(iter->tr))
@@ -318,20 +202,87 @@ static void wakeup_print_header(struct seq_file *s)
318 else 202 else
319 trace_default_header(s); 203 trace_default_header(s);
320} 204}
205#endif /* else CONFIG_FUNCTION_GRAPH_TRACER */
321 206
207/*
208 * wakeup uses its own tracer function to keep the overhead down:
209 */
322static void 210static void
323__trace_function(struct trace_array *tr, 211wakeup_tracer_call(unsigned long ip, unsigned long parent_ip,
324 unsigned long ip, unsigned long parent_ip, 212 struct ftrace_ops *op, struct pt_regs *pt_regs)
325 unsigned long flags, int pc)
326{ 213{
327 if (is_graph(tr)) 214 struct trace_array *tr = wakeup_trace;
328 trace_graph_function(tr, ip, parent_ip, flags, pc); 215 struct trace_array_cpu *data;
216 unsigned long flags;
217 int pc;
218
219 if (!func_prolog_preempt_disable(tr, &data, &pc))
220 return;
221
222 local_irq_save(flags);
223 trace_function(tr, ip, parent_ip, flags, pc);
224 local_irq_restore(flags);
225
226 atomic_dec(&data->disabled);
227 preempt_enable_notrace();
228}
229
230static int register_wakeup_function(struct trace_array *tr, int graph, int set)
231{
232 int ret;
233
234 /* 'set' is set if TRACE_ITER_FUNCTION is about to be set */
235 if (function_enabled || (!set && !(tr->trace_flags & TRACE_ITER_FUNCTION)))
236 return 0;
237
238 if (graph)
239 ret = register_ftrace_graph(&fgraph_wakeup_ops);
329 else 240 else
330 trace_function(tr, ip, parent_ip, flags, pc); 241 ret = register_ftrace_function(tr->ops);
242
243 if (!ret)
244 function_enabled = true;
245
246 return ret;
331} 247}
332#else
333#define __trace_function trace_function
334 248
249static void unregister_wakeup_function(struct trace_array *tr, int graph)
250{
251 if (!function_enabled)
252 return;
253
254 if (graph)
255 unregister_ftrace_graph(&fgraph_wakeup_ops);
256 else
257 unregister_ftrace_function(tr->ops);
258
259 function_enabled = false;
260}
261
262static int wakeup_function_set(struct trace_array *tr, u32 mask, int set)
263{
264 if (!(mask & TRACE_ITER_FUNCTION))
265 return 0;
266
267 if (set)
268 register_wakeup_function(tr, is_graph(tr), 1);
269 else
270 unregister_wakeup_function(tr, is_graph(tr));
271 return 1;
272}
273#else /* CONFIG_FUNCTION_TRACER */
274static int register_wakeup_function(struct trace_array *tr, int graph, int set)
275{
276 return 0;
277}
278static void unregister_wakeup_function(struct trace_array *tr, int graph) { }
279static int wakeup_function_set(struct trace_array *tr, u32 mask, int set)
280{
281 return 0;
282}
283#endif /* else CONFIG_FUNCTION_TRACER */
284
285#ifndef CONFIG_FUNCTION_GRAPH_TRACER
335static enum print_line_t wakeup_print_line(struct trace_iterator *iter) 286static enum print_line_t wakeup_print_line(struct trace_iterator *iter)
336{ 287{
337 return TRACE_TYPE_UNHANDLED; 288 return TRACE_TYPE_UNHANDLED;
@@ -340,23 +291,58 @@ static enum print_line_t wakeup_print_line(struct trace_iterator *iter)
340static void wakeup_trace_open(struct trace_iterator *iter) { } 291static void wakeup_trace_open(struct trace_iterator *iter) { }
341static void wakeup_trace_close(struct trace_iterator *iter) { } 292static void wakeup_trace_close(struct trace_iterator *iter) { }
342 293
343#ifdef CONFIG_FUNCTION_TRACER
344static int wakeup_graph_entry(struct ftrace_graph_ent *trace)
345{
346 return -1;
347}
348static void wakeup_graph_return(struct ftrace_graph_ret *trace) { }
349static void wakeup_print_header(struct seq_file *s) 294static void wakeup_print_header(struct seq_file *s)
350{ 295{
351 trace_default_header(s); 296 trace_default_header(s);
352} 297}
353#else 298#endif /* !CONFIG_FUNCTION_GRAPH_TRACER */
354static void wakeup_print_header(struct seq_file *s) 299
300static void
301__trace_function(struct trace_array *tr,
302 unsigned long ip, unsigned long parent_ip,
303 unsigned long flags, int pc)
355{ 304{
356 trace_latency_header(s); 305 if (is_graph(tr))
306 trace_graph_function(tr, ip, parent_ip, flags, pc);
307 else
308 trace_function(tr, ip, parent_ip, flags, pc);
309}
310
311static int wakeup_flag_changed(struct trace_array *tr, u32 mask, int set)
312{
313 struct tracer *tracer = tr->current_trace;
314
315 if (wakeup_function_set(tr, mask, set))
316 return 0;
317
318#ifdef CONFIG_FUNCTION_GRAPH_TRACER
319 if (mask & TRACE_ITER_DISPLAY_GRAPH)
320 return wakeup_display_graph(tr, set);
321#endif
322
323 return trace_keep_overwrite(tracer, mask, set);
324}
325
326static int start_func_tracer(struct trace_array *tr, int graph)
327{
328 int ret;
329
330 ret = register_wakeup_function(tr, graph, 0);
331
332 if (!ret && tracing_is_enabled())
333 tracer_enabled = 1;
334 else
335 tracer_enabled = 0;
336
337 return ret;
338}
339
340static void stop_func_tracer(struct trace_array *tr, int graph)
341{
342 tracer_enabled = 0;
343
344 unregister_wakeup_function(tr, graph);
357} 345}
358#endif /* CONFIG_FUNCTION_TRACER */
359#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
360 346
361/* 347/*
362 * Should this new latency be reported/recorded? 348 * Should this new latency be reported/recorded?
diff --git a/kernel/trace/trace_selftest.c b/kernel/trace/trace_selftest.c
index 11e9daa4a568..9d402e7fc949 100644
--- a/kernel/trace/trace_selftest.c
+++ b/kernel/trace/trace_selftest.c
@@ -741,6 +741,11 @@ static int trace_graph_entry_watchdog(struct ftrace_graph_ent *trace)
741 return trace_graph_entry(trace); 741 return trace_graph_entry(trace);
742} 742}
743 743
744static struct fgraph_ops fgraph_ops __initdata = {
745 .entryfunc = &trace_graph_entry_watchdog,
746 .retfunc = &trace_graph_return,
747};
748
744/* 749/*
745 * Pretty much the same than for the function tracer from which the selftest 750 * Pretty much the same than for the function tracer from which the selftest
746 * has been borrowed. 751 * has been borrowed.
@@ -765,8 +770,7 @@ trace_selftest_startup_function_graph(struct tracer *trace,
765 */ 770 */
766 tracing_reset_online_cpus(&tr->trace_buffer); 771 tracing_reset_online_cpus(&tr->trace_buffer);
767 set_graph_array(tr); 772 set_graph_array(tr);
768 ret = register_ftrace_graph(&trace_graph_return, 773 ret = register_ftrace_graph(&fgraph_ops);
769 &trace_graph_entry_watchdog);
770 if (ret) { 774 if (ret) {
771 warn_failed_init_tracer(trace, ret); 775 warn_failed_init_tracer(trace, ret);
772 goto out; 776 goto out;
diff --git a/kernel/trace/trace_stack.c b/kernel/trace/trace_stack.c
index 2b0d1ee3241c..eec648a0d673 100644
--- a/kernel/trace/trace_stack.c
+++ b/kernel/trace/trace_stack.c
@@ -286,7 +286,7 @@ __next(struct seq_file *m, loff_t *pos)
286{ 286{
287 long n = *pos - 1; 287 long n = *pos - 1;
288 288
289 if (n > stack_trace_max.nr_entries || stack_dump_trace[n] == ULONG_MAX) 289 if (n >= stack_trace_max.nr_entries || stack_dump_trace[n] == ULONG_MAX)
290 return NULL; 290 return NULL;
291 291
292 m->private = (void *)n; 292 m->private = (void *)n;
@@ -448,8 +448,10 @@ static char stack_trace_filter_buf[COMMAND_LINE_SIZE+1] __initdata;
448 448
449static __init int enable_stacktrace(char *str) 449static __init int enable_stacktrace(char *str)
450{ 450{
451 if (strncmp(str, "_filter=", 8) == 0) 451 int len;
452 strncpy(stack_trace_filter_buf, str+8, COMMAND_LINE_SIZE); 452
453 if ((len = str_has_prefix(str, "_filter=")))
454 strncpy(stack_trace_filter_buf, str + len, COMMAND_LINE_SIZE);
453 455
454 stack_tracer_enabled = 1; 456 stack_tracer_enabled = 1;
455 last_stack_tracer_enabled = 1; 457 last_stack_tracer_enabled = 1;
diff --git a/kernel/trace/trace_uprobe.c b/kernel/trace/trace_uprobe.c
index 31ea48eceda1..e335576b9411 100644
--- a/kernel/trace/trace_uprobe.c
+++ b/kernel/trace/trace_uprobe.c
@@ -7,6 +7,7 @@
7 */ 7 */
8#define pr_fmt(fmt) "trace_kprobe: " fmt 8#define pr_fmt(fmt) "trace_kprobe: " fmt
9 9
10#include <linux/ctype.h>
10#include <linux/module.h> 11#include <linux/module.h>
11#include <linux/uaccess.h> 12#include <linux/uaccess.h>
12#include <linux/uprobes.h> 13#include <linux/uprobes.h>
@@ -14,6 +15,7 @@
14#include <linux/string.h> 15#include <linux/string.h>
15#include <linux/rculist.h> 16#include <linux/rculist.h>
16 17
18#include "trace_dynevent.h"
17#include "trace_probe.h" 19#include "trace_probe.h"
18#include "trace_probe_tmpl.h" 20#include "trace_probe_tmpl.h"
19 21
@@ -37,11 +39,26 @@ struct trace_uprobe_filter {
37 struct list_head perf_events; 39 struct list_head perf_events;
38}; 40};
39 41
42static int trace_uprobe_create(int argc, const char **argv);
43static int trace_uprobe_show(struct seq_file *m, struct dyn_event *ev);
44static int trace_uprobe_release(struct dyn_event *ev);
45static bool trace_uprobe_is_busy(struct dyn_event *ev);
46static bool trace_uprobe_match(const char *system, const char *event,
47 struct dyn_event *ev);
48
49static struct dyn_event_operations trace_uprobe_ops = {
50 .create = trace_uprobe_create,
51 .show = trace_uprobe_show,
52 .is_busy = trace_uprobe_is_busy,
53 .free = trace_uprobe_release,
54 .match = trace_uprobe_match,
55};
56
40/* 57/*
41 * uprobe event core functions 58 * uprobe event core functions
42 */ 59 */
43struct trace_uprobe { 60struct trace_uprobe {
44 struct list_head list; 61 struct dyn_event devent;
45 struct trace_uprobe_filter filter; 62 struct trace_uprobe_filter filter;
46 struct uprobe_consumer consumer; 63 struct uprobe_consumer consumer;
47 struct path path; 64 struct path path;
@@ -53,6 +70,25 @@ struct trace_uprobe {
53 struct trace_probe tp; 70 struct trace_probe tp;
54}; 71};
55 72
73static bool is_trace_uprobe(struct dyn_event *ev)
74{
75 return ev->ops == &trace_uprobe_ops;
76}
77
78static struct trace_uprobe *to_trace_uprobe(struct dyn_event *ev)
79{
80 return container_of(ev, struct trace_uprobe, devent);
81}
82
83/**
84 * for_each_trace_uprobe - iterate over the trace_uprobe list
85 * @pos: the struct trace_uprobe * for each entry
86 * @dpos: the struct dyn_event * to use as a loop cursor
87 */
88#define for_each_trace_uprobe(pos, dpos) \
89 for_each_dyn_event(dpos) \
90 if (is_trace_uprobe(dpos) && (pos = to_trace_uprobe(dpos)))
91
56#define SIZEOF_TRACE_UPROBE(n) \ 92#define SIZEOF_TRACE_UPROBE(n) \
57 (offsetof(struct trace_uprobe, tp.args) + \ 93 (offsetof(struct trace_uprobe, tp.args) + \
58 (sizeof(struct probe_arg) * (n))) 94 (sizeof(struct probe_arg) * (n)))
@@ -60,9 +96,6 @@ struct trace_uprobe {
60static int register_uprobe_event(struct trace_uprobe *tu); 96static int register_uprobe_event(struct trace_uprobe *tu);
61static int unregister_uprobe_event(struct trace_uprobe *tu); 97static int unregister_uprobe_event(struct trace_uprobe *tu);
62 98
63static DEFINE_MUTEX(uprobe_lock);
64static LIST_HEAD(uprobe_list);
65
66struct uprobe_dispatch_data { 99struct uprobe_dispatch_data {
67 struct trace_uprobe *tu; 100 struct trace_uprobe *tu;
68 unsigned long bp_addr; 101 unsigned long bp_addr;
@@ -209,6 +242,22 @@ static inline bool is_ret_probe(struct trace_uprobe *tu)
209 return tu->consumer.ret_handler != NULL; 242 return tu->consumer.ret_handler != NULL;
210} 243}
211 244
245static bool trace_uprobe_is_busy(struct dyn_event *ev)
246{
247 struct trace_uprobe *tu = to_trace_uprobe(ev);
248
249 return trace_probe_is_enabled(&tu->tp);
250}
251
252static bool trace_uprobe_match(const char *system, const char *event,
253 struct dyn_event *ev)
254{
255 struct trace_uprobe *tu = to_trace_uprobe(ev);
256
257 return strcmp(trace_event_name(&tu->tp.call), event) == 0 &&
258 (!system || strcmp(tu->tp.call.class->system, system) == 0);
259}
260
212/* 261/*
213 * Allocate new trace_uprobe and initialize it (including uprobes). 262 * Allocate new trace_uprobe and initialize it (including uprobes).
214 */ 263 */
@@ -236,7 +285,7 @@ alloc_trace_uprobe(const char *group, const char *event, int nargs, bool is_ret)
236 if (!tu->tp.class.system) 285 if (!tu->tp.class.system)
237 goto error; 286 goto error;
238 287
239 INIT_LIST_HEAD(&tu->list); 288 dyn_event_init(&tu->devent, &trace_uprobe_ops);
240 INIT_LIST_HEAD(&tu->tp.files); 289 INIT_LIST_HEAD(&tu->tp.files);
241 tu->consumer.handler = uprobe_dispatcher; 290 tu->consumer.handler = uprobe_dispatcher;
242 if (is_ret) 291 if (is_ret)
@@ -255,6 +304,9 @@ static void free_trace_uprobe(struct trace_uprobe *tu)
255{ 304{
256 int i; 305 int i;
257 306
307 if (!tu)
308 return;
309
258 for (i = 0; i < tu->tp.nr_args; i++) 310 for (i = 0; i < tu->tp.nr_args; i++)
259 traceprobe_free_probe_arg(&tu->tp.args[i]); 311 traceprobe_free_probe_arg(&tu->tp.args[i]);
260 312
@@ -267,9 +319,10 @@ static void free_trace_uprobe(struct trace_uprobe *tu)
267 319
268static struct trace_uprobe *find_probe_event(const char *event, const char *group) 320static struct trace_uprobe *find_probe_event(const char *event, const char *group)
269{ 321{
322 struct dyn_event *pos;
270 struct trace_uprobe *tu; 323 struct trace_uprobe *tu;
271 324
272 list_for_each_entry(tu, &uprobe_list, list) 325 for_each_trace_uprobe(tu, pos)
273 if (strcmp(trace_event_name(&tu->tp.call), event) == 0 && 326 if (strcmp(trace_event_name(&tu->tp.call), event) == 0 &&
274 strcmp(tu->tp.call.class->system, group) == 0) 327 strcmp(tu->tp.call.class->system, group) == 0)
275 return tu; 328 return tu;
@@ -277,7 +330,7 @@ static struct trace_uprobe *find_probe_event(const char *event, const char *grou
277 return NULL; 330 return NULL;
278} 331}
279 332
280/* Unregister a trace_uprobe and probe_event: call with locking uprobe_lock */ 333/* Unregister a trace_uprobe and probe_event */
281static int unregister_trace_uprobe(struct trace_uprobe *tu) 334static int unregister_trace_uprobe(struct trace_uprobe *tu)
282{ 335{
283 int ret; 336 int ret;
@@ -286,7 +339,7 @@ static int unregister_trace_uprobe(struct trace_uprobe *tu)
286 if (ret) 339 if (ret)
287 return ret; 340 return ret;
288 341
289 list_del(&tu->list); 342 dyn_event_remove(&tu->devent);
290 free_trace_uprobe(tu); 343 free_trace_uprobe(tu);
291 return 0; 344 return 0;
292} 345}
@@ -302,13 +355,14 @@ static int unregister_trace_uprobe(struct trace_uprobe *tu)
302 */ 355 */
303static struct trace_uprobe *find_old_trace_uprobe(struct trace_uprobe *new) 356static struct trace_uprobe *find_old_trace_uprobe(struct trace_uprobe *new)
304{ 357{
358 struct dyn_event *pos;
305 struct trace_uprobe *tmp, *old = NULL; 359 struct trace_uprobe *tmp, *old = NULL;
306 struct inode *new_inode = d_real_inode(new->path.dentry); 360 struct inode *new_inode = d_real_inode(new->path.dentry);
307 361
308 old = find_probe_event(trace_event_name(&new->tp.call), 362 old = find_probe_event(trace_event_name(&new->tp.call),
309 new->tp.call.class->system); 363 new->tp.call.class->system);
310 364
311 list_for_each_entry(tmp, &uprobe_list, list) { 365 for_each_trace_uprobe(tmp, pos) {
312 if ((old ? old != tmp : true) && 366 if ((old ? old != tmp : true) &&
313 new_inode == d_real_inode(tmp->path.dentry) && 367 new_inode == d_real_inode(tmp->path.dentry) &&
314 new->offset == tmp->offset && 368 new->offset == tmp->offset &&
@@ -326,7 +380,7 @@ static int register_trace_uprobe(struct trace_uprobe *tu)
326 struct trace_uprobe *old_tu; 380 struct trace_uprobe *old_tu;
327 int ret; 381 int ret;
328 382
329 mutex_lock(&uprobe_lock); 383 mutex_lock(&event_mutex);
330 384
331 /* register as an event */ 385 /* register as an event */
332 old_tu = find_old_trace_uprobe(tu); 386 old_tu = find_old_trace_uprobe(tu);
@@ -348,10 +402,10 @@ static int register_trace_uprobe(struct trace_uprobe *tu)
348 goto end; 402 goto end;
349 } 403 }
350 404
351 list_add_tail(&tu->list, &uprobe_list); 405 dyn_event_add(&tu->devent);
352 406
353end: 407end:
354 mutex_unlock(&uprobe_lock); 408 mutex_unlock(&event_mutex);
355 409
356 return ret; 410 return ret;
357} 411}
@@ -362,91 +416,49 @@ end:
362 * 416 *
363 * - Remove uprobe: -:[GRP/]EVENT 417 * - Remove uprobe: -:[GRP/]EVENT
364 */ 418 */
365static int create_trace_uprobe(int argc, char **argv) 419static int trace_uprobe_create(int argc, const char **argv)
366{ 420{
367 struct trace_uprobe *tu; 421 struct trace_uprobe *tu;
368 char *arg, *event, *group, *filename, *rctr, *rctr_end; 422 const char *event = NULL, *group = UPROBE_EVENT_SYSTEM;
423 char *arg, *filename, *rctr, *rctr_end, *tmp;
369 char buf[MAX_EVENT_NAME_LEN]; 424 char buf[MAX_EVENT_NAME_LEN];
370 struct path path; 425 struct path path;
371 unsigned long offset, ref_ctr_offset; 426 unsigned long offset, ref_ctr_offset;
372 bool is_delete, is_return; 427 bool is_return = false;
373 int i, ret; 428 int i, ret;
374 429
375 ret = 0; 430 ret = 0;
376 is_delete = false;
377 is_return = false;
378 event = NULL;
379 group = NULL;
380 ref_ctr_offset = 0; 431 ref_ctr_offset = 0;
381 432
382 /* argc must be >= 1 */ 433 /* argc must be >= 1 */
383 if (argv[0][0] == '-') 434 if (argv[0][0] == 'r')
384 is_delete = true;
385 else if (argv[0][0] == 'r')
386 is_return = true; 435 is_return = true;
387 else if (argv[0][0] != 'p') { 436 else if (argv[0][0] != 'p' || argc < 2)
388 pr_info("Probe definition must be started with 'p', 'r' or '-'.\n"); 437 return -ECANCELED;
389 return -EINVAL;
390 }
391 438
392 if (argv[0][1] == ':') { 439 if (argv[0][1] == ':')
393 event = &argv[0][2]; 440 event = &argv[0][2];
394 arg = strchr(event, '/');
395
396 if (arg) {
397 group = event;
398 event = arg + 1;
399 event[-1] = '\0';
400
401 if (strlen(group) == 0) {
402 pr_info("Group name is not specified\n");
403 return -EINVAL;
404 }
405 }
406 if (strlen(event) == 0) {
407 pr_info("Event name is not specified\n");
408 return -EINVAL;
409 }
410 }
411 if (!group)
412 group = UPROBE_EVENT_SYSTEM;
413 441
414 if (is_delete) { 442 if (!strchr(argv[1], '/'))
415 int ret; 443 return -ECANCELED;
416 444
417 if (!event) { 445 filename = kstrdup(argv[1], GFP_KERNEL);
418 pr_info("Delete command needs an event name.\n"); 446 if (!filename)
419 return -EINVAL; 447 return -ENOMEM;
420 }
421 mutex_lock(&uprobe_lock);
422 tu = find_probe_event(event, group);
423
424 if (!tu) {
425 mutex_unlock(&uprobe_lock);
426 pr_info("Event %s/%s doesn't exist.\n", group, event);
427 return -ENOENT;
428 }
429 /* delete an event */
430 ret = unregister_trace_uprobe(tu);
431 mutex_unlock(&uprobe_lock);
432 return ret;
433 }
434 448
435 if (argc < 2) {
436 pr_info("Probe point is not specified.\n");
437 return -EINVAL;
438 }
439 /* Find the last occurrence, in case the path contains ':' too. */ 449 /* Find the last occurrence, in case the path contains ':' too. */
440 arg = strrchr(argv[1], ':'); 450 arg = strrchr(filename, ':');
441 if (!arg) 451 if (!arg || !isdigit(arg[1])) {
442 return -EINVAL; 452 kfree(filename);
453 return -ECANCELED;
454 }
443 455
444 *arg++ = '\0'; 456 *arg++ = '\0';
445 filename = argv[1];
446 ret = kern_path(filename, LOOKUP_FOLLOW, &path); 457 ret = kern_path(filename, LOOKUP_FOLLOW, &path);
447 if (ret) 458 if (ret) {
459 kfree(filename);
448 return ret; 460 return ret;
449 461 }
450 if (!d_is_reg(path.dentry)) { 462 if (!d_is_reg(path.dentry)) {
451 ret = -EINVAL; 463 ret = -EINVAL;
452 goto fail_address_parse; 464 goto fail_address_parse;
@@ -480,7 +492,11 @@ static int create_trace_uprobe(int argc, char **argv)
480 argv += 2; 492 argv += 2;
481 493
482 /* setup a probe */ 494 /* setup a probe */
483 if (!event) { 495 if (event) {
496 ret = traceprobe_parse_event_name(&event, &group, buf);
497 if (ret)
498 goto fail_address_parse;
499 } else {
484 char *tail; 500 char *tail;
485 char *ptr; 501 char *ptr;
486 502
@@ -508,60 +524,21 @@ static int create_trace_uprobe(int argc, char **argv)
508 tu->offset = offset; 524 tu->offset = offset;
509 tu->ref_ctr_offset = ref_ctr_offset; 525 tu->ref_ctr_offset = ref_ctr_offset;
510 tu->path = path; 526 tu->path = path;
511 tu->filename = kstrdup(filename, GFP_KERNEL); 527 tu->filename = filename;
512
513 if (!tu->filename) {
514 pr_info("Failed to allocate filename.\n");
515 ret = -ENOMEM;
516 goto error;
517 }
518 528
519 /* parse arguments */ 529 /* parse arguments */
520 ret = 0;
521 for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) { 530 for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) {
522 struct probe_arg *parg = &tu->tp.args[i]; 531 tmp = kstrdup(argv[i], GFP_KERNEL);
523 532 if (!tmp) {
524 /* Increment count for freeing args in error case */
525 tu->tp.nr_args++;
526
527 /* Parse argument name */
528 arg = strchr(argv[i], '=');
529 if (arg) {
530 *arg++ = '\0';
531 parg->name = kstrdup(argv[i], GFP_KERNEL);
532 } else {
533 arg = argv[i];
534 /* If argument name is omitted, set "argN" */
535 snprintf(buf, MAX_EVENT_NAME_LEN, "arg%d", i + 1);
536 parg->name = kstrdup(buf, GFP_KERNEL);
537 }
538
539 if (!parg->name) {
540 pr_info("Failed to allocate argument[%d] name.\n", i);
541 ret = -ENOMEM; 533 ret = -ENOMEM;
542 goto error; 534 goto error;
543 } 535 }
544 536
545 if (!is_good_name(parg->name)) { 537 ret = traceprobe_parse_probe_arg(&tu->tp, i, tmp,
546 pr_info("Invalid argument[%d] name: %s\n", i, parg->name);
547 ret = -EINVAL;
548 goto error;
549 }
550
551 if (traceprobe_conflict_field_name(parg->name, tu->tp.args, i)) {
552 pr_info("Argument[%d] name '%s' conflicts with "
553 "another field.\n", i, argv[i]);
554 ret = -EINVAL;
555 goto error;
556 }
557
558 /* Parse fetch argument */
559 ret = traceprobe_parse_probe_arg(arg, &tu->tp.size, parg,
560 is_return ? TPARG_FL_RETURN : 0); 538 is_return ? TPARG_FL_RETURN : 0);
561 if (ret) { 539 kfree(tmp);
562 pr_info("Parse error at argument[%d]. (%d)\n", i, ret); 540 if (ret)
563 goto error; 541 goto error;
564 }
565 } 542 }
566 543
567 ret = register_trace_uprobe(tu); 544 ret = register_trace_uprobe(tu);
@@ -575,48 +552,35 @@ error:
575 552
576fail_address_parse: 553fail_address_parse:
577 path_put(&path); 554 path_put(&path);
555 kfree(filename);
578 556
579 pr_info("Failed to parse address or file.\n"); 557 pr_info("Failed to parse address or file.\n");
580 558
581 return ret; 559 return ret;
582} 560}
583 561
584static int cleanup_all_probes(void) 562static int create_or_delete_trace_uprobe(int argc, char **argv)
585{ 563{
586 struct trace_uprobe *tu; 564 int ret;
587 int ret = 0;
588 565
589 mutex_lock(&uprobe_lock); 566 if (argv[0][0] == '-')
590 while (!list_empty(&uprobe_list)) { 567 return dyn_event_release(argc, argv, &trace_uprobe_ops);
591 tu = list_entry(uprobe_list.next, struct trace_uprobe, list);
592 ret = unregister_trace_uprobe(tu);
593 if (ret)
594 break;
595 }
596 mutex_unlock(&uprobe_lock);
597 return ret;
598}
599 568
600/* Probes listing interfaces */ 569 ret = trace_uprobe_create(argc, (const char **)argv);
601static void *probes_seq_start(struct seq_file *m, loff_t *pos) 570 return ret == -ECANCELED ? -EINVAL : ret;
602{
603 mutex_lock(&uprobe_lock);
604 return seq_list_start(&uprobe_list, *pos);
605} 571}
606 572
607static void *probes_seq_next(struct seq_file *m, void *v, loff_t *pos) 573static int trace_uprobe_release(struct dyn_event *ev)
608{ 574{
609 return seq_list_next(v, &uprobe_list, pos); 575 struct trace_uprobe *tu = to_trace_uprobe(ev);
610}
611 576
612static void probes_seq_stop(struct seq_file *m, void *v) 577 return unregister_trace_uprobe(tu);
613{
614 mutex_unlock(&uprobe_lock);
615} 578}
616 579
617static int probes_seq_show(struct seq_file *m, void *v) 580/* Probes listing interfaces */
581static int trace_uprobe_show(struct seq_file *m, struct dyn_event *ev)
618{ 582{
619 struct trace_uprobe *tu = v; 583 struct trace_uprobe *tu = to_trace_uprobe(ev);
620 char c = is_ret_probe(tu) ? 'r' : 'p'; 584 char c = is_ret_probe(tu) ? 'r' : 'p';
621 int i; 585 int i;
622 586
@@ -634,11 +598,21 @@ static int probes_seq_show(struct seq_file *m, void *v)
634 return 0; 598 return 0;
635} 599}
636 600
601static int probes_seq_show(struct seq_file *m, void *v)
602{
603 struct dyn_event *ev = v;
604
605 if (!is_trace_uprobe(ev))
606 return 0;
607
608 return trace_uprobe_show(m, ev);
609}
610
637static const struct seq_operations probes_seq_op = { 611static const struct seq_operations probes_seq_op = {
638 .start = probes_seq_start, 612 .start = dyn_event_seq_start,
639 .next = probes_seq_next, 613 .next = dyn_event_seq_next,
640 .stop = probes_seq_stop, 614 .stop = dyn_event_seq_stop,
641 .show = probes_seq_show 615 .show = probes_seq_show
642}; 616};
643 617
644static int probes_open(struct inode *inode, struct file *file) 618static int probes_open(struct inode *inode, struct file *file)
@@ -646,7 +620,7 @@ static int probes_open(struct inode *inode, struct file *file)
646 int ret; 620 int ret;
647 621
648 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) { 622 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
649 ret = cleanup_all_probes(); 623 ret = dyn_events_release_all(&trace_uprobe_ops);
650 if (ret) 624 if (ret)
651 return ret; 625 return ret;
652 } 626 }
@@ -657,7 +631,8 @@ static int probes_open(struct inode *inode, struct file *file)
657static ssize_t probes_write(struct file *file, const char __user *buffer, 631static ssize_t probes_write(struct file *file, const char __user *buffer,
658 size_t count, loff_t *ppos) 632 size_t count, loff_t *ppos)
659{ 633{
660 return trace_parse_run_command(file, buffer, count, ppos, create_trace_uprobe); 634 return trace_parse_run_command(file, buffer, count, ppos,
635 create_or_delete_trace_uprobe);
661} 636}
662 637
663static const struct file_operations uprobe_events_ops = { 638static const struct file_operations uprobe_events_ops = {
@@ -672,17 +647,22 @@ static const struct file_operations uprobe_events_ops = {
672/* Probes profiling interfaces */ 647/* Probes profiling interfaces */
673static int probes_profile_seq_show(struct seq_file *m, void *v) 648static int probes_profile_seq_show(struct seq_file *m, void *v)
674{ 649{
675 struct trace_uprobe *tu = v; 650 struct dyn_event *ev = v;
651 struct trace_uprobe *tu;
676 652
653 if (!is_trace_uprobe(ev))
654 return 0;
655
656 tu = to_trace_uprobe(ev);
677 seq_printf(m, " %s %-44s %15lu\n", tu->filename, 657 seq_printf(m, " %s %-44s %15lu\n", tu->filename,
678 trace_event_name(&tu->tp.call), tu->nhit); 658 trace_event_name(&tu->tp.call), tu->nhit);
679 return 0; 659 return 0;
680} 660}
681 661
682static const struct seq_operations profile_seq_op = { 662static const struct seq_operations profile_seq_op = {
683 .start = probes_seq_start, 663 .start = dyn_event_seq_start,
684 .next = probes_seq_next, 664 .next = dyn_event_seq_next,
685 .stop = probes_seq_stop, 665 .stop = dyn_event_seq_stop,
686 .show = probes_profile_seq_show 666 .show = probes_profile_seq_show
687}; 667};
688 668
@@ -1384,7 +1364,7 @@ create_local_trace_uprobe(char *name, unsigned long offs,
1384 } 1364 }
1385 1365
1386 /* 1366 /*
1387 * local trace_kprobes are not added to probe_list, so they are never 1367 * local trace_kprobes are not added to dyn_event, so they are never
1388 * searched in find_trace_kprobe(). Therefore, there is no concern of 1368 * searched in find_trace_kprobe(). Therefore, there is no concern of
1389 * duplicated name "DUMMY_EVENT" here. 1369 * duplicated name "DUMMY_EVENT" here.
1390 */ 1370 */
@@ -1432,6 +1412,11 @@ void destroy_local_trace_uprobe(struct trace_event_call *event_call)
1432static __init int init_uprobe_trace(void) 1412static __init int init_uprobe_trace(void)
1433{ 1413{
1434 struct dentry *d_tracer; 1414 struct dentry *d_tracer;
1415 int ret;
1416
1417 ret = dyn_event_register(&trace_uprobe_ops);
1418 if (ret)
1419 return ret;
1435 1420
1436 d_tracer = tracing_init_dentry(); 1421 d_tracer = tracing_init_dentry();
1437 if (IS_ERR(d_tracer)) 1422 if (IS_ERR(d_tracer))
diff --git a/lib/seq_buf.c b/lib/seq_buf.c
index 11f2ae0f9099..bd807f545a9d 100644
--- a/lib/seq_buf.c
+++ b/lib/seq_buf.c
@@ -140,13 +140,17 @@ int seq_buf_bprintf(struct seq_buf *s, const char *fmt, const u32 *binary)
140 */ 140 */
141int seq_buf_puts(struct seq_buf *s, const char *str) 141int seq_buf_puts(struct seq_buf *s, const char *str)
142{ 142{
143 unsigned int len = strlen(str); 143 size_t len = strlen(str);
144 144
145 WARN_ON(s->size == 0); 145 WARN_ON(s->size == 0);
146 146
147 /* Add 1 to len for the trailing null byte which must be there */
148 len += 1;
149
147 if (seq_buf_can_fit(s, len)) { 150 if (seq_buf_can_fit(s, len)) {
148 memcpy(s->buffer + s->len, str, len); 151 memcpy(s->buffer + s->len, str, len);
149 s->len += len; 152 /* Don't count the trailing null byte against the capacity */
153 s->len += len - 1;
150 return 0; 154 return 0;
151 } 155 }
152 seq_buf_set_overflow(s); 156 seq_buf_set_overflow(s);
diff --git a/scripts/recordmcount.c b/scripts/recordmcount.c
index 895c40e8679f..a50a2aa963ad 100644
--- a/scripts/recordmcount.c
+++ b/scripts/recordmcount.c
@@ -397,7 +397,7 @@ static uint32_t (*w2)(uint16_t);
397static int 397static int
398is_mcounted_section_name(char const *const txtname) 398is_mcounted_section_name(char const *const txtname)
399{ 399{
400 return strcmp(".text", txtname) == 0 || 400 return strncmp(".text", txtname, 5) == 0 ||
401 strcmp(".init.text", txtname) == 0 || 401 strcmp(".init.text", txtname) == 0 ||
402 strcmp(".ref.text", txtname) == 0 || 402 strcmp(".ref.text", txtname) == 0 ||
403 strcmp(".sched.text", txtname) == 0 || 403 strcmp(".sched.text", txtname) == 0 ||
diff --git a/scripts/recordmcount.pl b/scripts/recordmcount.pl
index f599031260d5..68841d01162c 100755
--- a/scripts/recordmcount.pl
+++ b/scripts/recordmcount.pl
@@ -142,6 +142,11 @@ my %text_sections = (
142 ".text.unlikely" => 1, 142 ".text.unlikely" => 1,
143); 143);
144 144
145# Acceptable section-prefixes to record.
146my %text_section_prefixes = (
147 ".text." => 1,
148);
149
145# Note: we are nice to C-programmers here, thus we skip the '||='-idiom. 150# Note: we are nice to C-programmers here, thus we skip the '||='-idiom.
146$objdump = 'objdump' if (!$objdump); 151$objdump = 'objdump' if (!$objdump);
147$objcopy = 'objcopy' if (!$objcopy); 152$objcopy = 'objcopy' if (!$objcopy);
@@ -519,6 +524,14 @@ while (<IN>) {
519 524
520 # Only record text sections that we know are safe 525 # Only record text sections that we know are safe
521 $read_function = defined($text_sections{$1}); 526 $read_function = defined($text_sections{$1});
527 if (!$read_function) {
528 foreach my $prefix (keys %text_section_prefixes) {
529 if (substr($1, 0, length $prefix) eq $prefix) {
530 $read_function = 1;
531 last;
532 }
533 }
534 }
522 # print out any recorded offsets 535 # print out any recorded offsets
523 update_funcs(); 536 update_funcs();
524 537
diff --git a/tools/testing/selftests/ftrace/test.d/dynevent/add_remove_kprobe.tc b/tools/testing/selftests/ftrace/test.d/dynevent/add_remove_kprobe.tc
new file mode 100644
index 000000000000..c6d8387dbbb8
--- /dev/null
+++ b/tools/testing/selftests/ftrace/test.d/dynevent/add_remove_kprobe.tc
@@ -0,0 +1,30 @@
1#!/bin/sh
2# SPDX-License-Identifier: GPL-2.0
3# description: Generic dynamic event - add/remove kprobe events
4
5[ -f dynamic_events ] || exit_unsupported
6
7grep -q "place: \[<module>:\]<symbol>" README || exit_unsupported
8grep -q "place (kretprobe): \[<module>:\]<symbol>" README || exit_unsupported
9
10echo 0 > events/enable
11echo > dynamic_events
12
13PLACE=_do_fork
14
15echo "p:myevent1 $PLACE" >> dynamic_events
16echo "r:myevent2 $PLACE" >> dynamic_events
17
18grep -q myevent1 dynamic_events
19grep -q myevent2 dynamic_events
20test -d events/kprobes/myevent1
21test -d events/kprobes/myevent2
22
23echo "-:myevent2" >> dynamic_events
24
25grep -q myevent1 dynamic_events
26! grep -q myevent2 dynamic_events
27
28echo > dynamic_events
29
30clear_trace
diff --git a/tools/testing/selftests/ftrace/test.d/dynevent/add_remove_synth.tc b/tools/testing/selftests/ftrace/test.d/dynevent/add_remove_synth.tc
new file mode 100644
index 000000000000..62b77b5941d0
--- /dev/null
+++ b/tools/testing/selftests/ftrace/test.d/dynevent/add_remove_synth.tc
@@ -0,0 +1,27 @@
1#!/bin/sh
2# SPDX-License-Identifier: GPL-2.0
3# description: Generic dynamic event - add/remove synthetic events
4
5[ -f dynamic_events ] || exit_unsupported
6
7grep -q "s:\[synthetic/\]" README || exit_unsupported
8
9echo 0 > events/enable
10echo > dynamic_events
11
12echo "s:latency1 u64 lat; pid_t pid;" >> dynamic_events
13echo "s:latency2 u64 lat; pid_t pid;" >> dynamic_events
14
15grep -q latency1 dynamic_events
16grep -q latency2 dynamic_events
17test -d events/synthetic/latency1
18test -d events/synthetic/latency2
19
20echo "-:synthetic/latency2" >> dynamic_events
21
22grep -q latency1 dynamic_events
23! grep -q latency2 dynamic_events
24
25echo > dynamic_events
26
27clear_trace
diff --git a/tools/testing/selftests/ftrace/test.d/dynevent/clear_select_events.tc b/tools/testing/selftests/ftrace/test.d/dynevent/clear_select_events.tc
new file mode 100644
index 000000000000..e0842109cb57
--- /dev/null
+++ b/tools/testing/selftests/ftrace/test.d/dynevent/clear_select_events.tc
@@ -0,0 +1,50 @@
1#!/bin/sh
2# SPDX-License-Identifier: GPL-2.0
3# description: Generic dynamic event - selective clear (compatibility)
4
5[ -f dynamic_events ] || exit_unsupported
6
7grep -q "place: \[<module>:\]<symbol>" README || exit_unsupported
8grep -q "place (kretprobe): \[<module>:\]<symbol>" README || exit_unsupported
9
10grep -q "s:\[synthetic/\]" README || exit_unsupported
11
12[ -f synthetic_events ] || exit_unsupported
13[ -f kprobe_events ] || exit_unsupported
14
15echo 0 > events/enable
16echo > dynamic_events
17
18PLACE=_do_fork
19
20setup_events() {
21echo "p:myevent1 $PLACE" >> dynamic_events
22echo "s:latency1 u64 lat; pid_t pid;" >> dynamic_events
23echo "r:myevent2 $PLACE" >> dynamic_events
24echo "s:latency2 u64 lat; pid_t pid;" >> dynamic_events
25
26grep -q myevent1 dynamic_events
27grep -q myevent2 dynamic_events
28grep -q latency1 dynamic_events
29grep -q latency2 dynamic_events
30}
31
32setup_events
33echo > synthetic_events
34
35grep -q myevent1 dynamic_events
36grep -q myevent2 dynamic_events
37! grep -q latency1 dynamic_events
38! grep -q latency2 dynamic_events
39
40echo > dynamic_events
41
42setup_events
43echo > kprobe_events
44
45! grep -q myevent1 dynamic_events
46! grep -q myevent2 dynamic_events
47grep -q latency1 dynamic_events
48grep -q latency2 dynamic_events
49
50echo > dynamic_events
diff --git a/tools/testing/selftests/ftrace/test.d/dynevent/generic_clear_event.tc b/tools/testing/selftests/ftrace/test.d/dynevent/generic_clear_event.tc
new file mode 100644
index 000000000000..901922e97878
--- /dev/null
+++ b/tools/testing/selftests/ftrace/test.d/dynevent/generic_clear_event.tc
@@ -0,0 +1,49 @@
1#!/bin/sh
2# SPDX-License-Identifier: GPL-2.0
3# description: Generic dynamic event - generic clear event
4
5[ -f dynamic_events ] || exit_unsupported
6
7grep -q "place: \[<module>:\]<symbol>" README || exit_unsupported
8grep -q "place (kretprobe): \[<module>:\]<symbol>" README || exit_unsupported
9
10grep -q "s:\[synthetic/\]" README || exit_unsupported
11
12echo 0 > events/enable
13echo > dynamic_events
14
15PLACE=_do_fork
16
17setup_events() {
18echo "p:myevent1 $PLACE" >> dynamic_events
19echo "s:latency1 u64 lat; pid_t pid;" >> dynamic_events
20echo "r:myevent2 $PLACE" >> dynamic_events
21echo "s:latency2 u64 lat; pid_t pid;" >> dynamic_events
22
23grep -q myevent1 dynamic_events
24grep -q myevent2 dynamic_events
25grep -q latency1 dynamic_events
26grep -q latency2 dynamic_events
27}
28
29setup_events
30
31echo "!p:myevent1 $PLACE" >> dynamic_events
32! grep -q myevent1 dynamic_events
33grep -q myevent2 dynamic_events
34grep -q latency1 dynamic_events
35grep -q latency2 dynamic_events
36
37echo "!s:latency1 u64 lat; pid_t pid;" >> dynamic_events
38grep -q myevent2 dynamic_events
39! grep -q latency1 dynamic_events
40grep -q latency2 dynamic_events
41
42echo "!r:myevent2 $PLACE" >> dynamic_events
43! grep -q myevent2 dynamic_events
44grep -q latency2 dynamic_events
45
46echo "!s:latency2 u64 lat; pid_t pid;" >> dynamic_events
47! grep -q latency2 dynamic_events
48
49echo > dynamic_events