diff options
Diffstat (limited to 'kernel/trace')
-rw-r--r-- | kernel/trace/Makefile | 2 | ||||
-rw-r--r-- | kernel/trace/ftrace.c | 30 | ||||
-rw-r--r-- | kernel/trace/ring_buffer.c | 31 | ||||
-rw-r--r-- | kernel/trace/ring_buffer_benchmark.c | 1 | ||||
-rw-r--r-- | kernel/trace/trace.c | 55 | ||||
-rw-r--r-- | kernel/trace/trace.h | 5 | ||||
-rw-r--r-- | kernel/trace/trace_clock.c | 1 | ||||
-rw-r--r-- | kernel/trace/trace_event_perf.c (renamed from kernel/trace/trace_event_profile.c) | 54 | ||||
-rw-r--r-- | kernel/trace/trace_events.c | 2 | ||||
-rw-r--r-- | kernel/trace/trace_functions_graph.c | 31 | ||||
-rw-r--r-- | kernel/trace/trace_kprobe.c | 29 | ||||
-rw-r--r-- | kernel/trace/trace_syscalls.c | 72 |
12 files changed, 193 insertions, 120 deletions
diff --git a/kernel/trace/Makefile b/kernel/trace/Makefile index d00c6fe23f54..78edc6490038 100644 --- a/kernel/trace/Makefile +++ b/kernel/trace/Makefile | |||
@@ -52,7 +52,7 @@ obj-$(CONFIG_EVENT_TRACING) += trace_events.o | |||
52 | obj-$(CONFIG_EVENT_TRACING) += trace_export.o | 52 | obj-$(CONFIG_EVENT_TRACING) += trace_export.o |
53 | obj-$(CONFIG_FTRACE_SYSCALLS) += trace_syscalls.o | 53 | obj-$(CONFIG_FTRACE_SYSCALLS) += trace_syscalls.o |
54 | ifeq ($(CONFIG_PERF_EVENTS),y) | 54 | ifeq ($(CONFIG_PERF_EVENTS),y) |
55 | obj-$(CONFIG_EVENT_TRACING) += trace_event_profile.o | 55 | obj-$(CONFIG_EVENT_TRACING) += trace_event_perf.o |
56 | endif | 56 | endif |
57 | obj-$(CONFIG_EVENT_TRACING) += trace_events_filter.o | 57 | obj-$(CONFIG_EVENT_TRACING) += trace_events_filter.o |
58 | obj-$(CONFIG_KPROBE_EVENT) += trace_kprobe.o | 58 | obj-$(CONFIG_KPROBE_EVENT) += trace_kprobe.o |
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 83783579378f..d9062f5cc0c0 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c | |||
@@ -27,6 +27,7 @@ | |||
27 | #include <linux/ctype.h> | 27 | #include <linux/ctype.h> |
28 | #include <linux/list.h> | 28 | #include <linux/list.h> |
29 | #include <linux/hash.h> | 29 | #include <linux/hash.h> |
30 | #include <linux/rcupdate.h> | ||
30 | 31 | ||
31 | #include <trace/events/sched.h> | 32 | #include <trace/events/sched.h> |
32 | 33 | ||
@@ -84,22 +85,22 @@ ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub; | |||
84 | ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub; | 85 | ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub; |
85 | ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub; | 86 | ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub; |
86 | 87 | ||
87 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | 88 | /* |
88 | static int ftrace_set_func(unsigned long *array, int *idx, char *buffer); | 89 | * Traverse the ftrace_list, invoking all entries. The reason that we |
89 | #endif | 90 | * can use rcu_dereference_raw() is that elements removed from this list |
90 | 91 | * are simply leaked, so there is no need to interact with a grace-period | |
92 | * mechanism. The rcu_dereference_raw() calls are needed to handle | ||
93 | * concurrent insertions into the ftrace_list. | ||
94 | * | ||
95 | * Silly Alpha and silly pointer-speculation compiler optimizations! | ||
96 | */ | ||
91 | static void ftrace_list_func(unsigned long ip, unsigned long parent_ip) | 97 | static void ftrace_list_func(unsigned long ip, unsigned long parent_ip) |
92 | { | 98 | { |
93 | struct ftrace_ops *op = ftrace_list; | 99 | struct ftrace_ops *op = rcu_dereference_raw(ftrace_list); /*see above*/ |
94 | |||
95 | /* in case someone actually ports this to alpha! */ | ||
96 | read_barrier_depends(); | ||
97 | 100 | ||
98 | while (op != &ftrace_list_end) { | 101 | while (op != &ftrace_list_end) { |
99 | /* silly alpha */ | ||
100 | read_barrier_depends(); | ||
101 | op->func(ip, parent_ip); | 102 | op->func(ip, parent_ip); |
102 | op = op->next; | 103 | op = rcu_dereference_raw(op->next); /*see above*/ |
103 | }; | 104 | }; |
104 | } | 105 | } |
105 | 106 | ||
@@ -154,8 +155,7 @@ static int __register_ftrace_function(struct ftrace_ops *ops) | |||
154 | * the ops->next pointer is valid before another CPU sees | 155 | * the ops->next pointer is valid before another CPU sees |
155 | * the ops pointer included into the ftrace_list. | 156 | * the ops pointer included into the ftrace_list. |
156 | */ | 157 | */ |
157 | smp_wmb(); | 158 | rcu_assign_pointer(ftrace_list, ops); |
158 | ftrace_list = ops; | ||
159 | 159 | ||
160 | if (ftrace_enabled) { | 160 | if (ftrace_enabled) { |
161 | ftrace_func_t func; | 161 | ftrace_func_t func; |
@@ -2276,6 +2276,8 @@ __setup("ftrace_filter=", set_ftrace_filter); | |||
2276 | 2276 | ||
2277 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | 2277 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
2278 | static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata; | 2278 | static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata; |
2279 | static int ftrace_set_func(unsigned long *array, int *idx, char *buffer); | ||
2280 | |||
2279 | static int __init set_graph_function(char *str) | 2281 | static int __init set_graph_function(char *str) |
2280 | { | 2282 | { |
2281 | strlcpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE); | 2283 | strlcpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE); |
@@ -3351,6 +3353,7 @@ void ftrace_graph_init_task(struct task_struct *t) | |||
3351 | { | 3353 | { |
3352 | /* Make sure we do not use the parent ret_stack */ | 3354 | /* Make sure we do not use the parent ret_stack */ |
3353 | t->ret_stack = NULL; | 3355 | t->ret_stack = NULL; |
3356 | t->curr_ret_stack = -1; | ||
3354 | 3357 | ||
3355 | if (ftrace_graph_active) { | 3358 | if (ftrace_graph_active) { |
3356 | struct ftrace_ret_stack *ret_stack; | 3359 | struct ftrace_ret_stack *ret_stack; |
@@ -3360,7 +3363,6 @@ void ftrace_graph_init_task(struct task_struct *t) | |||
3360 | GFP_KERNEL); | 3363 | GFP_KERNEL); |
3361 | if (!ret_stack) | 3364 | if (!ret_stack) |
3362 | return; | 3365 | return; |
3363 | t->curr_ret_stack = -1; | ||
3364 | atomic_set(&t->tracing_graph_pause, 0); | 3366 | atomic_set(&t->tracing_graph_pause, 0); |
3365 | atomic_set(&t->trace_overrun, 0); | 3367 | atomic_set(&t->trace_overrun, 0); |
3366 | t->ftrace_timestamp = 0; | 3368 | t->ftrace_timestamp = 0; |
diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 8c1b2d290718..d1187ef20caf 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c | |||
@@ -20,6 +20,7 @@ | |||
20 | #include <linux/cpu.h> | 20 | #include <linux/cpu.h> |
21 | #include <linux/fs.h> | 21 | #include <linux/fs.h> |
22 | 22 | ||
23 | #include <asm/local.h> | ||
23 | #include "trace.h" | 24 | #include "trace.h" |
24 | 25 | ||
25 | /* | 26 | /* |
@@ -206,6 +207,14 @@ EXPORT_SYMBOL_GPL(tracing_is_on); | |||
206 | #define RB_MAX_SMALL_DATA (RB_ALIGNMENT * RINGBUF_TYPE_DATA_TYPE_LEN_MAX) | 207 | #define RB_MAX_SMALL_DATA (RB_ALIGNMENT * RINGBUF_TYPE_DATA_TYPE_LEN_MAX) |
207 | #define RB_EVNT_MIN_SIZE 8U /* two 32bit words */ | 208 | #define RB_EVNT_MIN_SIZE 8U /* two 32bit words */ |
208 | 209 | ||
210 | #if !defined(CONFIG_64BIT) || defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) | ||
211 | # define RB_FORCE_8BYTE_ALIGNMENT 0 | ||
212 | # define RB_ARCH_ALIGNMENT RB_ALIGNMENT | ||
213 | #else | ||
214 | # define RB_FORCE_8BYTE_ALIGNMENT 1 | ||
215 | # define RB_ARCH_ALIGNMENT 8U | ||
216 | #endif | ||
217 | |||
209 | /* define RINGBUF_TYPE_DATA for 'case RINGBUF_TYPE_DATA:' */ | 218 | /* define RINGBUF_TYPE_DATA for 'case RINGBUF_TYPE_DATA:' */ |
210 | #define RINGBUF_TYPE_DATA 0 ... RINGBUF_TYPE_DATA_TYPE_LEN_MAX | 219 | #define RINGBUF_TYPE_DATA 0 ... RINGBUF_TYPE_DATA_TYPE_LEN_MAX |
211 | 220 | ||
@@ -1546,7 +1555,7 @@ rb_update_event(struct ring_buffer_event *event, | |||
1546 | 1555 | ||
1547 | case 0: | 1556 | case 0: |
1548 | length -= RB_EVNT_HDR_SIZE; | 1557 | length -= RB_EVNT_HDR_SIZE; |
1549 | if (length > RB_MAX_SMALL_DATA) | 1558 | if (length > RB_MAX_SMALL_DATA || RB_FORCE_8BYTE_ALIGNMENT) |
1550 | event->array[0] = length; | 1559 | event->array[0] = length; |
1551 | else | 1560 | else |
1552 | event->type_len = DIV_ROUND_UP(length, RB_ALIGNMENT); | 1561 | event->type_len = DIV_ROUND_UP(length, RB_ALIGNMENT); |
@@ -1721,11 +1730,11 @@ static unsigned rb_calculate_event_length(unsigned length) | |||
1721 | if (!length) | 1730 | if (!length) |
1722 | length = 1; | 1731 | length = 1; |
1723 | 1732 | ||
1724 | if (length > RB_MAX_SMALL_DATA) | 1733 | if (length > RB_MAX_SMALL_DATA || RB_FORCE_8BYTE_ALIGNMENT) |
1725 | length += sizeof(event.array[0]); | 1734 | length += sizeof(event.array[0]); |
1726 | 1735 | ||
1727 | length += RB_EVNT_HDR_SIZE; | 1736 | length += RB_EVNT_HDR_SIZE; |
1728 | length = ALIGN(length, RB_ALIGNMENT); | 1737 | length = ALIGN(length, RB_ARCH_ALIGNMENT); |
1729 | 1738 | ||
1730 | return length; | 1739 | return length; |
1731 | } | 1740 | } |
@@ -2232,12 +2241,12 @@ ring_buffer_lock_reserve(struct ring_buffer *buffer, unsigned long length) | |||
2232 | if (ring_buffer_flags != RB_BUFFERS_ON) | 2241 | if (ring_buffer_flags != RB_BUFFERS_ON) |
2233 | return NULL; | 2242 | return NULL; |
2234 | 2243 | ||
2235 | if (atomic_read(&buffer->record_disabled)) | ||
2236 | return NULL; | ||
2237 | |||
2238 | /* If we are tracing schedule, we don't want to recurse */ | 2244 | /* If we are tracing schedule, we don't want to recurse */ |
2239 | resched = ftrace_preempt_disable(); | 2245 | resched = ftrace_preempt_disable(); |
2240 | 2246 | ||
2247 | if (atomic_read(&buffer->record_disabled)) | ||
2248 | goto out_nocheck; | ||
2249 | |||
2241 | if (trace_recursive_lock()) | 2250 | if (trace_recursive_lock()) |
2242 | goto out_nocheck; | 2251 | goto out_nocheck; |
2243 | 2252 | ||
@@ -2469,11 +2478,11 @@ int ring_buffer_write(struct ring_buffer *buffer, | |||
2469 | if (ring_buffer_flags != RB_BUFFERS_ON) | 2478 | if (ring_buffer_flags != RB_BUFFERS_ON) |
2470 | return -EBUSY; | 2479 | return -EBUSY; |
2471 | 2480 | ||
2472 | if (atomic_read(&buffer->record_disabled)) | ||
2473 | return -EBUSY; | ||
2474 | |||
2475 | resched = ftrace_preempt_disable(); | 2481 | resched = ftrace_preempt_disable(); |
2476 | 2482 | ||
2483 | if (atomic_read(&buffer->record_disabled)) | ||
2484 | goto out; | ||
2485 | |||
2477 | cpu = raw_smp_processor_id(); | 2486 | cpu = raw_smp_processor_id(); |
2478 | 2487 | ||
2479 | if (!cpumask_test_cpu(cpu, buffer->cpumask)) | 2488 | if (!cpumask_test_cpu(cpu, buffer->cpumask)) |
@@ -2541,7 +2550,7 @@ EXPORT_SYMBOL_GPL(ring_buffer_record_disable); | |||
2541 | * @buffer: The ring buffer to enable writes | 2550 | * @buffer: The ring buffer to enable writes |
2542 | * | 2551 | * |
2543 | * Note, multiple disables will need the same number of enables | 2552 | * Note, multiple disables will need the same number of enables |
2544 | * to truely enable the writing (much like preempt_disable). | 2553 | * to truly enable the writing (much like preempt_disable). |
2545 | */ | 2554 | */ |
2546 | void ring_buffer_record_enable(struct ring_buffer *buffer) | 2555 | void ring_buffer_record_enable(struct ring_buffer *buffer) |
2547 | { | 2556 | { |
@@ -2577,7 +2586,7 @@ EXPORT_SYMBOL_GPL(ring_buffer_record_disable_cpu); | |||
2577 | * @cpu: The CPU to enable. | 2586 | * @cpu: The CPU to enable. |
2578 | * | 2587 | * |
2579 | * Note, multiple disables will need the same number of enables | 2588 | * Note, multiple disables will need the same number of enables |
2580 | * to truely enable the writing (much like preempt_disable). | 2589 | * to truly enable the writing (much like preempt_disable). |
2581 | */ | 2590 | */ |
2582 | void ring_buffer_record_enable_cpu(struct ring_buffer *buffer, int cpu) | 2591 | void ring_buffer_record_enable_cpu(struct ring_buffer *buffer, int cpu) |
2583 | { | 2592 | { |
diff --git a/kernel/trace/ring_buffer_benchmark.c b/kernel/trace/ring_buffer_benchmark.c index b2477caf09c2..df74c7982255 100644 --- a/kernel/trace/ring_buffer_benchmark.c +++ b/kernel/trace/ring_buffer_benchmark.c | |||
@@ -8,6 +8,7 @@ | |||
8 | #include <linux/kthread.h> | 8 | #include <linux/kthread.h> |
9 | #include <linux/module.h> | 9 | #include <linux/module.h> |
10 | #include <linux/time.h> | 10 | #include <linux/time.h> |
11 | #include <asm/local.h> | ||
11 | 12 | ||
12 | struct rb_page { | 13 | struct rb_page { |
13 | u64 ts; | 14 | u64 ts; |
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 032c57ca6502..3ec2ee6f6560 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c | |||
@@ -92,12 +92,12 @@ DEFINE_PER_CPU(int, ftrace_cpu_disabled); | |||
92 | static inline void ftrace_disable_cpu(void) | 92 | static inline void ftrace_disable_cpu(void) |
93 | { | 93 | { |
94 | preempt_disable(); | 94 | preempt_disable(); |
95 | __this_cpu_inc(per_cpu_var(ftrace_cpu_disabled)); | 95 | __this_cpu_inc(ftrace_cpu_disabled); |
96 | } | 96 | } |
97 | 97 | ||
98 | static inline void ftrace_enable_cpu(void) | 98 | static inline void ftrace_enable_cpu(void) |
99 | { | 99 | { |
100 | __this_cpu_dec(per_cpu_var(ftrace_cpu_disabled)); | 100 | __this_cpu_dec(ftrace_cpu_disabled); |
101 | preempt_enable(); | 101 | preempt_enable(); |
102 | } | 102 | } |
103 | 103 | ||
@@ -374,6 +374,21 @@ static int __init set_buf_size(char *str) | |||
374 | } | 374 | } |
375 | __setup("trace_buf_size=", set_buf_size); | 375 | __setup("trace_buf_size=", set_buf_size); |
376 | 376 | ||
377 | static int __init set_tracing_thresh(char *str) | ||
378 | { | ||
379 | unsigned long threshhold; | ||
380 | int ret; | ||
381 | |||
382 | if (!str) | ||
383 | return 0; | ||
384 | ret = strict_strtoul(str, 0, &threshhold); | ||
385 | if (ret < 0) | ||
386 | return 0; | ||
387 | tracing_thresh = threshhold * 1000; | ||
388 | return 1; | ||
389 | } | ||
390 | __setup("tracing_thresh=", set_tracing_thresh); | ||
391 | |||
377 | unsigned long nsecs_to_usecs(unsigned long nsecs) | 392 | unsigned long nsecs_to_usecs(unsigned long nsecs) |
378 | { | 393 | { |
379 | return nsecs / 1000; | 394 | return nsecs / 1000; |
@@ -579,9 +594,10 @@ static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt) | |||
579 | static arch_spinlock_t ftrace_max_lock = | 594 | static arch_spinlock_t ftrace_max_lock = |
580 | (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; | 595 | (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; |
581 | 596 | ||
597 | unsigned long __read_mostly tracing_thresh; | ||
598 | |||
582 | #ifdef CONFIG_TRACER_MAX_TRACE | 599 | #ifdef CONFIG_TRACER_MAX_TRACE |
583 | unsigned long __read_mostly tracing_max_latency; | 600 | unsigned long __read_mostly tracing_max_latency; |
584 | unsigned long __read_mostly tracing_thresh; | ||
585 | 601 | ||
586 | /* | 602 | /* |
587 | * Copy the new maximum trace into the separate maximum-trace | 603 | * Copy the new maximum trace into the separate maximum-trace |
@@ -592,7 +608,7 @@ static void | |||
592 | __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu) | 608 | __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu) |
593 | { | 609 | { |
594 | struct trace_array_cpu *data = tr->data[cpu]; | 610 | struct trace_array_cpu *data = tr->data[cpu]; |
595 | struct trace_array_cpu *max_data = tr->data[cpu]; | 611 | struct trace_array_cpu *max_data; |
596 | 612 | ||
597 | max_tr.cpu = cpu; | 613 | max_tr.cpu = cpu; |
598 | max_tr.time_start = data->preempt_timestamp; | 614 | max_tr.time_start = data->preempt_timestamp; |
@@ -602,7 +618,7 @@ __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu) | |||
602 | max_data->critical_start = data->critical_start; | 618 | max_data->critical_start = data->critical_start; |
603 | max_data->critical_end = data->critical_end; | 619 | max_data->critical_end = data->critical_end; |
604 | 620 | ||
605 | memcpy(data->comm, tsk->comm, TASK_COMM_LEN); | 621 | memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN); |
606 | max_data->pid = tsk->pid; | 622 | max_data->pid = tsk->pid; |
607 | max_data->uid = task_uid(tsk); | 623 | max_data->uid = task_uid(tsk); |
608 | max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO; | 624 | max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO; |
@@ -824,10 +840,10 @@ out: | |||
824 | mutex_unlock(&trace_types_lock); | 840 | mutex_unlock(&trace_types_lock); |
825 | } | 841 | } |
826 | 842 | ||
827 | static void __tracing_reset(struct trace_array *tr, int cpu) | 843 | static void __tracing_reset(struct ring_buffer *buffer, int cpu) |
828 | { | 844 | { |
829 | ftrace_disable_cpu(); | 845 | ftrace_disable_cpu(); |
830 | ring_buffer_reset_cpu(tr->buffer, cpu); | 846 | ring_buffer_reset_cpu(buffer, cpu); |
831 | ftrace_enable_cpu(); | 847 | ftrace_enable_cpu(); |
832 | } | 848 | } |
833 | 849 | ||
@@ -839,7 +855,7 @@ void tracing_reset(struct trace_array *tr, int cpu) | |||
839 | 855 | ||
840 | /* Make sure all commits have finished */ | 856 | /* Make sure all commits have finished */ |
841 | synchronize_sched(); | 857 | synchronize_sched(); |
842 | __tracing_reset(tr, cpu); | 858 | __tracing_reset(buffer, cpu); |
843 | 859 | ||
844 | ring_buffer_record_enable(buffer); | 860 | ring_buffer_record_enable(buffer); |
845 | } | 861 | } |
@@ -857,7 +873,7 @@ void tracing_reset_online_cpus(struct trace_array *tr) | |||
857 | tr->time_start = ftrace_now(tr->cpu); | 873 | tr->time_start = ftrace_now(tr->cpu); |
858 | 874 | ||
859 | for_each_online_cpu(cpu) | 875 | for_each_online_cpu(cpu) |
860 | __tracing_reset(tr, cpu); | 876 | __tracing_reset(buffer, cpu); |
861 | 877 | ||
862 | ring_buffer_record_enable(buffer); | 878 | ring_buffer_record_enable(buffer); |
863 | } | 879 | } |
@@ -934,6 +950,8 @@ void tracing_start(void) | |||
934 | goto out; | 950 | goto out; |
935 | } | 951 | } |
936 | 952 | ||
953 | /* Prevent the buffers from switching */ | ||
954 | arch_spin_lock(&ftrace_max_lock); | ||
937 | 955 | ||
938 | buffer = global_trace.buffer; | 956 | buffer = global_trace.buffer; |
939 | if (buffer) | 957 | if (buffer) |
@@ -943,6 +961,8 @@ void tracing_start(void) | |||
943 | if (buffer) | 961 | if (buffer) |
944 | ring_buffer_record_enable(buffer); | 962 | ring_buffer_record_enable(buffer); |
945 | 963 | ||
964 | arch_spin_unlock(&ftrace_max_lock); | ||
965 | |||
946 | ftrace_start(); | 966 | ftrace_start(); |
947 | out: | 967 | out: |
948 | spin_unlock_irqrestore(&tracing_start_lock, flags); | 968 | spin_unlock_irqrestore(&tracing_start_lock, flags); |
@@ -964,6 +984,9 @@ void tracing_stop(void) | |||
964 | if (trace_stop_count++) | 984 | if (trace_stop_count++) |
965 | goto out; | 985 | goto out; |
966 | 986 | ||
987 | /* Prevent the buffers from switching */ | ||
988 | arch_spin_lock(&ftrace_max_lock); | ||
989 | |||
967 | buffer = global_trace.buffer; | 990 | buffer = global_trace.buffer; |
968 | if (buffer) | 991 | if (buffer) |
969 | ring_buffer_record_disable(buffer); | 992 | ring_buffer_record_disable(buffer); |
@@ -972,6 +995,8 @@ void tracing_stop(void) | |||
972 | if (buffer) | 995 | if (buffer) |
973 | ring_buffer_record_disable(buffer); | 996 | ring_buffer_record_disable(buffer); |
974 | 997 | ||
998 | arch_spin_unlock(&ftrace_max_lock); | ||
999 | |||
975 | out: | 1000 | out: |
976 | spin_unlock_irqrestore(&tracing_start_lock, flags); | 1001 | spin_unlock_irqrestore(&tracing_start_lock, flags); |
977 | } | 1002 | } |
@@ -1166,7 +1191,7 @@ trace_function(struct trace_array *tr, | |||
1166 | struct ftrace_entry *entry; | 1191 | struct ftrace_entry *entry; |
1167 | 1192 | ||
1168 | /* If we are reading the ring buffer, don't trace */ | 1193 | /* If we are reading the ring buffer, don't trace */ |
1169 | if (unlikely(__this_cpu_read(per_cpu_var(ftrace_cpu_disabled)))) | 1194 | if (unlikely(__this_cpu_read(ftrace_cpu_disabled))) |
1170 | return; | 1195 | return; |
1171 | 1196 | ||
1172 | event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry), | 1197 | event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry), |
@@ -1259,6 +1284,13 @@ ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc) | |||
1259 | if (!(trace_flags & TRACE_ITER_USERSTACKTRACE)) | 1284 | if (!(trace_flags & TRACE_ITER_USERSTACKTRACE)) |
1260 | return; | 1285 | return; |
1261 | 1286 | ||
1287 | /* | ||
1288 | * NMIs can not handle page faults, even with fix ups. | ||
1289 | * The save user stack can (and often does) fault. | ||
1290 | */ | ||
1291 | if (unlikely(in_nmi())) | ||
1292 | return; | ||
1293 | |||
1262 | event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK, | 1294 | event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK, |
1263 | sizeof(*entry), flags, pc); | 1295 | sizeof(*entry), flags, pc); |
1264 | if (!event) | 1296 | if (!event) |
@@ -1703,6 +1735,7 @@ static void *s_start(struct seq_file *m, loff_t *pos) | |||
1703 | 1735 | ||
1704 | ftrace_enable_cpu(); | 1736 | ftrace_enable_cpu(); |
1705 | 1737 | ||
1738 | iter->leftover = 0; | ||
1706 | for (p = iter; p && l < *pos; p = s_next(m, p, &l)) | 1739 | for (p = iter; p && l < *pos; p = s_next(m, p, &l)) |
1707 | ; | 1740 | ; |
1708 | 1741 | ||
@@ -4248,10 +4281,10 @@ static __init int tracer_init_debugfs(void) | |||
4248 | #ifdef CONFIG_TRACER_MAX_TRACE | 4281 | #ifdef CONFIG_TRACER_MAX_TRACE |
4249 | trace_create_file("tracing_max_latency", 0644, d_tracer, | 4282 | trace_create_file("tracing_max_latency", 0644, d_tracer, |
4250 | &tracing_max_latency, &tracing_max_lat_fops); | 4283 | &tracing_max_latency, &tracing_max_lat_fops); |
4284 | #endif | ||
4251 | 4285 | ||
4252 | trace_create_file("tracing_thresh", 0644, d_tracer, | 4286 | trace_create_file("tracing_thresh", 0644, d_tracer, |
4253 | &tracing_thresh, &tracing_max_lat_fops); | 4287 | &tracing_thresh, &tracing_max_lat_fops); |
4254 | #endif | ||
4255 | 4288 | ||
4256 | trace_create_file("README", 0444, d_tracer, | 4289 | trace_create_file("README", 0444, d_tracer, |
4257 | NULL, &tracing_readme_fops); | 4290 | NULL, &tracing_readme_fops); |
diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index fd05bcaf91b0..2825ef2c0b15 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h | |||
@@ -396,9 +396,10 @@ extern int process_new_ksym_entry(char *ksymname, int op, unsigned long addr); | |||
396 | 396 | ||
397 | extern unsigned long nsecs_to_usecs(unsigned long nsecs); | 397 | extern unsigned long nsecs_to_usecs(unsigned long nsecs); |
398 | 398 | ||
399 | extern unsigned long tracing_thresh; | ||
400 | |||
399 | #ifdef CONFIG_TRACER_MAX_TRACE | 401 | #ifdef CONFIG_TRACER_MAX_TRACE |
400 | extern unsigned long tracing_max_latency; | 402 | extern unsigned long tracing_max_latency; |
401 | extern unsigned long tracing_thresh; | ||
402 | 403 | ||
403 | void update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu); | 404 | void update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu); |
404 | void update_max_tr_single(struct trace_array *tr, | 405 | void update_max_tr_single(struct trace_array *tr, |
@@ -550,7 +551,7 @@ static inline int ftrace_trace_task(struct task_struct *task) | |||
550 | * struct trace_parser - servers for reading the user input separated by spaces | 551 | * struct trace_parser - servers for reading the user input separated by spaces |
551 | * @cont: set if the input is not complete - no final space char was found | 552 | * @cont: set if the input is not complete - no final space char was found |
552 | * @buffer: holds the parsed user input | 553 | * @buffer: holds the parsed user input |
553 | * @idx: user input lenght | 554 | * @idx: user input length |
554 | * @size: buffer size | 555 | * @size: buffer size |
555 | */ | 556 | */ |
556 | struct trace_parser { | 557 | struct trace_parser { |
diff --git a/kernel/trace/trace_clock.c b/kernel/trace/trace_clock.c index 84a3a7ba072a..6fbfb8f417b9 100644 --- a/kernel/trace/trace_clock.c +++ b/kernel/trace/trace_clock.c | |||
@@ -13,6 +13,7 @@ | |||
13 | * Tracer plugins will chose a default from these clocks. | 13 | * Tracer plugins will chose a default from these clocks. |
14 | */ | 14 | */ |
15 | #include <linux/spinlock.h> | 15 | #include <linux/spinlock.h> |
16 | #include <linux/irqflags.h> | ||
16 | #include <linux/hardirq.h> | 17 | #include <linux/hardirq.h> |
17 | #include <linux/module.h> | 18 | #include <linux/module.h> |
18 | #include <linux/percpu.h> | 19 | #include <linux/percpu.h> |
diff --git a/kernel/trace/trace_event_profile.c b/kernel/trace/trace_event_perf.c index f0d693005075..81f691eb3a30 100644 --- a/kernel/trace/trace_event_profile.c +++ b/kernel/trace/trace_event_perf.c | |||
@@ -1,32 +1,36 @@ | |||
1 | /* | 1 | /* |
2 | * trace event based perf counter profiling | 2 | * trace event based perf event profiling/tracing |
3 | * | 3 | * |
4 | * Copyright (C) 2009 Red Hat Inc, Peter Zijlstra <pzijlstr@redhat.com> | 4 | * Copyright (C) 2009 Red Hat Inc, Peter Zijlstra <pzijlstr@redhat.com> |
5 | * | 5 | * Copyright (C) 2009-2010 Frederic Weisbecker <fweisbec@gmail.com> |
6 | */ | 6 | */ |
7 | 7 | ||
8 | #include <linux/module.h> | 8 | #include <linux/module.h> |
9 | #include <linux/kprobes.h> | 9 | #include <linux/kprobes.h> |
10 | #include "trace.h" | 10 | #include "trace.h" |
11 | 11 | ||
12 | DEFINE_PER_CPU(struct pt_regs, perf_trace_regs); | ||
13 | EXPORT_PER_CPU_SYMBOL_GPL(perf_trace_regs); | ||
14 | |||
15 | EXPORT_SYMBOL_GPL(perf_arch_fetch_caller_regs); | ||
12 | 16 | ||
13 | static char *perf_trace_buf; | 17 | static char *perf_trace_buf; |
14 | static char *perf_trace_buf_nmi; | 18 | static char *perf_trace_buf_nmi; |
15 | 19 | ||
16 | typedef typeof(char [FTRACE_MAX_PROFILE_SIZE]) perf_trace_t ; | 20 | typedef typeof(char [PERF_MAX_TRACE_SIZE]) perf_trace_t ; |
17 | 21 | ||
18 | /* Count the events in use (per event id, not per instance) */ | 22 | /* Count the events in use (per event id, not per instance) */ |
19 | static int total_profile_count; | 23 | static int total_ref_count; |
20 | 24 | ||
21 | static int ftrace_profile_enable_event(struct ftrace_event_call *event) | 25 | static int perf_trace_event_enable(struct ftrace_event_call *event) |
22 | { | 26 | { |
23 | char *buf; | 27 | char *buf; |
24 | int ret = -ENOMEM; | 28 | int ret = -ENOMEM; |
25 | 29 | ||
26 | if (event->profile_count++ > 0) | 30 | if (event->perf_refcount++ > 0) |
27 | return 0; | 31 | return 0; |
28 | 32 | ||
29 | if (!total_profile_count) { | 33 | if (!total_ref_count) { |
30 | buf = (char *)alloc_percpu(perf_trace_t); | 34 | buf = (char *)alloc_percpu(perf_trace_t); |
31 | if (!buf) | 35 | if (!buf) |
32 | goto fail_buf; | 36 | goto fail_buf; |
@@ -40,35 +44,35 @@ static int ftrace_profile_enable_event(struct ftrace_event_call *event) | |||
40 | rcu_assign_pointer(perf_trace_buf_nmi, buf); | 44 | rcu_assign_pointer(perf_trace_buf_nmi, buf); |
41 | } | 45 | } |
42 | 46 | ||
43 | ret = event->profile_enable(event); | 47 | ret = event->perf_event_enable(event); |
44 | if (!ret) { | 48 | if (!ret) { |
45 | total_profile_count++; | 49 | total_ref_count++; |
46 | return 0; | 50 | return 0; |
47 | } | 51 | } |
48 | 52 | ||
49 | fail_buf_nmi: | 53 | fail_buf_nmi: |
50 | if (!total_profile_count) { | 54 | if (!total_ref_count) { |
51 | free_percpu(perf_trace_buf_nmi); | 55 | free_percpu(perf_trace_buf_nmi); |
52 | free_percpu(perf_trace_buf); | 56 | free_percpu(perf_trace_buf); |
53 | perf_trace_buf_nmi = NULL; | 57 | perf_trace_buf_nmi = NULL; |
54 | perf_trace_buf = NULL; | 58 | perf_trace_buf = NULL; |
55 | } | 59 | } |
56 | fail_buf: | 60 | fail_buf: |
57 | event->profile_count--; | 61 | event->perf_refcount--; |
58 | 62 | ||
59 | return ret; | 63 | return ret; |
60 | } | 64 | } |
61 | 65 | ||
62 | int ftrace_profile_enable(int event_id) | 66 | int perf_trace_enable(int event_id) |
63 | { | 67 | { |
64 | struct ftrace_event_call *event; | 68 | struct ftrace_event_call *event; |
65 | int ret = -EINVAL; | 69 | int ret = -EINVAL; |
66 | 70 | ||
67 | mutex_lock(&event_mutex); | 71 | mutex_lock(&event_mutex); |
68 | list_for_each_entry(event, &ftrace_events, list) { | 72 | list_for_each_entry(event, &ftrace_events, list) { |
69 | if (event->id == event_id && event->profile_enable && | 73 | if (event->id == event_id && event->perf_event_enable && |
70 | try_module_get(event->mod)) { | 74 | try_module_get(event->mod)) { |
71 | ret = ftrace_profile_enable_event(event); | 75 | ret = perf_trace_event_enable(event); |
72 | break; | 76 | break; |
73 | } | 77 | } |
74 | } | 78 | } |
@@ -77,16 +81,16 @@ int ftrace_profile_enable(int event_id) | |||
77 | return ret; | 81 | return ret; |
78 | } | 82 | } |
79 | 83 | ||
80 | static void ftrace_profile_disable_event(struct ftrace_event_call *event) | 84 | static void perf_trace_event_disable(struct ftrace_event_call *event) |
81 | { | 85 | { |
82 | char *buf, *nmi_buf; | 86 | char *buf, *nmi_buf; |
83 | 87 | ||
84 | if (--event->profile_count > 0) | 88 | if (--event->perf_refcount > 0) |
85 | return; | 89 | return; |
86 | 90 | ||
87 | event->profile_disable(event); | 91 | event->perf_event_disable(event); |
88 | 92 | ||
89 | if (!--total_profile_count) { | 93 | if (!--total_ref_count) { |
90 | buf = perf_trace_buf; | 94 | buf = perf_trace_buf; |
91 | rcu_assign_pointer(perf_trace_buf, NULL); | 95 | rcu_assign_pointer(perf_trace_buf, NULL); |
92 | 96 | ||
@@ -104,14 +108,14 @@ static void ftrace_profile_disable_event(struct ftrace_event_call *event) | |||
104 | } | 108 | } |
105 | } | 109 | } |
106 | 110 | ||
107 | void ftrace_profile_disable(int event_id) | 111 | void perf_trace_disable(int event_id) |
108 | { | 112 | { |
109 | struct ftrace_event_call *event; | 113 | struct ftrace_event_call *event; |
110 | 114 | ||
111 | mutex_lock(&event_mutex); | 115 | mutex_lock(&event_mutex); |
112 | list_for_each_entry(event, &ftrace_events, list) { | 116 | list_for_each_entry(event, &ftrace_events, list) { |
113 | if (event->id == event_id) { | 117 | if (event->id == event_id) { |
114 | ftrace_profile_disable_event(event); | 118 | perf_trace_event_disable(event); |
115 | module_put(event->mod); | 119 | module_put(event->mod); |
116 | break; | 120 | break; |
117 | } | 121 | } |
@@ -119,8 +123,8 @@ void ftrace_profile_disable(int event_id) | |||
119 | mutex_unlock(&event_mutex); | 123 | mutex_unlock(&event_mutex); |
120 | } | 124 | } |
121 | 125 | ||
122 | __kprobes void *ftrace_perf_buf_prepare(int size, unsigned short type, | 126 | __kprobes void *perf_trace_buf_prepare(int size, unsigned short type, |
123 | int *rctxp, unsigned long *irq_flags) | 127 | int *rctxp, unsigned long *irq_flags) |
124 | { | 128 | { |
125 | struct trace_entry *entry; | 129 | struct trace_entry *entry; |
126 | char *trace_buf, *raw_data; | 130 | char *trace_buf, *raw_data; |
@@ -138,9 +142,9 @@ __kprobes void *ftrace_perf_buf_prepare(int size, unsigned short type, | |||
138 | cpu = smp_processor_id(); | 142 | cpu = smp_processor_id(); |
139 | 143 | ||
140 | if (in_nmi()) | 144 | if (in_nmi()) |
141 | trace_buf = rcu_dereference(perf_trace_buf_nmi); | 145 | trace_buf = rcu_dereference_sched(perf_trace_buf_nmi); |
142 | else | 146 | else |
143 | trace_buf = rcu_dereference(perf_trace_buf); | 147 | trace_buf = rcu_dereference_sched(perf_trace_buf); |
144 | 148 | ||
145 | if (!trace_buf) | 149 | if (!trace_buf) |
146 | goto err; | 150 | goto err; |
@@ -161,4 +165,4 @@ err_recursion: | |||
161 | local_irq_restore(*irq_flags); | 165 | local_irq_restore(*irq_flags); |
162 | return NULL; | 166 | return NULL; |
163 | } | 167 | } |
164 | EXPORT_SYMBOL_GPL(ftrace_perf_buf_prepare); | 168 | EXPORT_SYMBOL_GPL(perf_trace_buf_prepare); |
diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c index 3f972ad98d04..beab8bf2f310 100644 --- a/kernel/trace/trace_events.c +++ b/kernel/trace/trace_events.c | |||
@@ -938,7 +938,7 @@ event_create_dir(struct ftrace_event_call *call, struct dentry *d_events, | |||
938 | trace_create_file("enable", 0644, call->dir, call, | 938 | trace_create_file("enable", 0644, call->dir, call, |
939 | enable); | 939 | enable); |
940 | 940 | ||
941 | if (call->id && call->profile_enable) | 941 | if (call->id && call->perf_event_enable) |
942 | trace_create_file("id", 0444, call->dir, call, | 942 | trace_create_file("id", 0444, call->dir, call, |
943 | id); | 943 | id); |
944 | 944 | ||
diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c index e998a824e9db..e6989d9b44da 100644 --- a/kernel/trace/trace_functions_graph.c +++ b/kernel/trace/trace_functions_graph.c | |||
@@ -188,7 +188,7 @@ static int __trace_graph_entry(struct trace_array *tr, | |||
188 | struct ring_buffer *buffer = tr->buffer; | 188 | struct ring_buffer *buffer = tr->buffer; |
189 | struct ftrace_graph_ent_entry *entry; | 189 | struct ftrace_graph_ent_entry *entry; |
190 | 190 | ||
191 | if (unlikely(__this_cpu_read(per_cpu_var(ftrace_cpu_disabled)))) | 191 | if (unlikely(__this_cpu_read(ftrace_cpu_disabled))) |
192 | return 0; | 192 | return 0; |
193 | 193 | ||
194 | event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_ENT, | 194 | event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_ENT, |
@@ -237,6 +237,14 @@ int trace_graph_entry(struct ftrace_graph_ent *trace) | |||
237 | return ret; | 237 | return ret; |
238 | } | 238 | } |
239 | 239 | ||
240 | int trace_graph_thresh_entry(struct ftrace_graph_ent *trace) | ||
241 | { | ||
242 | if (tracing_thresh) | ||
243 | return 1; | ||
244 | else | ||
245 | return trace_graph_entry(trace); | ||
246 | } | ||
247 | |||
240 | static void __trace_graph_return(struct trace_array *tr, | 248 | static void __trace_graph_return(struct trace_array *tr, |
241 | struct ftrace_graph_ret *trace, | 249 | struct ftrace_graph_ret *trace, |
242 | unsigned long flags, | 250 | unsigned long flags, |
@@ -247,7 +255,7 @@ static void __trace_graph_return(struct trace_array *tr, | |||
247 | struct ring_buffer *buffer = tr->buffer; | 255 | struct ring_buffer *buffer = tr->buffer; |
248 | struct ftrace_graph_ret_entry *entry; | 256 | struct ftrace_graph_ret_entry *entry; |
249 | 257 | ||
250 | if (unlikely(__this_cpu_read(per_cpu_var(ftrace_cpu_disabled)))) | 258 | if (unlikely(__this_cpu_read(ftrace_cpu_disabled))) |
251 | return; | 259 | return; |
252 | 260 | ||
253 | event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_RET, | 261 | event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_RET, |
@@ -290,13 +298,26 @@ void set_graph_array(struct trace_array *tr) | |||
290 | smp_mb(); | 298 | smp_mb(); |
291 | } | 299 | } |
292 | 300 | ||
301 | void trace_graph_thresh_return(struct ftrace_graph_ret *trace) | ||
302 | { | ||
303 | if (tracing_thresh && | ||
304 | (trace->rettime - trace->calltime < tracing_thresh)) | ||
305 | return; | ||
306 | else | ||
307 | trace_graph_return(trace); | ||
308 | } | ||
309 | |||
293 | static int graph_trace_init(struct trace_array *tr) | 310 | static int graph_trace_init(struct trace_array *tr) |
294 | { | 311 | { |
295 | int ret; | 312 | int ret; |
296 | 313 | ||
297 | set_graph_array(tr); | 314 | set_graph_array(tr); |
298 | ret = register_ftrace_graph(&trace_graph_return, | 315 | if (tracing_thresh) |
299 | &trace_graph_entry); | 316 | ret = register_ftrace_graph(&trace_graph_thresh_return, |
317 | &trace_graph_thresh_entry); | ||
318 | else | ||
319 | ret = register_ftrace_graph(&trace_graph_return, | ||
320 | &trace_graph_entry); | ||
300 | if (ret) | 321 | if (ret) |
301 | return ret; | 322 | return ret; |
302 | tracing_start_cmdline_record(); | 323 | tracing_start_cmdline_record(); |
@@ -920,7 +941,7 @@ print_graph_return(struct ftrace_graph_ret *trace, struct trace_seq *s, | |||
920 | if (!ret) | 941 | if (!ret) |
921 | return TRACE_TYPE_PARTIAL_LINE; | 942 | return TRACE_TYPE_PARTIAL_LINE; |
922 | } else { | 943 | } else { |
923 | ret = trace_seq_printf(s, "} (%ps)\n", (void *)trace->func); | 944 | ret = trace_seq_printf(s, "} /* %ps */\n", (void *)trace->func); |
924 | if (!ret) | 945 | if (!ret) |
925 | return TRACE_TYPE_PARTIAL_LINE; | 946 | return TRACE_TYPE_PARTIAL_LINE; |
926 | } | 947 | } |
diff --git a/kernel/trace/trace_kprobe.c b/kernel/trace/trace_kprobe.c index 505c92273b1a..1251e367bae9 100644 --- a/kernel/trace/trace_kprobe.c +++ b/kernel/trace/trace_kprobe.c | |||
@@ -1214,7 +1214,7 @@ static int set_print_fmt(struct trace_probe *tp) | |||
1214 | #ifdef CONFIG_PERF_EVENTS | 1214 | #ifdef CONFIG_PERF_EVENTS |
1215 | 1215 | ||
1216 | /* Kprobe profile handler */ | 1216 | /* Kprobe profile handler */ |
1217 | static __kprobes void kprobe_profile_func(struct kprobe *kp, | 1217 | static __kprobes void kprobe_perf_func(struct kprobe *kp, |
1218 | struct pt_regs *regs) | 1218 | struct pt_regs *regs) |
1219 | { | 1219 | { |
1220 | struct trace_probe *tp = container_of(kp, struct trace_probe, rp.kp); | 1220 | struct trace_probe *tp = container_of(kp, struct trace_probe, rp.kp); |
@@ -1227,11 +1227,11 @@ static __kprobes void kprobe_profile_func(struct kprobe *kp, | |||
1227 | __size = SIZEOF_KPROBE_TRACE_ENTRY(tp->nr_args); | 1227 | __size = SIZEOF_KPROBE_TRACE_ENTRY(tp->nr_args); |
1228 | size = ALIGN(__size + sizeof(u32), sizeof(u64)); | 1228 | size = ALIGN(__size + sizeof(u32), sizeof(u64)); |
1229 | size -= sizeof(u32); | 1229 | size -= sizeof(u32); |
1230 | if (WARN_ONCE(size > FTRACE_MAX_PROFILE_SIZE, | 1230 | if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE, |
1231 | "profile buffer not large enough")) | 1231 | "profile buffer not large enough")) |
1232 | return; | 1232 | return; |
1233 | 1233 | ||
1234 | entry = ftrace_perf_buf_prepare(size, call->id, &rctx, &irq_flags); | 1234 | entry = perf_trace_buf_prepare(size, call->id, &rctx, &irq_flags); |
1235 | if (!entry) | 1235 | if (!entry) |
1236 | return; | 1236 | return; |
1237 | 1237 | ||
@@ -1240,11 +1240,11 @@ static __kprobes void kprobe_profile_func(struct kprobe *kp, | |||
1240 | for (i = 0; i < tp->nr_args; i++) | 1240 | for (i = 0; i < tp->nr_args; i++) |
1241 | entry->args[i] = call_fetch(&tp->args[i].fetch, regs); | 1241 | entry->args[i] = call_fetch(&tp->args[i].fetch, regs); |
1242 | 1242 | ||
1243 | ftrace_perf_buf_submit(entry, size, rctx, entry->ip, 1, irq_flags); | 1243 | perf_trace_buf_submit(entry, size, rctx, entry->ip, 1, irq_flags, regs); |
1244 | } | 1244 | } |
1245 | 1245 | ||
1246 | /* Kretprobe profile handler */ | 1246 | /* Kretprobe profile handler */ |
1247 | static __kprobes void kretprobe_profile_func(struct kretprobe_instance *ri, | 1247 | static __kprobes void kretprobe_perf_func(struct kretprobe_instance *ri, |
1248 | struct pt_regs *regs) | 1248 | struct pt_regs *regs) |
1249 | { | 1249 | { |
1250 | struct trace_probe *tp = container_of(ri->rp, struct trace_probe, rp); | 1250 | struct trace_probe *tp = container_of(ri->rp, struct trace_probe, rp); |
@@ -1257,11 +1257,11 @@ static __kprobes void kretprobe_profile_func(struct kretprobe_instance *ri, | |||
1257 | __size = SIZEOF_KRETPROBE_TRACE_ENTRY(tp->nr_args); | 1257 | __size = SIZEOF_KRETPROBE_TRACE_ENTRY(tp->nr_args); |
1258 | size = ALIGN(__size + sizeof(u32), sizeof(u64)); | 1258 | size = ALIGN(__size + sizeof(u32), sizeof(u64)); |
1259 | size -= sizeof(u32); | 1259 | size -= sizeof(u32); |
1260 | if (WARN_ONCE(size > FTRACE_MAX_PROFILE_SIZE, | 1260 | if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE, |
1261 | "profile buffer not large enough")) | 1261 | "profile buffer not large enough")) |
1262 | return; | 1262 | return; |
1263 | 1263 | ||
1264 | entry = ftrace_perf_buf_prepare(size, call->id, &rctx, &irq_flags); | 1264 | entry = perf_trace_buf_prepare(size, call->id, &rctx, &irq_flags); |
1265 | if (!entry) | 1265 | if (!entry) |
1266 | return; | 1266 | return; |
1267 | 1267 | ||
@@ -1271,10 +1271,11 @@ static __kprobes void kretprobe_profile_func(struct kretprobe_instance *ri, | |||
1271 | for (i = 0; i < tp->nr_args; i++) | 1271 | for (i = 0; i < tp->nr_args; i++) |
1272 | entry->args[i] = call_fetch(&tp->args[i].fetch, regs); | 1272 | entry->args[i] = call_fetch(&tp->args[i].fetch, regs); |
1273 | 1273 | ||
1274 | ftrace_perf_buf_submit(entry, size, rctx, entry->ret_ip, 1, irq_flags); | 1274 | perf_trace_buf_submit(entry, size, rctx, entry->ret_ip, 1, |
1275 | irq_flags, regs); | ||
1275 | } | 1276 | } |
1276 | 1277 | ||
1277 | static int probe_profile_enable(struct ftrace_event_call *call) | 1278 | static int probe_perf_enable(struct ftrace_event_call *call) |
1278 | { | 1279 | { |
1279 | struct trace_probe *tp = (struct trace_probe *)call->data; | 1280 | struct trace_probe *tp = (struct trace_probe *)call->data; |
1280 | 1281 | ||
@@ -1286,7 +1287,7 @@ static int probe_profile_enable(struct ftrace_event_call *call) | |||
1286 | return enable_kprobe(&tp->rp.kp); | 1287 | return enable_kprobe(&tp->rp.kp); |
1287 | } | 1288 | } |
1288 | 1289 | ||
1289 | static void probe_profile_disable(struct ftrace_event_call *call) | 1290 | static void probe_perf_disable(struct ftrace_event_call *call) |
1290 | { | 1291 | { |
1291 | struct trace_probe *tp = (struct trace_probe *)call->data; | 1292 | struct trace_probe *tp = (struct trace_probe *)call->data; |
1292 | 1293 | ||
@@ -1311,7 +1312,7 @@ int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs) | |||
1311 | kprobe_trace_func(kp, regs); | 1312 | kprobe_trace_func(kp, regs); |
1312 | #ifdef CONFIG_PERF_EVENTS | 1313 | #ifdef CONFIG_PERF_EVENTS |
1313 | if (tp->flags & TP_FLAG_PROFILE) | 1314 | if (tp->flags & TP_FLAG_PROFILE) |
1314 | kprobe_profile_func(kp, regs); | 1315 | kprobe_perf_func(kp, regs); |
1315 | #endif | 1316 | #endif |
1316 | return 0; /* We don't tweek kernel, so just return 0 */ | 1317 | return 0; /* We don't tweek kernel, so just return 0 */ |
1317 | } | 1318 | } |
@@ -1325,7 +1326,7 @@ int kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs) | |||
1325 | kretprobe_trace_func(ri, regs); | 1326 | kretprobe_trace_func(ri, regs); |
1326 | #ifdef CONFIG_PERF_EVENTS | 1327 | #ifdef CONFIG_PERF_EVENTS |
1327 | if (tp->flags & TP_FLAG_PROFILE) | 1328 | if (tp->flags & TP_FLAG_PROFILE) |
1328 | kretprobe_profile_func(ri, regs); | 1329 | kretprobe_perf_func(ri, regs); |
1329 | #endif | 1330 | #endif |
1330 | return 0; /* We don't tweek kernel, so just return 0 */ | 1331 | return 0; /* We don't tweek kernel, so just return 0 */ |
1331 | } | 1332 | } |
@@ -1358,8 +1359,8 @@ static int register_probe_event(struct trace_probe *tp) | |||
1358 | call->unregfunc = probe_event_disable; | 1359 | call->unregfunc = probe_event_disable; |
1359 | 1360 | ||
1360 | #ifdef CONFIG_PERF_EVENTS | 1361 | #ifdef CONFIG_PERF_EVENTS |
1361 | call->profile_enable = probe_profile_enable; | 1362 | call->perf_event_enable = probe_perf_enable; |
1362 | call->profile_disable = probe_profile_disable; | 1363 | call->perf_event_disable = probe_perf_disable; |
1363 | #endif | 1364 | #endif |
1364 | call->data = tp; | 1365 | call->data = tp; |
1365 | ret = trace_add_event_call(call); | 1366 | ret = trace_add_event_call(call); |
diff --git a/kernel/trace/trace_syscalls.c b/kernel/trace/trace_syscalls.c index cba47d7935cc..33c2a5b769dc 100644 --- a/kernel/trace/trace_syscalls.c +++ b/kernel/trace/trace_syscalls.c | |||
@@ -428,12 +428,12 @@ core_initcall(init_ftrace_syscalls); | |||
428 | 428 | ||
429 | #ifdef CONFIG_PERF_EVENTS | 429 | #ifdef CONFIG_PERF_EVENTS |
430 | 430 | ||
431 | static DECLARE_BITMAP(enabled_prof_enter_syscalls, NR_syscalls); | 431 | static DECLARE_BITMAP(enabled_perf_enter_syscalls, NR_syscalls); |
432 | static DECLARE_BITMAP(enabled_prof_exit_syscalls, NR_syscalls); | 432 | static DECLARE_BITMAP(enabled_perf_exit_syscalls, NR_syscalls); |
433 | static int sys_prof_refcount_enter; | 433 | static int sys_perf_refcount_enter; |
434 | static int sys_prof_refcount_exit; | 434 | static int sys_perf_refcount_exit; |
435 | 435 | ||
436 | static void prof_syscall_enter(struct pt_regs *regs, long id) | 436 | static void perf_syscall_enter(struct pt_regs *regs, long id) |
437 | { | 437 | { |
438 | struct syscall_metadata *sys_data; | 438 | struct syscall_metadata *sys_data; |
439 | struct syscall_trace_enter *rec; | 439 | struct syscall_trace_enter *rec; |
@@ -443,7 +443,7 @@ static void prof_syscall_enter(struct pt_regs *regs, long id) | |||
443 | int size; | 443 | int size; |
444 | 444 | ||
445 | syscall_nr = syscall_get_nr(current, regs); | 445 | syscall_nr = syscall_get_nr(current, regs); |
446 | if (!test_bit(syscall_nr, enabled_prof_enter_syscalls)) | 446 | if (!test_bit(syscall_nr, enabled_perf_enter_syscalls)) |
447 | return; | 447 | return; |
448 | 448 | ||
449 | sys_data = syscall_nr_to_meta(syscall_nr); | 449 | sys_data = syscall_nr_to_meta(syscall_nr); |
@@ -455,11 +455,11 @@ static void prof_syscall_enter(struct pt_regs *regs, long id) | |||
455 | size = ALIGN(size + sizeof(u32), sizeof(u64)); | 455 | size = ALIGN(size + sizeof(u32), sizeof(u64)); |
456 | size -= sizeof(u32); | 456 | size -= sizeof(u32); |
457 | 457 | ||
458 | if (WARN_ONCE(size > FTRACE_MAX_PROFILE_SIZE, | 458 | if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE, |
459 | "profile buffer not large enough")) | 459 | "perf buffer not large enough")) |
460 | return; | 460 | return; |
461 | 461 | ||
462 | rec = (struct syscall_trace_enter *)ftrace_perf_buf_prepare(size, | 462 | rec = (struct syscall_trace_enter *)perf_trace_buf_prepare(size, |
463 | sys_data->enter_event->id, &rctx, &flags); | 463 | sys_data->enter_event->id, &rctx, &flags); |
464 | if (!rec) | 464 | if (!rec) |
465 | return; | 465 | return; |
@@ -467,10 +467,10 @@ static void prof_syscall_enter(struct pt_regs *regs, long id) | |||
467 | rec->nr = syscall_nr; | 467 | rec->nr = syscall_nr; |
468 | syscall_get_arguments(current, regs, 0, sys_data->nb_args, | 468 | syscall_get_arguments(current, regs, 0, sys_data->nb_args, |
469 | (unsigned long *)&rec->args); | 469 | (unsigned long *)&rec->args); |
470 | ftrace_perf_buf_submit(rec, size, rctx, 0, 1, flags); | 470 | perf_trace_buf_submit(rec, size, rctx, 0, 1, flags, regs); |
471 | } | 471 | } |
472 | 472 | ||
473 | int prof_sysenter_enable(struct ftrace_event_call *call) | 473 | int perf_sysenter_enable(struct ftrace_event_call *call) |
474 | { | 474 | { |
475 | int ret = 0; | 475 | int ret = 0; |
476 | int num; | 476 | int num; |
@@ -478,34 +478,34 @@ int prof_sysenter_enable(struct ftrace_event_call *call) | |||
478 | num = ((struct syscall_metadata *)call->data)->syscall_nr; | 478 | num = ((struct syscall_metadata *)call->data)->syscall_nr; |
479 | 479 | ||
480 | mutex_lock(&syscall_trace_lock); | 480 | mutex_lock(&syscall_trace_lock); |
481 | if (!sys_prof_refcount_enter) | 481 | if (!sys_perf_refcount_enter) |
482 | ret = register_trace_sys_enter(prof_syscall_enter); | 482 | ret = register_trace_sys_enter(perf_syscall_enter); |
483 | if (ret) { | 483 | if (ret) { |
484 | pr_info("event trace: Could not activate" | 484 | pr_info("event trace: Could not activate" |
485 | "syscall entry trace point"); | 485 | "syscall entry trace point"); |
486 | } else { | 486 | } else { |
487 | set_bit(num, enabled_prof_enter_syscalls); | 487 | set_bit(num, enabled_perf_enter_syscalls); |
488 | sys_prof_refcount_enter++; | 488 | sys_perf_refcount_enter++; |
489 | } | 489 | } |
490 | mutex_unlock(&syscall_trace_lock); | 490 | mutex_unlock(&syscall_trace_lock); |
491 | return ret; | 491 | return ret; |
492 | } | 492 | } |
493 | 493 | ||
494 | void prof_sysenter_disable(struct ftrace_event_call *call) | 494 | void perf_sysenter_disable(struct ftrace_event_call *call) |
495 | { | 495 | { |
496 | int num; | 496 | int num; |
497 | 497 | ||
498 | num = ((struct syscall_metadata *)call->data)->syscall_nr; | 498 | num = ((struct syscall_metadata *)call->data)->syscall_nr; |
499 | 499 | ||
500 | mutex_lock(&syscall_trace_lock); | 500 | mutex_lock(&syscall_trace_lock); |
501 | sys_prof_refcount_enter--; | 501 | sys_perf_refcount_enter--; |
502 | clear_bit(num, enabled_prof_enter_syscalls); | 502 | clear_bit(num, enabled_perf_enter_syscalls); |
503 | if (!sys_prof_refcount_enter) | 503 | if (!sys_perf_refcount_enter) |
504 | unregister_trace_sys_enter(prof_syscall_enter); | 504 | unregister_trace_sys_enter(perf_syscall_enter); |
505 | mutex_unlock(&syscall_trace_lock); | 505 | mutex_unlock(&syscall_trace_lock); |
506 | } | 506 | } |
507 | 507 | ||
508 | static void prof_syscall_exit(struct pt_regs *regs, long ret) | 508 | static void perf_syscall_exit(struct pt_regs *regs, long ret) |
509 | { | 509 | { |
510 | struct syscall_metadata *sys_data; | 510 | struct syscall_metadata *sys_data; |
511 | struct syscall_trace_exit *rec; | 511 | struct syscall_trace_exit *rec; |
@@ -515,7 +515,7 @@ static void prof_syscall_exit(struct pt_regs *regs, long ret) | |||
515 | int size; | 515 | int size; |
516 | 516 | ||
517 | syscall_nr = syscall_get_nr(current, regs); | 517 | syscall_nr = syscall_get_nr(current, regs); |
518 | if (!test_bit(syscall_nr, enabled_prof_exit_syscalls)) | 518 | if (!test_bit(syscall_nr, enabled_perf_exit_syscalls)) |
519 | return; | 519 | return; |
520 | 520 | ||
521 | sys_data = syscall_nr_to_meta(syscall_nr); | 521 | sys_data = syscall_nr_to_meta(syscall_nr); |
@@ -530,11 +530,11 @@ static void prof_syscall_exit(struct pt_regs *regs, long ret) | |||
530 | * Impossible, but be paranoid with the future | 530 | * Impossible, but be paranoid with the future |
531 | * How to put this check outside runtime? | 531 | * How to put this check outside runtime? |
532 | */ | 532 | */ |
533 | if (WARN_ONCE(size > FTRACE_MAX_PROFILE_SIZE, | 533 | if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE, |
534 | "exit event has grown above profile buffer size")) | 534 | "exit event has grown above perf buffer size")) |
535 | return; | 535 | return; |
536 | 536 | ||
537 | rec = (struct syscall_trace_exit *)ftrace_perf_buf_prepare(size, | 537 | rec = (struct syscall_trace_exit *)perf_trace_buf_prepare(size, |
538 | sys_data->exit_event->id, &rctx, &flags); | 538 | sys_data->exit_event->id, &rctx, &flags); |
539 | if (!rec) | 539 | if (!rec) |
540 | return; | 540 | return; |
@@ -542,10 +542,10 @@ static void prof_syscall_exit(struct pt_regs *regs, long ret) | |||
542 | rec->nr = syscall_nr; | 542 | rec->nr = syscall_nr; |
543 | rec->ret = syscall_get_return_value(current, regs); | 543 | rec->ret = syscall_get_return_value(current, regs); |
544 | 544 | ||
545 | ftrace_perf_buf_submit(rec, size, rctx, 0, 1, flags); | 545 | perf_trace_buf_submit(rec, size, rctx, 0, 1, flags, regs); |
546 | } | 546 | } |
547 | 547 | ||
548 | int prof_sysexit_enable(struct ftrace_event_call *call) | 548 | int perf_sysexit_enable(struct ftrace_event_call *call) |
549 | { | 549 | { |
550 | int ret = 0; | 550 | int ret = 0; |
551 | int num; | 551 | int num; |
@@ -553,30 +553,30 @@ int prof_sysexit_enable(struct ftrace_event_call *call) | |||
553 | num = ((struct syscall_metadata *)call->data)->syscall_nr; | 553 | num = ((struct syscall_metadata *)call->data)->syscall_nr; |
554 | 554 | ||
555 | mutex_lock(&syscall_trace_lock); | 555 | mutex_lock(&syscall_trace_lock); |
556 | if (!sys_prof_refcount_exit) | 556 | if (!sys_perf_refcount_exit) |
557 | ret = register_trace_sys_exit(prof_syscall_exit); | 557 | ret = register_trace_sys_exit(perf_syscall_exit); |
558 | if (ret) { | 558 | if (ret) { |
559 | pr_info("event trace: Could not activate" | 559 | pr_info("event trace: Could not activate" |
560 | "syscall exit trace point"); | 560 | "syscall exit trace point"); |
561 | } else { | 561 | } else { |
562 | set_bit(num, enabled_prof_exit_syscalls); | 562 | set_bit(num, enabled_perf_exit_syscalls); |
563 | sys_prof_refcount_exit++; | 563 | sys_perf_refcount_exit++; |
564 | } | 564 | } |
565 | mutex_unlock(&syscall_trace_lock); | 565 | mutex_unlock(&syscall_trace_lock); |
566 | return ret; | 566 | return ret; |
567 | } | 567 | } |
568 | 568 | ||
569 | void prof_sysexit_disable(struct ftrace_event_call *call) | 569 | void perf_sysexit_disable(struct ftrace_event_call *call) |
570 | { | 570 | { |
571 | int num; | 571 | int num; |
572 | 572 | ||
573 | num = ((struct syscall_metadata *)call->data)->syscall_nr; | 573 | num = ((struct syscall_metadata *)call->data)->syscall_nr; |
574 | 574 | ||
575 | mutex_lock(&syscall_trace_lock); | 575 | mutex_lock(&syscall_trace_lock); |
576 | sys_prof_refcount_exit--; | 576 | sys_perf_refcount_exit--; |
577 | clear_bit(num, enabled_prof_exit_syscalls); | 577 | clear_bit(num, enabled_perf_exit_syscalls); |
578 | if (!sys_prof_refcount_exit) | 578 | if (!sys_perf_refcount_exit) |
579 | unregister_trace_sys_exit(prof_syscall_exit); | 579 | unregister_trace_sys_exit(perf_syscall_exit); |
580 | mutex_unlock(&syscall_trace_lock); | 580 | mutex_unlock(&syscall_trace_lock); |
581 | } | 581 | } |
582 | 582 | ||