diff options
Diffstat (limited to 'kernel/trace')
| -rw-r--r-- | kernel/trace/Makefile | 2 | ||||
| -rw-r--r-- | kernel/trace/ftrace.c | 30 | ||||
| -rw-r--r-- | kernel/trace/ring_buffer.c | 31 | ||||
| -rw-r--r-- | kernel/trace/ring_buffer_benchmark.c | 1 | ||||
| -rw-r--r-- | kernel/trace/trace.c | 55 | ||||
| -rw-r--r-- | kernel/trace/trace.h | 5 | ||||
| -rw-r--r-- | kernel/trace/trace_clock.c | 1 | ||||
| -rw-r--r-- | kernel/trace/trace_event_perf.c (renamed from kernel/trace/trace_event_profile.c) | 54 | ||||
| -rw-r--r-- | kernel/trace/trace_events.c | 2 | ||||
| -rw-r--r-- | kernel/trace/trace_functions_graph.c | 31 | ||||
| -rw-r--r-- | kernel/trace/trace_kprobe.c | 29 | ||||
| -rw-r--r-- | kernel/trace/trace_syscalls.c | 72 |
12 files changed, 193 insertions, 120 deletions
diff --git a/kernel/trace/Makefile b/kernel/trace/Makefile index d00c6fe23f54..78edc6490038 100644 --- a/kernel/trace/Makefile +++ b/kernel/trace/Makefile | |||
| @@ -52,7 +52,7 @@ obj-$(CONFIG_EVENT_TRACING) += trace_events.o | |||
| 52 | obj-$(CONFIG_EVENT_TRACING) += trace_export.o | 52 | obj-$(CONFIG_EVENT_TRACING) += trace_export.o |
| 53 | obj-$(CONFIG_FTRACE_SYSCALLS) += trace_syscalls.o | 53 | obj-$(CONFIG_FTRACE_SYSCALLS) += trace_syscalls.o |
| 54 | ifeq ($(CONFIG_PERF_EVENTS),y) | 54 | ifeq ($(CONFIG_PERF_EVENTS),y) |
| 55 | obj-$(CONFIG_EVENT_TRACING) += trace_event_profile.o | 55 | obj-$(CONFIG_EVENT_TRACING) += trace_event_perf.o |
| 56 | endif | 56 | endif |
| 57 | obj-$(CONFIG_EVENT_TRACING) += trace_events_filter.o | 57 | obj-$(CONFIG_EVENT_TRACING) += trace_events_filter.o |
| 58 | obj-$(CONFIG_KPROBE_EVENT) += trace_kprobe.o | 58 | obj-$(CONFIG_KPROBE_EVENT) += trace_kprobe.o |
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 83783579378f..d9062f5cc0c0 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c | |||
| @@ -27,6 +27,7 @@ | |||
| 27 | #include <linux/ctype.h> | 27 | #include <linux/ctype.h> |
| 28 | #include <linux/list.h> | 28 | #include <linux/list.h> |
| 29 | #include <linux/hash.h> | 29 | #include <linux/hash.h> |
| 30 | #include <linux/rcupdate.h> | ||
| 30 | 31 | ||
| 31 | #include <trace/events/sched.h> | 32 | #include <trace/events/sched.h> |
| 32 | 33 | ||
| @@ -84,22 +85,22 @@ ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub; | |||
| 84 | ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub; | 85 | ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub; |
| 85 | ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub; | 86 | ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub; |
| 86 | 87 | ||
| 87 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | 88 | /* |
| 88 | static int ftrace_set_func(unsigned long *array, int *idx, char *buffer); | 89 | * Traverse the ftrace_list, invoking all entries. The reason that we |
| 89 | #endif | 90 | * can use rcu_dereference_raw() is that elements removed from this list |
| 90 | 91 | * are simply leaked, so there is no need to interact with a grace-period | |
| 92 | * mechanism. The rcu_dereference_raw() calls are needed to handle | ||
| 93 | * concurrent insertions into the ftrace_list. | ||
| 94 | * | ||
| 95 | * Silly Alpha and silly pointer-speculation compiler optimizations! | ||
| 96 | */ | ||
| 91 | static void ftrace_list_func(unsigned long ip, unsigned long parent_ip) | 97 | static void ftrace_list_func(unsigned long ip, unsigned long parent_ip) |
| 92 | { | 98 | { |
| 93 | struct ftrace_ops *op = ftrace_list; | 99 | struct ftrace_ops *op = rcu_dereference_raw(ftrace_list); /*see above*/ |
| 94 | |||
| 95 | /* in case someone actually ports this to alpha! */ | ||
| 96 | read_barrier_depends(); | ||
| 97 | 100 | ||
| 98 | while (op != &ftrace_list_end) { | 101 | while (op != &ftrace_list_end) { |
| 99 | /* silly alpha */ | ||
| 100 | read_barrier_depends(); | ||
| 101 | op->func(ip, parent_ip); | 102 | op->func(ip, parent_ip); |
| 102 | op = op->next; | 103 | op = rcu_dereference_raw(op->next); /*see above*/ |
| 103 | }; | 104 | }; |
| 104 | } | 105 | } |
| 105 | 106 | ||
| @@ -154,8 +155,7 @@ static int __register_ftrace_function(struct ftrace_ops *ops) | |||
| 154 | * the ops->next pointer is valid before another CPU sees | 155 | * the ops->next pointer is valid before another CPU sees |
| 155 | * the ops pointer included into the ftrace_list. | 156 | * the ops pointer included into the ftrace_list. |
| 156 | */ | 157 | */ |
| 157 | smp_wmb(); | 158 | rcu_assign_pointer(ftrace_list, ops); |
| 158 | ftrace_list = ops; | ||
| 159 | 159 | ||
| 160 | if (ftrace_enabled) { | 160 | if (ftrace_enabled) { |
| 161 | ftrace_func_t func; | 161 | ftrace_func_t func; |
| @@ -2276,6 +2276,8 @@ __setup("ftrace_filter=", set_ftrace_filter); | |||
| 2276 | 2276 | ||
| 2277 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | 2277 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
| 2278 | static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata; | 2278 | static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata; |
| 2279 | static int ftrace_set_func(unsigned long *array, int *idx, char *buffer); | ||
| 2280 | |||
| 2279 | static int __init set_graph_function(char *str) | 2281 | static int __init set_graph_function(char *str) |
| 2280 | { | 2282 | { |
| 2281 | strlcpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE); | 2283 | strlcpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE); |
| @@ -3351,6 +3353,7 @@ void ftrace_graph_init_task(struct task_struct *t) | |||
| 3351 | { | 3353 | { |
| 3352 | /* Make sure we do not use the parent ret_stack */ | 3354 | /* Make sure we do not use the parent ret_stack */ |
| 3353 | t->ret_stack = NULL; | 3355 | t->ret_stack = NULL; |
| 3356 | t->curr_ret_stack = -1; | ||
| 3354 | 3357 | ||
| 3355 | if (ftrace_graph_active) { | 3358 | if (ftrace_graph_active) { |
| 3356 | struct ftrace_ret_stack *ret_stack; | 3359 | struct ftrace_ret_stack *ret_stack; |
| @@ -3360,7 +3363,6 @@ void ftrace_graph_init_task(struct task_struct *t) | |||
| 3360 | GFP_KERNEL); | 3363 | GFP_KERNEL); |
| 3361 | if (!ret_stack) | 3364 | if (!ret_stack) |
| 3362 | return; | 3365 | return; |
| 3363 | t->curr_ret_stack = -1; | ||
| 3364 | atomic_set(&t->tracing_graph_pause, 0); | 3366 | atomic_set(&t->tracing_graph_pause, 0); |
| 3365 | atomic_set(&t->trace_overrun, 0); | 3367 | atomic_set(&t->trace_overrun, 0); |
| 3366 | t->ftrace_timestamp = 0; | 3368 | t->ftrace_timestamp = 0; |
diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 8c1b2d290718..d1187ef20caf 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c | |||
| @@ -20,6 +20,7 @@ | |||
| 20 | #include <linux/cpu.h> | 20 | #include <linux/cpu.h> |
| 21 | #include <linux/fs.h> | 21 | #include <linux/fs.h> |
| 22 | 22 | ||
| 23 | #include <asm/local.h> | ||
| 23 | #include "trace.h" | 24 | #include "trace.h" |
| 24 | 25 | ||
| 25 | /* | 26 | /* |
| @@ -206,6 +207,14 @@ EXPORT_SYMBOL_GPL(tracing_is_on); | |||
| 206 | #define RB_MAX_SMALL_DATA (RB_ALIGNMENT * RINGBUF_TYPE_DATA_TYPE_LEN_MAX) | 207 | #define RB_MAX_SMALL_DATA (RB_ALIGNMENT * RINGBUF_TYPE_DATA_TYPE_LEN_MAX) |
| 207 | #define RB_EVNT_MIN_SIZE 8U /* two 32bit words */ | 208 | #define RB_EVNT_MIN_SIZE 8U /* two 32bit words */ |
| 208 | 209 | ||
| 210 | #if !defined(CONFIG_64BIT) || defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) | ||
| 211 | # define RB_FORCE_8BYTE_ALIGNMENT 0 | ||
| 212 | # define RB_ARCH_ALIGNMENT RB_ALIGNMENT | ||
| 213 | #else | ||
| 214 | # define RB_FORCE_8BYTE_ALIGNMENT 1 | ||
| 215 | # define RB_ARCH_ALIGNMENT 8U | ||
| 216 | #endif | ||
| 217 | |||
| 209 | /* define RINGBUF_TYPE_DATA for 'case RINGBUF_TYPE_DATA:' */ | 218 | /* define RINGBUF_TYPE_DATA for 'case RINGBUF_TYPE_DATA:' */ |
| 210 | #define RINGBUF_TYPE_DATA 0 ... RINGBUF_TYPE_DATA_TYPE_LEN_MAX | 219 | #define RINGBUF_TYPE_DATA 0 ... RINGBUF_TYPE_DATA_TYPE_LEN_MAX |
| 211 | 220 | ||
| @@ -1546,7 +1555,7 @@ rb_update_event(struct ring_buffer_event *event, | |||
| 1546 | 1555 | ||
| 1547 | case 0: | 1556 | case 0: |
| 1548 | length -= RB_EVNT_HDR_SIZE; | 1557 | length -= RB_EVNT_HDR_SIZE; |
| 1549 | if (length > RB_MAX_SMALL_DATA) | 1558 | if (length > RB_MAX_SMALL_DATA || RB_FORCE_8BYTE_ALIGNMENT) |
| 1550 | event->array[0] = length; | 1559 | event->array[0] = length; |
| 1551 | else | 1560 | else |
| 1552 | event->type_len = DIV_ROUND_UP(length, RB_ALIGNMENT); | 1561 | event->type_len = DIV_ROUND_UP(length, RB_ALIGNMENT); |
| @@ -1721,11 +1730,11 @@ static unsigned rb_calculate_event_length(unsigned length) | |||
| 1721 | if (!length) | 1730 | if (!length) |
| 1722 | length = 1; | 1731 | length = 1; |
| 1723 | 1732 | ||
| 1724 | if (length > RB_MAX_SMALL_DATA) | 1733 | if (length > RB_MAX_SMALL_DATA || RB_FORCE_8BYTE_ALIGNMENT) |
| 1725 | length += sizeof(event.array[0]); | 1734 | length += sizeof(event.array[0]); |
| 1726 | 1735 | ||
| 1727 | length += RB_EVNT_HDR_SIZE; | 1736 | length += RB_EVNT_HDR_SIZE; |
| 1728 | length = ALIGN(length, RB_ALIGNMENT); | 1737 | length = ALIGN(length, RB_ARCH_ALIGNMENT); |
| 1729 | 1738 | ||
| 1730 | return length; | 1739 | return length; |
| 1731 | } | 1740 | } |
| @@ -2232,12 +2241,12 @@ ring_buffer_lock_reserve(struct ring_buffer *buffer, unsigned long length) | |||
| 2232 | if (ring_buffer_flags != RB_BUFFERS_ON) | 2241 | if (ring_buffer_flags != RB_BUFFERS_ON) |
| 2233 | return NULL; | 2242 | return NULL; |
| 2234 | 2243 | ||
| 2235 | if (atomic_read(&buffer->record_disabled)) | ||
| 2236 | return NULL; | ||
| 2237 | |||
| 2238 | /* If we are tracing schedule, we don't want to recurse */ | 2244 | /* If we are tracing schedule, we don't want to recurse */ |
| 2239 | resched = ftrace_preempt_disable(); | 2245 | resched = ftrace_preempt_disable(); |
| 2240 | 2246 | ||
| 2247 | if (atomic_read(&buffer->record_disabled)) | ||
| 2248 | goto out_nocheck; | ||
| 2249 | |||
| 2241 | if (trace_recursive_lock()) | 2250 | if (trace_recursive_lock()) |
| 2242 | goto out_nocheck; | 2251 | goto out_nocheck; |
| 2243 | 2252 | ||
| @@ -2469,11 +2478,11 @@ int ring_buffer_write(struct ring_buffer *buffer, | |||
| 2469 | if (ring_buffer_flags != RB_BUFFERS_ON) | 2478 | if (ring_buffer_flags != RB_BUFFERS_ON) |
| 2470 | return -EBUSY; | 2479 | return -EBUSY; |
| 2471 | 2480 | ||
| 2472 | if (atomic_read(&buffer->record_disabled)) | ||
| 2473 | return -EBUSY; | ||
| 2474 | |||
| 2475 | resched = ftrace_preempt_disable(); | 2481 | resched = ftrace_preempt_disable(); |
| 2476 | 2482 | ||
| 2483 | if (atomic_read(&buffer->record_disabled)) | ||
| 2484 | goto out; | ||
| 2485 | |||
| 2477 | cpu = raw_smp_processor_id(); | 2486 | cpu = raw_smp_processor_id(); |
| 2478 | 2487 | ||
| 2479 | if (!cpumask_test_cpu(cpu, buffer->cpumask)) | 2488 | if (!cpumask_test_cpu(cpu, buffer->cpumask)) |
| @@ -2541,7 +2550,7 @@ EXPORT_SYMBOL_GPL(ring_buffer_record_disable); | |||
| 2541 | * @buffer: The ring buffer to enable writes | 2550 | * @buffer: The ring buffer to enable writes |
| 2542 | * | 2551 | * |
| 2543 | * Note, multiple disables will need the same number of enables | 2552 | * Note, multiple disables will need the same number of enables |
| 2544 | * to truely enable the writing (much like preempt_disable). | 2553 | * to truly enable the writing (much like preempt_disable). |
| 2545 | */ | 2554 | */ |
| 2546 | void ring_buffer_record_enable(struct ring_buffer *buffer) | 2555 | void ring_buffer_record_enable(struct ring_buffer *buffer) |
| 2547 | { | 2556 | { |
| @@ -2577,7 +2586,7 @@ EXPORT_SYMBOL_GPL(ring_buffer_record_disable_cpu); | |||
| 2577 | * @cpu: The CPU to enable. | 2586 | * @cpu: The CPU to enable. |
| 2578 | * | 2587 | * |
| 2579 | * Note, multiple disables will need the same number of enables | 2588 | * Note, multiple disables will need the same number of enables |
| 2580 | * to truely enable the writing (much like preempt_disable). | 2589 | * to truly enable the writing (much like preempt_disable). |
| 2581 | */ | 2590 | */ |
| 2582 | void ring_buffer_record_enable_cpu(struct ring_buffer *buffer, int cpu) | 2591 | void ring_buffer_record_enable_cpu(struct ring_buffer *buffer, int cpu) |
| 2583 | { | 2592 | { |
diff --git a/kernel/trace/ring_buffer_benchmark.c b/kernel/trace/ring_buffer_benchmark.c index b2477caf09c2..df74c7982255 100644 --- a/kernel/trace/ring_buffer_benchmark.c +++ b/kernel/trace/ring_buffer_benchmark.c | |||
| @@ -8,6 +8,7 @@ | |||
| 8 | #include <linux/kthread.h> | 8 | #include <linux/kthread.h> |
| 9 | #include <linux/module.h> | 9 | #include <linux/module.h> |
| 10 | #include <linux/time.h> | 10 | #include <linux/time.h> |
| 11 | #include <asm/local.h> | ||
| 11 | 12 | ||
| 12 | struct rb_page { | 13 | struct rb_page { |
| 13 | u64 ts; | 14 | u64 ts; |
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 032c57ca6502..3ec2ee6f6560 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c | |||
| @@ -92,12 +92,12 @@ DEFINE_PER_CPU(int, ftrace_cpu_disabled); | |||
| 92 | static inline void ftrace_disable_cpu(void) | 92 | static inline void ftrace_disable_cpu(void) |
| 93 | { | 93 | { |
| 94 | preempt_disable(); | 94 | preempt_disable(); |
| 95 | __this_cpu_inc(per_cpu_var(ftrace_cpu_disabled)); | 95 | __this_cpu_inc(ftrace_cpu_disabled); |
| 96 | } | 96 | } |
| 97 | 97 | ||
| 98 | static inline void ftrace_enable_cpu(void) | 98 | static inline void ftrace_enable_cpu(void) |
| 99 | { | 99 | { |
| 100 | __this_cpu_dec(per_cpu_var(ftrace_cpu_disabled)); | 100 | __this_cpu_dec(ftrace_cpu_disabled); |
| 101 | preempt_enable(); | 101 | preempt_enable(); |
| 102 | } | 102 | } |
| 103 | 103 | ||
| @@ -374,6 +374,21 @@ static int __init set_buf_size(char *str) | |||
| 374 | } | 374 | } |
| 375 | __setup("trace_buf_size=", set_buf_size); | 375 | __setup("trace_buf_size=", set_buf_size); |
| 376 | 376 | ||
| 377 | static int __init set_tracing_thresh(char *str) | ||
| 378 | { | ||
| 379 | unsigned long threshhold; | ||
| 380 | int ret; | ||
| 381 | |||
| 382 | if (!str) | ||
| 383 | return 0; | ||
| 384 | ret = strict_strtoul(str, 0, &threshhold); | ||
| 385 | if (ret < 0) | ||
| 386 | return 0; | ||
| 387 | tracing_thresh = threshhold * 1000; | ||
| 388 | return 1; | ||
| 389 | } | ||
| 390 | __setup("tracing_thresh=", set_tracing_thresh); | ||
| 391 | |||
| 377 | unsigned long nsecs_to_usecs(unsigned long nsecs) | 392 | unsigned long nsecs_to_usecs(unsigned long nsecs) |
| 378 | { | 393 | { |
| 379 | return nsecs / 1000; | 394 | return nsecs / 1000; |
| @@ -579,9 +594,10 @@ static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt) | |||
| 579 | static arch_spinlock_t ftrace_max_lock = | 594 | static arch_spinlock_t ftrace_max_lock = |
| 580 | (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; | 595 | (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; |
| 581 | 596 | ||
| 597 | unsigned long __read_mostly tracing_thresh; | ||
| 598 | |||
| 582 | #ifdef CONFIG_TRACER_MAX_TRACE | 599 | #ifdef CONFIG_TRACER_MAX_TRACE |
| 583 | unsigned long __read_mostly tracing_max_latency; | 600 | unsigned long __read_mostly tracing_max_latency; |
| 584 | unsigned long __read_mostly tracing_thresh; | ||
| 585 | 601 | ||
| 586 | /* | 602 | /* |
| 587 | * Copy the new maximum trace into the separate maximum-trace | 603 | * Copy the new maximum trace into the separate maximum-trace |
| @@ -592,7 +608,7 @@ static void | |||
| 592 | __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu) | 608 | __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu) |
| 593 | { | 609 | { |
| 594 | struct trace_array_cpu *data = tr->data[cpu]; | 610 | struct trace_array_cpu *data = tr->data[cpu]; |
| 595 | struct trace_array_cpu *max_data = tr->data[cpu]; | 611 | struct trace_array_cpu *max_data; |
| 596 | 612 | ||
| 597 | max_tr.cpu = cpu; | 613 | max_tr.cpu = cpu; |
| 598 | max_tr.time_start = data->preempt_timestamp; | 614 | max_tr.time_start = data->preempt_timestamp; |
| @@ -602,7 +618,7 @@ __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu) | |||
| 602 | max_data->critical_start = data->critical_start; | 618 | max_data->critical_start = data->critical_start; |
| 603 | max_data->critical_end = data->critical_end; | 619 | max_data->critical_end = data->critical_end; |
| 604 | 620 | ||
| 605 | memcpy(data->comm, tsk->comm, TASK_COMM_LEN); | 621 | memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN); |
| 606 | max_data->pid = tsk->pid; | 622 | max_data->pid = tsk->pid; |
| 607 | max_data->uid = task_uid(tsk); | 623 | max_data->uid = task_uid(tsk); |
| 608 | max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO; | 624 | max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO; |
| @@ -824,10 +840,10 @@ out: | |||
| 824 | mutex_unlock(&trace_types_lock); | 840 | mutex_unlock(&trace_types_lock); |
| 825 | } | 841 | } |
| 826 | 842 | ||
| 827 | static void __tracing_reset(struct trace_array *tr, int cpu) | 843 | static void __tracing_reset(struct ring_buffer *buffer, int cpu) |
| 828 | { | 844 | { |
| 829 | ftrace_disable_cpu(); | 845 | ftrace_disable_cpu(); |
| 830 | ring_buffer_reset_cpu(tr->buffer, cpu); | 846 | ring_buffer_reset_cpu(buffer, cpu); |
| 831 | ftrace_enable_cpu(); | 847 | ftrace_enable_cpu(); |
| 832 | } | 848 | } |
| 833 | 849 | ||
| @@ -839,7 +855,7 @@ void tracing_reset(struct trace_array *tr, int cpu) | |||
| 839 | 855 | ||
| 840 | /* Make sure all commits have finished */ | 856 | /* Make sure all commits have finished */ |
| 841 | synchronize_sched(); | 857 | synchronize_sched(); |
| 842 | __tracing_reset(tr, cpu); | 858 | __tracing_reset(buffer, cpu); |
| 843 | 859 | ||
| 844 | ring_buffer_record_enable(buffer); | 860 | ring_buffer_record_enable(buffer); |
| 845 | } | 861 | } |
| @@ -857,7 +873,7 @@ void tracing_reset_online_cpus(struct trace_array *tr) | |||
| 857 | tr->time_start = ftrace_now(tr->cpu); | 873 | tr->time_start = ftrace_now(tr->cpu); |
| 858 | 874 | ||
| 859 | for_each_online_cpu(cpu) | 875 | for_each_online_cpu(cpu) |
| 860 | __tracing_reset(tr, cpu); | 876 | __tracing_reset(buffer, cpu); |
| 861 | 877 | ||
| 862 | ring_buffer_record_enable(buffer); | 878 | ring_buffer_record_enable(buffer); |
| 863 | } | 879 | } |
| @@ -934,6 +950,8 @@ void tracing_start(void) | |||
| 934 | goto out; | 950 | goto out; |
| 935 | } | 951 | } |
| 936 | 952 | ||
| 953 | /* Prevent the buffers from switching */ | ||
| 954 | arch_spin_lock(&ftrace_max_lock); | ||
| 937 | 955 | ||
| 938 | buffer = global_trace.buffer; | 956 | buffer = global_trace.buffer; |
| 939 | if (buffer) | 957 | if (buffer) |
| @@ -943,6 +961,8 @@ void tracing_start(void) | |||
| 943 | if (buffer) | 961 | if (buffer) |
| 944 | ring_buffer_record_enable(buffer); | 962 | ring_buffer_record_enable(buffer); |
| 945 | 963 | ||
| 964 | arch_spin_unlock(&ftrace_max_lock); | ||
| 965 | |||
| 946 | ftrace_start(); | 966 | ftrace_start(); |
| 947 | out: | 967 | out: |
| 948 | spin_unlock_irqrestore(&tracing_start_lock, flags); | 968 | spin_unlock_irqrestore(&tracing_start_lock, flags); |
| @@ -964,6 +984,9 @@ void tracing_stop(void) | |||
| 964 | if (trace_stop_count++) | 984 | if (trace_stop_count++) |
| 965 | goto out; | 985 | goto out; |
| 966 | 986 | ||
| 987 | /* Prevent the buffers from switching */ | ||
| 988 | arch_spin_lock(&ftrace_max_lock); | ||
| 989 | |||
| 967 | buffer = global_trace.buffer; | 990 | buffer = global_trace.buffer; |
| 968 | if (buffer) | 991 | if (buffer) |
| 969 | ring_buffer_record_disable(buffer); | 992 | ring_buffer_record_disable(buffer); |
| @@ -972,6 +995,8 @@ void tracing_stop(void) | |||
| 972 | if (buffer) | 995 | if (buffer) |
| 973 | ring_buffer_record_disable(buffer); | 996 | ring_buffer_record_disable(buffer); |
| 974 | 997 | ||
| 998 | arch_spin_unlock(&ftrace_max_lock); | ||
| 999 | |||
| 975 | out: | 1000 | out: |
| 976 | spin_unlock_irqrestore(&tracing_start_lock, flags); | 1001 | spin_unlock_irqrestore(&tracing_start_lock, flags); |
| 977 | } | 1002 | } |
| @@ -1166,7 +1191,7 @@ trace_function(struct trace_array *tr, | |||
| 1166 | struct ftrace_entry *entry; | 1191 | struct ftrace_entry *entry; |
| 1167 | 1192 | ||
| 1168 | /* If we are reading the ring buffer, don't trace */ | 1193 | /* If we are reading the ring buffer, don't trace */ |
| 1169 | if (unlikely(__this_cpu_read(per_cpu_var(ftrace_cpu_disabled)))) | 1194 | if (unlikely(__this_cpu_read(ftrace_cpu_disabled))) |
| 1170 | return; | 1195 | return; |
| 1171 | 1196 | ||
| 1172 | event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry), | 1197 | event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry), |
| @@ -1259,6 +1284,13 @@ ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc) | |||
| 1259 | if (!(trace_flags & TRACE_ITER_USERSTACKTRACE)) | 1284 | if (!(trace_flags & TRACE_ITER_USERSTACKTRACE)) |
| 1260 | return; | 1285 | return; |
| 1261 | 1286 | ||
| 1287 | /* | ||
| 1288 | * NMIs can not handle page faults, even with fix ups. | ||
| 1289 | * The save user stack can (and often does) fault. | ||
| 1290 | */ | ||
| 1291 | if (unlikely(in_nmi())) | ||
| 1292 | return; | ||
| 1293 | |||
| 1262 | event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK, | 1294 | event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK, |
| 1263 | sizeof(*entry), flags, pc); | 1295 | sizeof(*entry), flags, pc); |
| 1264 | if (!event) | 1296 | if (!event) |
| @@ -1703,6 +1735,7 @@ static void *s_start(struct seq_file *m, loff_t *pos) | |||
| 1703 | 1735 | ||
| 1704 | ftrace_enable_cpu(); | 1736 | ftrace_enable_cpu(); |
| 1705 | 1737 | ||
| 1738 | iter->leftover = 0; | ||
| 1706 | for (p = iter; p && l < *pos; p = s_next(m, p, &l)) | 1739 | for (p = iter; p && l < *pos; p = s_next(m, p, &l)) |
| 1707 | ; | 1740 | ; |
| 1708 | 1741 | ||
| @@ -4248,10 +4281,10 @@ static __init int tracer_init_debugfs(void) | |||
| 4248 | #ifdef CONFIG_TRACER_MAX_TRACE | 4281 | #ifdef CONFIG_TRACER_MAX_TRACE |
| 4249 | trace_create_file("tracing_max_latency", 0644, d_tracer, | 4282 | trace_create_file("tracing_max_latency", 0644, d_tracer, |
| 4250 | &tracing_max_latency, &tracing_max_lat_fops); | 4283 | &tracing_max_latency, &tracing_max_lat_fops); |
| 4284 | #endif | ||
| 4251 | 4285 | ||
| 4252 | trace_create_file("tracing_thresh", 0644, d_tracer, | 4286 | trace_create_file("tracing_thresh", 0644, d_tracer, |
| 4253 | &tracing_thresh, &tracing_max_lat_fops); | 4287 | &tracing_thresh, &tracing_max_lat_fops); |
| 4254 | #endif | ||
| 4255 | 4288 | ||
| 4256 | trace_create_file("README", 0444, d_tracer, | 4289 | trace_create_file("README", 0444, d_tracer, |
| 4257 | NULL, &tracing_readme_fops); | 4290 | NULL, &tracing_readme_fops); |
diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index fd05bcaf91b0..2825ef2c0b15 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h | |||
| @@ -396,9 +396,10 @@ extern int process_new_ksym_entry(char *ksymname, int op, unsigned long addr); | |||
| 396 | 396 | ||
| 397 | extern unsigned long nsecs_to_usecs(unsigned long nsecs); | 397 | extern unsigned long nsecs_to_usecs(unsigned long nsecs); |
| 398 | 398 | ||
| 399 | extern unsigned long tracing_thresh; | ||
| 400 | |||
| 399 | #ifdef CONFIG_TRACER_MAX_TRACE | 401 | #ifdef CONFIG_TRACER_MAX_TRACE |
| 400 | extern unsigned long tracing_max_latency; | 402 | extern unsigned long tracing_max_latency; |
| 401 | extern unsigned long tracing_thresh; | ||
| 402 | 403 | ||
| 403 | void update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu); | 404 | void update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu); |
| 404 | void update_max_tr_single(struct trace_array *tr, | 405 | void update_max_tr_single(struct trace_array *tr, |
| @@ -550,7 +551,7 @@ static inline int ftrace_trace_task(struct task_struct *task) | |||
| 550 | * struct trace_parser - servers for reading the user input separated by spaces | 551 | * struct trace_parser - servers for reading the user input separated by spaces |
| 551 | * @cont: set if the input is not complete - no final space char was found | 552 | * @cont: set if the input is not complete - no final space char was found |
| 552 | * @buffer: holds the parsed user input | 553 | * @buffer: holds the parsed user input |
| 553 | * @idx: user input lenght | 554 | * @idx: user input length |
| 554 | * @size: buffer size | 555 | * @size: buffer size |
| 555 | */ | 556 | */ |
| 556 | struct trace_parser { | 557 | struct trace_parser { |
diff --git a/kernel/trace/trace_clock.c b/kernel/trace/trace_clock.c index 84a3a7ba072a..6fbfb8f417b9 100644 --- a/kernel/trace/trace_clock.c +++ b/kernel/trace/trace_clock.c | |||
| @@ -13,6 +13,7 @@ | |||
| 13 | * Tracer plugins will chose a default from these clocks. | 13 | * Tracer plugins will chose a default from these clocks. |
| 14 | */ | 14 | */ |
| 15 | #include <linux/spinlock.h> | 15 | #include <linux/spinlock.h> |
| 16 | #include <linux/irqflags.h> | ||
| 16 | #include <linux/hardirq.h> | 17 | #include <linux/hardirq.h> |
| 17 | #include <linux/module.h> | 18 | #include <linux/module.h> |
| 18 | #include <linux/percpu.h> | 19 | #include <linux/percpu.h> |
diff --git a/kernel/trace/trace_event_profile.c b/kernel/trace/trace_event_perf.c index f0d693005075..81f691eb3a30 100644 --- a/kernel/trace/trace_event_profile.c +++ b/kernel/trace/trace_event_perf.c | |||
| @@ -1,32 +1,36 @@ | |||
| 1 | /* | 1 | /* |
| 2 | * trace event based perf counter profiling | 2 | * trace event based perf event profiling/tracing |
| 3 | * | 3 | * |
| 4 | * Copyright (C) 2009 Red Hat Inc, Peter Zijlstra <pzijlstr@redhat.com> | 4 | * Copyright (C) 2009 Red Hat Inc, Peter Zijlstra <pzijlstr@redhat.com> |
| 5 | * | 5 | * Copyright (C) 2009-2010 Frederic Weisbecker <fweisbec@gmail.com> |
| 6 | */ | 6 | */ |
| 7 | 7 | ||
| 8 | #include <linux/module.h> | 8 | #include <linux/module.h> |
| 9 | #include <linux/kprobes.h> | 9 | #include <linux/kprobes.h> |
| 10 | #include "trace.h" | 10 | #include "trace.h" |
| 11 | 11 | ||
| 12 | DEFINE_PER_CPU(struct pt_regs, perf_trace_regs); | ||
| 13 | EXPORT_PER_CPU_SYMBOL_GPL(perf_trace_regs); | ||
| 14 | |||
| 15 | EXPORT_SYMBOL_GPL(perf_arch_fetch_caller_regs); | ||
| 12 | 16 | ||
| 13 | static char *perf_trace_buf; | 17 | static char *perf_trace_buf; |
| 14 | static char *perf_trace_buf_nmi; | 18 | static char *perf_trace_buf_nmi; |
| 15 | 19 | ||
| 16 | typedef typeof(char [FTRACE_MAX_PROFILE_SIZE]) perf_trace_t ; | 20 | typedef typeof(char [PERF_MAX_TRACE_SIZE]) perf_trace_t ; |
| 17 | 21 | ||
| 18 | /* Count the events in use (per event id, not per instance) */ | 22 | /* Count the events in use (per event id, not per instance) */ |
| 19 | static int total_profile_count; | 23 | static int total_ref_count; |
| 20 | 24 | ||
| 21 | static int ftrace_profile_enable_event(struct ftrace_event_call *event) | 25 | static int perf_trace_event_enable(struct ftrace_event_call *event) |
| 22 | { | 26 | { |
| 23 | char *buf; | 27 | char *buf; |
| 24 | int ret = -ENOMEM; | 28 | int ret = -ENOMEM; |
| 25 | 29 | ||
| 26 | if (event->profile_count++ > 0) | 30 | if (event->perf_refcount++ > 0) |
| 27 | return 0; | 31 | return 0; |
| 28 | 32 | ||
| 29 | if (!total_profile_count) { | 33 | if (!total_ref_count) { |
| 30 | buf = (char *)alloc_percpu(perf_trace_t); | 34 | buf = (char *)alloc_percpu(perf_trace_t); |
| 31 | if (!buf) | 35 | if (!buf) |
| 32 | goto fail_buf; | 36 | goto fail_buf; |
| @@ -40,35 +44,35 @@ static int ftrace_profile_enable_event(struct ftrace_event_call *event) | |||
| 40 | rcu_assign_pointer(perf_trace_buf_nmi, buf); | 44 | rcu_assign_pointer(perf_trace_buf_nmi, buf); |
| 41 | } | 45 | } |
| 42 | 46 | ||
| 43 | ret = event->profile_enable(event); | 47 | ret = event->perf_event_enable(event); |
| 44 | if (!ret) { | 48 | if (!ret) { |
| 45 | total_profile_count++; | 49 | total_ref_count++; |
| 46 | return 0; | 50 | return 0; |
| 47 | } | 51 | } |
| 48 | 52 | ||
| 49 | fail_buf_nmi: | 53 | fail_buf_nmi: |
| 50 | if (!total_profile_count) { | 54 | if (!total_ref_count) { |
| 51 | free_percpu(perf_trace_buf_nmi); | 55 | free_percpu(perf_trace_buf_nmi); |
| 52 | free_percpu(perf_trace_buf); | 56 | free_percpu(perf_trace_buf); |
| 53 | perf_trace_buf_nmi = NULL; | 57 | perf_trace_buf_nmi = NULL; |
| 54 | perf_trace_buf = NULL; | 58 | perf_trace_buf = NULL; |
| 55 | } | 59 | } |
| 56 | fail_buf: | 60 | fail_buf: |
| 57 | event->profile_count--; | 61 | event->perf_refcount--; |
| 58 | 62 | ||
| 59 | return ret; | 63 | return ret; |
| 60 | } | 64 | } |
| 61 | 65 | ||
| 62 | int ftrace_profile_enable(int event_id) | 66 | int perf_trace_enable(int event_id) |
| 63 | { | 67 | { |
| 64 | struct ftrace_event_call *event; | 68 | struct ftrace_event_call *event; |
| 65 | int ret = -EINVAL; | 69 | int ret = -EINVAL; |
| 66 | 70 | ||
| 67 | mutex_lock(&event_mutex); | 71 | mutex_lock(&event_mutex); |
| 68 | list_for_each_entry(event, &ftrace_events, list) { | 72 | list_for_each_entry(event, &ftrace_events, list) { |
| 69 | if (event->id == event_id && event->profile_enable && | 73 | if (event->id == event_id && event->perf_event_enable && |
| 70 | try_module_get(event->mod)) { | 74 | try_module_get(event->mod)) { |
| 71 | ret = ftrace_profile_enable_event(event); | 75 | ret = perf_trace_event_enable(event); |
| 72 | break; | 76 | break; |
| 73 | } | 77 | } |
| 74 | } | 78 | } |
| @@ -77,16 +81,16 @@ int ftrace_profile_enable(int event_id) | |||
| 77 | return ret; | 81 | return ret; |
| 78 | } | 82 | } |
| 79 | 83 | ||
| 80 | static void ftrace_profile_disable_event(struct ftrace_event_call *event) | 84 | static void perf_trace_event_disable(struct ftrace_event_call *event) |
| 81 | { | 85 | { |
| 82 | char *buf, *nmi_buf; | 86 | char *buf, *nmi_buf; |
| 83 | 87 | ||
| 84 | if (--event->profile_count > 0) | 88 | if (--event->perf_refcount > 0) |
| 85 | return; | 89 | return; |
| 86 | 90 | ||
| 87 | event->profile_disable(event); | 91 | event->perf_event_disable(event); |
| 88 | 92 | ||
| 89 | if (!--total_profile_count) { | 93 | if (!--total_ref_count) { |
| 90 | buf = perf_trace_buf; | 94 | buf = perf_trace_buf; |
| 91 | rcu_assign_pointer(perf_trace_buf, NULL); | 95 | rcu_assign_pointer(perf_trace_buf, NULL); |
| 92 | 96 | ||
| @@ -104,14 +108,14 @@ static void ftrace_profile_disable_event(struct ftrace_event_call *event) | |||
| 104 | } | 108 | } |
| 105 | } | 109 | } |
| 106 | 110 | ||
| 107 | void ftrace_profile_disable(int event_id) | 111 | void perf_trace_disable(int event_id) |
| 108 | { | 112 | { |
| 109 | struct ftrace_event_call *event; | 113 | struct ftrace_event_call *event; |
| 110 | 114 | ||
| 111 | mutex_lock(&event_mutex); | 115 | mutex_lock(&event_mutex); |
| 112 | list_for_each_entry(event, &ftrace_events, list) { | 116 | list_for_each_entry(event, &ftrace_events, list) { |
| 113 | if (event->id == event_id) { | 117 | if (event->id == event_id) { |
| 114 | ftrace_profile_disable_event(event); | 118 | perf_trace_event_disable(event); |
| 115 | module_put(event->mod); | 119 | module_put(event->mod); |
| 116 | break; | 120 | break; |
| 117 | } | 121 | } |
| @@ -119,8 +123,8 @@ void ftrace_profile_disable(int event_id) | |||
| 119 | mutex_unlock(&event_mutex); | 123 | mutex_unlock(&event_mutex); |
| 120 | } | 124 | } |
| 121 | 125 | ||
| 122 | __kprobes void *ftrace_perf_buf_prepare(int size, unsigned short type, | 126 | __kprobes void *perf_trace_buf_prepare(int size, unsigned short type, |
| 123 | int *rctxp, unsigned long *irq_flags) | 127 | int *rctxp, unsigned long *irq_flags) |
| 124 | { | 128 | { |
| 125 | struct trace_entry *entry; | 129 | struct trace_entry *entry; |
| 126 | char *trace_buf, *raw_data; | 130 | char *trace_buf, *raw_data; |
| @@ -138,9 +142,9 @@ __kprobes void *ftrace_perf_buf_prepare(int size, unsigned short type, | |||
| 138 | cpu = smp_processor_id(); | 142 | cpu = smp_processor_id(); |
| 139 | 143 | ||
| 140 | if (in_nmi()) | 144 | if (in_nmi()) |
| 141 | trace_buf = rcu_dereference(perf_trace_buf_nmi); | 145 | trace_buf = rcu_dereference_sched(perf_trace_buf_nmi); |
| 142 | else | 146 | else |
| 143 | trace_buf = rcu_dereference(perf_trace_buf); | 147 | trace_buf = rcu_dereference_sched(perf_trace_buf); |
| 144 | 148 | ||
| 145 | if (!trace_buf) | 149 | if (!trace_buf) |
| 146 | goto err; | 150 | goto err; |
| @@ -161,4 +165,4 @@ err_recursion: | |||
| 161 | local_irq_restore(*irq_flags); | 165 | local_irq_restore(*irq_flags); |
| 162 | return NULL; | 166 | return NULL; |
| 163 | } | 167 | } |
| 164 | EXPORT_SYMBOL_GPL(ftrace_perf_buf_prepare); | 168 | EXPORT_SYMBOL_GPL(perf_trace_buf_prepare); |
diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c index 3f972ad98d04..beab8bf2f310 100644 --- a/kernel/trace/trace_events.c +++ b/kernel/trace/trace_events.c | |||
| @@ -938,7 +938,7 @@ event_create_dir(struct ftrace_event_call *call, struct dentry *d_events, | |||
| 938 | trace_create_file("enable", 0644, call->dir, call, | 938 | trace_create_file("enable", 0644, call->dir, call, |
| 939 | enable); | 939 | enable); |
| 940 | 940 | ||
| 941 | if (call->id && call->profile_enable) | 941 | if (call->id && call->perf_event_enable) |
| 942 | trace_create_file("id", 0444, call->dir, call, | 942 | trace_create_file("id", 0444, call->dir, call, |
| 943 | id); | 943 | id); |
| 944 | 944 | ||
diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c index e998a824e9db..e6989d9b44da 100644 --- a/kernel/trace/trace_functions_graph.c +++ b/kernel/trace/trace_functions_graph.c | |||
| @@ -188,7 +188,7 @@ static int __trace_graph_entry(struct trace_array *tr, | |||
| 188 | struct ring_buffer *buffer = tr->buffer; | 188 | struct ring_buffer *buffer = tr->buffer; |
| 189 | struct ftrace_graph_ent_entry *entry; | 189 | struct ftrace_graph_ent_entry *entry; |
| 190 | 190 | ||
| 191 | if (unlikely(__this_cpu_read(per_cpu_var(ftrace_cpu_disabled)))) | 191 | if (unlikely(__this_cpu_read(ftrace_cpu_disabled))) |
| 192 | return 0; | 192 | return 0; |
| 193 | 193 | ||
| 194 | event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_ENT, | 194 | event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_ENT, |
| @@ -237,6 +237,14 @@ int trace_graph_entry(struct ftrace_graph_ent *trace) | |||
| 237 | return ret; | 237 | return ret; |
| 238 | } | 238 | } |
| 239 | 239 | ||
| 240 | int trace_graph_thresh_entry(struct ftrace_graph_ent *trace) | ||
| 241 | { | ||
| 242 | if (tracing_thresh) | ||
| 243 | return 1; | ||
| 244 | else | ||
| 245 | return trace_graph_entry(trace); | ||
| 246 | } | ||
| 247 | |||
| 240 | static void __trace_graph_return(struct trace_array *tr, | 248 | static void __trace_graph_return(struct trace_array *tr, |
| 241 | struct ftrace_graph_ret *trace, | 249 | struct ftrace_graph_ret *trace, |
| 242 | unsigned long flags, | 250 | unsigned long flags, |
| @@ -247,7 +255,7 @@ static void __trace_graph_return(struct trace_array *tr, | |||
| 247 | struct ring_buffer *buffer = tr->buffer; | 255 | struct ring_buffer *buffer = tr->buffer; |
| 248 | struct ftrace_graph_ret_entry *entry; | 256 | struct ftrace_graph_ret_entry *entry; |
| 249 | 257 | ||
| 250 | if (unlikely(__this_cpu_read(per_cpu_var(ftrace_cpu_disabled)))) | 258 | if (unlikely(__this_cpu_read(ftrace_cpu_disabled))) |
| 251 | return; | 259 | return; |
| 252 | 260 | ||
| 253 | event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_RET, | 261 | event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_RET, |
| @@ -290,13 +298,26 @@ void set_graph_array(struct trace_array *tr) | |||
| 290 | smp_mb(); | 298 | smp_mb(); |
| 291 | } | 299 | } |
| 292 | 300 | ||
| 301 | void trace_graph_thresh_return(struct ftrace_graph_ret *trace) | ||
| 302 | { | ||
| 303 | if (tracing_thresh && | ||
| 304 | (trace->rettime - trace->calltime < tracing_thresh)) | ||
| 305 | return; | ||
| 306 | else | ||
| 307 | trace_graph_return(trace); | ||
| 308 | } | ||
| 309 | |||
| 293 | static int graph_trace_init(struct trace_array *tr) | 310 | static int graph_trace_init(struct trace_array *tr) |
| 294 | { | 311 | { |
| 295 | int ret; | 312 | int ret; |
| 296 | 313 | ||
| 297 | set_graph_array(tr); | 314 | set_graph_array(tr); |
| 298 | ret = register_ftrace_graph(&trace_graph_return, | 315 | if (tracing_thresh) |
| 299 | &trace_graph_entry); | 316 | ret = register_ftrace_graph(&trace_graph_thresh_return, |
| 317 | &trace_graph_thresh_entry); | ||
| 318 | else | ||
| 319 | ret = register_ftrace_graph(&trace_graph_return, | ||
| 320 | &trace_graph_entry); | ||
| 300 | if (ret) | 321 | if (ret) |
| 301 | return ret; | 322 | return ret; |
| 302 | tracing_start_cmdline_record(); | 323 | tracing_start_cmdline_record(); |
| @@ -920,7 +941,7 @@ print_graph_return(struct ftrace_graph_ret *trace, struct trace_seq *s, | |||
| 920 | if (!ret) | 941 | if (!ret) |
| 921 | return TRACE_TYPE_PARTIAL_LINE; | 942 | return TRACE_TYPE_PARTIAL_LINE; |
| 922 | } else { | 943 | } else { |
| 923 | ret = trace_seq_printf(s, "} (%ps)\n", (void *)trace->func); | 944 | ret = trace_seq_printf(s, "} /* %ps */\n", (void *)trace->func); |
| 924 | if (!ret) | 945 | if (!ret) |
| 925 | return TRACE_TYPE_PARTIAL_LINE; | 946 | return TRACE_TYPE_PARTIAL_LINE; |
| 926 | } | 947 | } |
diff --git a/kernel/trace/trace_kprobe.c b/kernel/trace/trace_kprobe.c index 505c92273b1a..1251e367bae9 100644 --- a/kernel/trace/trace_kprobe.c +++ b/kernel/trace/trace_kprobe.c | |||
| @@ -1214,7 +1214,7 @@ static int set_print_fmt(struct trace_probe *tp) | |||
| 1214 | #ifdef CONFIG_PERF_EVENTS | 1214 | #ifdef CONFIG_PERF_EVENTS |
| 1215 | 1215 | ||
| 1216 | /* Kprobe profile handler */ | 1216 | /* Kprobe profile handler */ |
| 1217 | static __kprobes void kprobe_profile_func(struct kprobe *kp, | 1217 | static __kprobes void kprobe_perf_func(struct kprobe *kp, |
| 1218 | struct pt_regs *regs) | 1218 | struct pt_regs *regs) |
| 1219 | { | 1219 | { |
| 1220 | struct trace_probe *tp = container_of(kp, struct trace_probe, rp.kp); | 1220 | struct trace_probe *tp = container_of(kp, struct trace_probe, rp.kp); |
| @@ -1227,11 +1227,11 @@ static __kprobes void kprobe_profile_func(struct kprobe *kp, | |||
| 1227 | __size = SIZEOF_KPROBE_TRACE_ENTRY(tp->nr_args); | 1227 | __size = SIZEOF_KPROBE_TRACE_ENTRY(tp->nr_args); |
| 1228 | size = ALIGN(__size + sizeof(u32), sizeof(u64)); | 1228 | size = ALIGN(__size + sizeof(u32), sizeof(u64)); |
| 1229 | size -= sizeof(u32); | 1229 | size -= sizeof(u32); |
| 1230 | if (WARN_ONCE(size > FTRACE_MAX_PROFILE_SIZE, | 1230 | if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE, |
| 1231 | "profile buffer not large enough")) | 1231 | "profile buffer not large enough")) |
| 1232 | return; | 1232 | return; |
| 1233 | 1233 | ||
| 1234 | entry = ftrace_perf_buf_prepare(size, call->id, &rctx, &irq_flags); | 1234 | entry = perf_trace_buf_prepare(size, call->id, &rctx, &irq_flags); |
| 1235 | if (!entry) | 1235 | if (!entry) |
| 1236 | return; | 1236 | return; |
| 1237 | 1237 | ||
| @@ -1240,11 +1240,11 @@ static __kprobes void kprobe_profile_func(struct kprobe *kp, | |||
| 1240 | for (i = 0; i < tp->nr_args; i++) | 1240 | for (i = 0; i < tp->nr_args; i++) |
| 1241 | entry->args[i] = call_fetch(&tp->args[i].fetch, regs); | 1241 | entry->args[i] = call_fetch(&tp->args[i].fetch, regs); |
| 1242 | 1242 | ||
| 1243 | ftrace_perf_buf_submit(entry, size, rctx, entry->ip, 1, irq_flags); | 1243 | perf_trace_buf_submit(entry, size, rctx, entry->ip, 1, irq_flags, regs); |
| 1244 | } | 1244 | } |
| 1245 | 1245 | ||
| 1246 | /* Kretprobe profile handler */ | 1246 | /* Kretprobe profile handler */ |
| 1247 | static __kprobes void kretprobe_profile_func(struct kretprobe_instance *ri, | 1247 | static __kprobes void kretprobe_perf_func(struct kretprobe_instance *ri, |
| 1248 | struct pt_regs *regs) | 1248 | struct pt_regs *regs) |
| 1249 | { | 1249 | { |
| 1250 | struct trace_probe *tp = container_of(ri->rp, struct trace_probe, rp); | 1250 | struct trace_probe *tp = container_of(ri->rp, struct trace_probe, rp); |
| @@ -1257,11 +1257,11 @@ static __kprobes void kretprobe_profile_func(struct kretprobe_instance *ri, | |||
| 1257 | __size = SIZEOF_KRETPROBE_TRACE_ENTRY(tp->nr_args); | 1257 | __size = SIZEOF_KRETPROBE_TRACE_ENTRY(tp->nr_args); |
| 1258 | size = ALIGN(__size + sizeof(u32), sizeof(u64)); | 1258 | size = ALIGN(__size + sizeof(u32), sizeof(u64)); |
| 1259 | size -= sizeof(u32); | 1259 | size -= sizeof(u32); |
| 1260 | if (WARN_ONCE(size > FTRACE_MAX_PROFILE_SIZE, | 1260 | if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE, |
| 1261 | "profile buffer not large enough")) | 1261 | "profile buffer not large enough")) |
| 1262 | return; | 1262 | return; |
| 1263 | 1263 | ||
| 1264 | entry = ftrace_perf_buf_prepare(size, call->id, &rctx, &irq_flags); | 1264 | entry = perf_trace_buf_prepare(size, call->id, &rctx, &irq_flags); |
| 1265 | if (!entry) | 1265 | if (!entry) |
| 1266 | return; | 1266 | return; |
| 1267 | 1267 | ||
| @@ -1271,10 +1271,11 @@ static __kprobes void kretprobe_profile_func(struct kretprobe_instance *ri, | |||
| 1271 | for (i = 0; i < tp->nr_args; i++) | 1271 | for (i = 0; i < tp->nr_args; i++) |
| 1272 | entry->args[i] = call_fetch(&tp->args[i].fetch, regs); | 1272 | entry->args[i] = call_fetch(&tp->args[i].fetch, regs); |
| 1273 | 1273 | ||
| 1274 | ftrace_perf_buf_submit(entry, size, rctx, entry->ret_ip, 1, irq_flags); | 1274 | perf_trace_buf_submit(entry, size, rctx, entry->ret_ip, 1, |
| 1275 | irq_flags, regs); | ||
| 1275 | } | 1276 | } |
| 1276 | 1277 | ||
| 1277 | static int probe_profile_enable(struct ftrace_event_call *call) | 1278 | static int probe_perf_enable(struct ftrace_event_call *call) |
| 1278 | { | 1279 | { |
| 1279 | struct trace_probe *tp = (struct trace_probe *)call->data; | 1280 | struct trace_probe *tp = (struct trace_probe *)call->data; |
| 1280 | 1281 | ||
| @@ -1286,7 +1287,7 @@ static int probe_profile_enable(struct ftrace_event_call *call) | |||
| 1286 | return enable_kprobe(&tp->rp.kp); | 1287 | return enable_kprobe(&tp->rp.kp); |
| 1287 | } | 1288 | } |
| 1288 | 1289 | ||
| 1289 | static void probe_profile_disable(struct ftrace_event_call *call) | 1290 | static void probe_perf_disable(struct ftrace_event_call *call) |
| 1290 | { | 1291 | { |
| 1291 | struct trace_probe *tp = (struct trace_probe *)call->data; | 1292 | struct trace_probe *tp = (struct trace_probe *)call->data; |
| 1292 | 1293 | ||
| @@ -1311,7 +1312,7 @@ int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs) | |||
| 1311 | kprobe_trace_func(kp, regs); | 1312 | kprobe_trace_func(kp, regs); |
| 1312 | #ifdef CONFIG_PERF_EVENTS | 1313 | #ifdef CONFIG_PERF_EVENTS |
| 1313 | if (tp->flags & TP_FLAG_PROFILE) | 1314 | if (tp->flags & TP_FLAG_PROFILE) |
| 1314 | kprobe_profile_func(kp, regs); | 1315 | kprobe_perf_func(kp, regs); |
| 1315 | #endif | 1316 | #endif |
| 1316 | return 0; /* We don't tweek kernel, so just return 0 */ | 1317 | return 0; /* We don't tweek kernel, so just return 0 */ |
| 1317 | } | 1318 | } |
| @@ -1325,7 +1326,7 @@ int kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs) | |||
| 1325 | kretprobe_trace_func(ri, regs); | 1326 | kretprobe_trace_func(ri, regs); |
| 1326 | #ifdef CONFIG_PERF_EVENTS | 1327 | #ifdef CONFIG_PERF_EVENTS |
| 1327 | if (tp->flags & TP_FLAG_PROFILE) | 1328 | if (tp->flags & TP_FLAG_PROFILE) |
| 1328 | kretprobe_profile_func(ri, regs); | 1329 | kretprobe_perf_func(ri, regs); |
| 1329 | #endif | 1330 | #endif |
| 1330 | return 0; /* We don't tweek kernel, so just return 0 */ | 1331 | return 0; /* We don't tweek kernel, so just return 0 */ |
| 1331 | } | 1332 | } |
| @@ -1358,8 +1359,8 @@ static int register_probe_event(struct trace_probe *tp) | |||
| 1358 | call->unregfunc = probe_event_disable; | 1359 | call->unregfunc = probe_event_disable; |
| 1359 | 1360 | ||
| 1360 | #ifdef CONFIG_PERF_EVENTS | 1361 | #ifdef CONFIG_PERF_EVENTS |
| 1361 | call->profile_enable = probe_profile_enable; | 1362 | call->perf_event_enable = probe_perf_enable; |
| 1362 | call->profile_disable = probe_profile_disable; | 1363 | call->perf_event_disable = probe_perf_disable; |
| 1363 | #endif | 1364 | #endif |
| 1364 | call->data = tp; | 1365 | call->data = tp; |
| 1365 | ret = trace_add_event_call(call); | 1366 | ret = trace_add_event_call(call); |
diff --git a/kernel/trace/trace_syscalls.c b/kernel/trace/trace_syscalls.c index cba47d7935cc..33c2a5b769dc 100644 --- a/kernel/trace/trace_syscalls.c +++ b/kernel/trace/trace_syscalls.c | |||
| @@ -428,12 +428,12 @@ core_initcall(init_ftrace_syscalls); | |||
| 428 | 428 | ||
| 429 | #ifdef CONFIG_PERF_EVENTS | 429 | #ifdef CONFIG_PERF_EVENTS |
| 430 | 430 | ||
| 431 | static DECLARE_BITMAP(enabled_prof_enter_syscalls, NR_syscalls); | 431 | static DECLARE_BITMAP(enabled_perf_enter_syscalls, NR_syscalls); |
| 432 | static DECLARE_BITMAP(enabled_prof_exit_syscalls, NR_syscalls); | 432 | static DECLARE_BITMAP(enabled_perf_exit_syscalls, NR_syscalls); |
| 433 | static int sys_prof_refcount_enter; | 433 | static int sys_perf_refcount_enter; |
| 434 | static int sys_prof_refcount_exit; | 434 | static int sys_perf_refcount_exit; |
| 435 | 435 | ||
| 436 | static void prof_syscall_enter(struct pt_regs *regs, long id) | 436 | static void perf_syscall_enter(struct pt_regs *regs, long id) |
| 437 | { | 437 | { |
| 438 | struct syscall_metadata *sys_data; | 438 | struct syscall_metadata *sys_data; |
| 439 | struct syscall_trace_enter *rec; | 439 | struct syscall_trace_enter *rec; |
| @@ -443,7 +443,7 @@ static void prof_syscall_enter(struct pt_regs *regs, long id) | |||
| 443 | int size; | 443 | int size; |
| 444 | 444 | ||
| 445 | syscall_nr = syscall_get_nr(current, regs); | 445 | syscall_nr = syscall_get_nr(current, regs); |
| 446 | if (!test_bit(syscall_nr, enabled_prof_enter_syscalls)) | 446 | if (!test_bit(syscall_nr, enabled_perf_enter_syscalls)) |
| 447 | return; | 447 | return; |
| 448 | 448 | ||
| 449 | sys_data = syscall_nr_to_meta(syscall_nr); | 449 | sys_data = syscall_nr_to_meta(syscall_nr); |
| @@ -455,11 +455,11 @@ static void prof_syscall_enter(struct pt_regs *regs, long id) | |||
| 455 | size = ALIGN(size + sizeof(u32), sizeof(u64)); | 455 | size = ALIGN(size + sizeof(u32), sizeof(u64)); |
| 456 | size -= sizeof(u32); | 456 | size -= sizeof(u32); |
| 457 | 457 | ||
| 458 | if (WARN_ONCE(size > FTRACE_MAX_PROFILE_SIZE, | 458 | if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE, |
| 459 | "profile buffer not large enough")) | 459 | "perf buffer not large enough")) |
| 460 | return; | 460 | return; |
| 461 | 461 | ||
| 462 | rec = (struct syscall_trace_enter *)ftrace_perf_buf_prepare(size, | 462 | rec = (struct syscall_trace_enter *)perf_trace_buf_prepare(size, |
| 463 | sys_data->enter_event->id, &rctx, &flags); | 463 | sys_data->enter_event->id, &rctx, &flags); |
| 464 | if (!rec) | 464 | if (!rec) |
| 465 | return; | 465 | return; |
| @@ -467,10 +467,10 @@ static void prof_syscall_enter(struct pt_regs *regs, long id) | |||
| 467 | rec->nr = syscall_nr; | 467 | rec->nr = syscall_nr; |
| 468 | syscall_get_arguments(current, regs, 0, sys_data->nb_args, | 468 | syscall_get_arguments(current, regs, 0, sys_data->nb_args, |
| 469 | (unsigned long *)&rec->args); | 469 | (unsigned long *)&rec->args); |
| 470 | ftrace_perf_buf_submit(rec, size, rctx, 0, 1, flags); | 470 | perf_trace_buf_submit(rec, size, rctx, 0, 1, flags, regs); |
| 471 | } | 471 | } |
| 472 | 472 | ||
| 473 | int prof_sysenter_enable(struct ftrace_event_call *call) | 473 | int perf_sysenter_enable(struct ftrace_event_call *call) |
| 474 | { | 474 | { |
| 475 | int ret = 0; | 475 | int ret = 0; |
| 476 | int num; | 476 | int num; |
| @@ -478,34 +478,34 @@ int prof_sysenter_enable(struct ftrace_event_call *call) | |||
| 478 | num = ((struct syscall_metadata *)call->data)->syscall_nr; | 478 | num = ((struct syscall_metadata *)call->data)->syscall_nr; |
| 479 | 479 | ||
| 480 | mutex_lock(&syscall_trace_lock); | 480 | mutex_lock(&syscall_trace_lock); |
| 481 | if (!sys_prof_refcount_enter) | 481 | if (!sys_perf_refcount_enter) |
| 482 | ret = register_trace_sys_enter(prof_syscall_enter); | 482 | ret = register_trace_sys_enter(perf_syscall_enter); |
| 483 | if (ret) { | 483 | if (ret) { |
| 484 | pr_info("event trace: Could not activate" | 484 | pr_info("event trace: Could not activate" |
| 485 | "syscall entry trace point"); | 485 | "syscall entry trace point"); |
| 486 | } else { | 486 | } else { |
| 487 | set_bit(num, enabled_prof_enter_syscalls); | 487 | set_bit(num, enabled_perf_enter_syscalls); |
| 488 | sys_prof_refcount_enter++; | 488 | sys_perf_refcount_enter++; |
| 489 | } | 489 | } |
| 490 | mutex_unlock(&syscall_trace_lock); | 490 | mutex_unlock(&syscall_trace_lock); |
| 491 | return ret; | 491 | return ret; |
| 492 | } | 492 | } |
| 493 | 493 | ||
| 494 | void prof_sysenter_disable(struct ftrace_event_call *call) | 494 | void perf_sysenter_disable(struct ftrace_event_call *call) |
| 495 | { | 495 | { |
| 496 | int num; | 496 | int num; |
| 497 | 497 | ||
| 498 | num = ((struct syscall_metadata *)call->data)->syscall_nr; | 498 | num = ((struct syscall_metadata *)call->data)->syscall_nr; |
| 499 | 499 | ||
| 500 | mutex_lock(&syscall_trace_lock); | 500 | mutex_lock(&syscall_trace_lock); |
| 501 | sys_prof_refcount_enter--; | 501 | sys_perf_refcount_enter--; |
| 502 | clear_bit(num, enabled_prof_enter_syscalls); | 502 | clear_bit(num, enabled_perf_enter_syscalls); |
| 503 | if (!sys_prof_refcount_enter) | 503 | if (!sys_perf_refcount_enter) |
| 504 | unregister_trace_sys_enter(prof_syscall_enter); | 504 | unregister_trace_sys_enter(perf_syscall_enter); |
| 505 | mutex_unlock(&syscall_trace_lock); | 505 | mutex_unlock(&syscall_trace_lock); |
| 506 | } | 506 | } |
| 507 | 507 | ||
| 508 | static void prof_syscall_exit(struct pt_regs *regs, long ret) | 508 | static void perf_syscall_exit(struct pt_regs *regs, long ret) |
| 509 | { | 509 | { |
| 510 | struct syscall_metadata *sys_data; | 510 | struct syscall_metadata *sys_data; |
| 511 | struct syscall_trace_exit *rec; | 511 | struct syscall_trace_exit *rec; |
| @@ -515,7 +515,7 @@ static void prof_syscall_exit(struct pt_regs *regs, long ret) | |||
| 515 | int size; | 515 | int size; |
| 516 | 516 | ||
| 517 | syscall_nr = syscall_get_nr(current, regs); | 517 | syscall_nr = syscall_get_nr(current, regs); |
| 518 | if (!test_bit(syscall_nr, enabled_prof_exit_syscalls)) | 518 | if (!test_bit(syscall_nr, enabled_perf_exit_syscalls)) |
| 519 | return; | 519 | return; |
| 520 | 520 | ||
| 521 | sys_data = syscall_nr_to_meta(syscall_nr); | 521 | sys_data = syscall_nr_to_meta(syscall_nr); |
| @@ -530,11 +530,11 @@ static void prof_syscall_exit(struct pt_regs *regs, long ret) | |||
| 530 | * Impossible, but be paranoid with the future | 530 | * Impossible, but be paranoid with the future |
| 531 | * How to put this check outside runtime? | 531 | * How to put this check outside runtime? |
| 532 | */ | 532 | */ |
| 533 | if (WARN_ONCE(size > FTRACE_MAX_PROFILE_SIZE, | 533 | if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE, |
| 534 | "exit event has grown above profile buffer size")) | 534 | "exit event has grown above perf buffer size")) |
| 535 | return; | 535 | return; |
| 536 | 536 | ||
| 537 | rec = (struct syscall_trace_exit *)ftrace_perf_buf_prepare(size, | 537 | rec = (struct syscall_trace_exit *)perf_trace_buf_prepare(size, |
| 538 | sys_data->exit_event->id, &rctx, &flags); | 538 | sys_data->exit_event->id, &rctx, &flags); |
| 539 | if (!rec) | 539 | if (!rec) |
| 540 | return; | 540 | return; |
| @@ -542,10 +542,10 @@ static void prof_syscall_exit(struct pt_regs *regs, long ret) | |||
| 542 | rec->nr = syscall_nr; | 542 | rec->nr = syscall_nr; |
| 543 | rec->ret = syscall_get_return_value(current, regs); | 543 | rec->ret = syscall_get_return_value(current, regs); |
| 544 | 544 | ||
| 545 | ftrace_perf_buf_submit(rec, size, rctx, 0, 1, flags); | 545 | perf_trace_buf_submit(rec, size, rctx, 0, 1, flags, regs); |
| 546 | } | 546 | } |
| 547 | 547 | ||
| 548 | int prof_sysexit_enable(struct ftrace_event_call *call) | 548 | int perf_sysexit_enable(struct ftrace_event_call *call) |
| 549 | { | 549 | { |
| 550 | int ret = 0; | 550 | int ret = 0; |
| 551 | int num; | 551 | int num; |
| @@ -553,30 +553,30 @@ int prof_sysexit_enable(struct ftrace_event_call *call) | |||
| 553 | num = ((struct syscall_metadata *)call->data)->syscall_nr; | 553 | num = ((struct syscall_metadata *)call->data)->syscall_nr; |
| 554 | 554 | ||
| 555 | mutex_lock(&syscall_trace_lock); | 555 | mutex_lock(&syscall_trace_lock); |
| 556 | if (!sys_prof_refcount_exit) | 556 | if (!sys_perf_refcount_exit) |
| 557 | ret = register_trace_sys_exit(prof_syscall_exit); | 557 | ret = register_trace_sys_exit(perf_syscall_exit); |
| 558 | if (ret) { | 558 | if (ret) { |
| 559 | pr_info("event trace: Could not activate" | 559 | pr_info("event trace: Could not activate" |
| 560 | "syscall exit trace point"); | 560 | "syscall exit trace point"); |
| 561 | } else { | 561 | } else { |
| 562 | set_bit(num, enabled_prof_exit_syscalls); | 562 | set_bit(num, enabled_perf_exit_syscalls); |
| 563 | sys_prof_refcount_exit++; | 563 | sys_perf_refcount_exit++; |
| 564 | } | 564 | } |
| 565 | mutex_unlock(&syscall_trace_lock); | 565 | mutex_unlock(&syscall_trace_lock); |
| 566 | return ret; | 566 | return ret; |
| 567 | } | 567 | } |
| 568 | 568 | ||
| 569 | void prof_sysexit_disable(struct ftrace_event_call *call) | 569 | void perf_sysexit_disable(struct ftrace_event_call *call) |
| 570 | { | 570 | { |
| 571 | int num; | 571 | int num; |
| 572 | 572 | ||
| 573 | num = ((struct syscall_metadata *)call->data)->syscall_nr; | 573 | num = ((struct syscall_metadata *)call->data)->syscall_nr; |
| 574 | 574 | ||
| 575 | mutex_lock(&syscall_trace_lock); | 575 | mutex_lock(&syscall_trace_lock); |
| 576 | sys_prof_refcount_exit--; | 576 | sys_perf_refcount_exit--; |
| 577 | clear_bit(num, enabled_prof_exit_syscalls); | 577 | clear_bit(num, enabled_perf_exit_syscalls); |
| 578 | if (!sys_prof_refcount_exit) | 578 | if (!sys_perf_refcount_exit) |
| 579 | unregister_trace_sys_exit(prof_syscall_exit); | 579 | unregister_trace_sys_exit(perf_syscall_exit); |
| 580 | mutex_unlock(&syscall_trace_lock); | 580 | mutex_unlock(&syscall_trace_lock); |
| 581 | } | 581 | } |
| 582 | 582 | ||
