diff options
author | David S. Miller <davem@davemloft.net> | 2010-02-28 22:23:06 -0500 |
---|---|---|
committer | David S. Miller <davem@davemloft.net> | 2010-02-28 22:23:06 -0500 |
commit | 47871889c601d8199c51a4086f77eebd77c29b0b (patch) | |
tree | 40cdcac3bff0ee40cc33dcca61d0577cdf965f77 /kernel/trace | |
parent | c16cc0b464b8876cfd57ce1c1dbcb6f9a6a0bce3 (diff) | |
parent | 30ff056c42c665b9ea535d8515890857ae382540 (diff) |
Merge branch 'master' of /home/davem/src/GIT/linux-2.6/
Conflicts:
drivers/firmware/iscsi_ibft.c
Diffstat (limited to 'kernel/trace')
-rw-r--r-- | kernel/trace/Kconfig | 11 | ||||
-rw-r--r-- | kernel/trace/Makefile | 4 | ||||
-rw-r--r-- | kernel/trace/ftrace.c | 105 | ||||
-rw-r--r-- | kernel/trace/trace.c | 144 | ||||
-rw-r--r-- | kernel/trace/trace.h | 6 | ||||
-rw-r--r-- | kernel/trace/trace_branch.c | 19 | ||||
-rw-r--r-- | kernel/trace/trace_event_profile.c | 52 | ||||
-rw-r--r-- | kernel/trace/trace_events.c | 81 | ||||
-rw-r--r-- | kernel/trace/trace_events_filter.c | 4 | ||||
-rw-r--r-- | kernel/trace/trace_export.c | 87 | ||||
-rw-r--r-- | kernel/trace/trace_functions_graph.c | 78 | ||||
-rw-r--r-- | kernel/trace/trace_kprobe.c | 304 | ||||
-rw-r--r-- | kernel/trace/trace_syscalls.c | 189 |
13 files changed, 522 insertions, 562 deletions
diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig index 60e2ce0181ee..13e13d428cd3 100644 --- a/kernel/trace/Kconfig +++ b/kernel/trace/Kconfig | |||
@@ -328,15 +328,6 @@ config BRANCH_TRACER | |||
328 | 328 | ||
329 | Say N if unsure. | 329 | Say N if unsure. |
330 | 330 | ||
331 | config POWER_TRACER | ||
332 | bool "Trace power consumption behavior" | ||
333 | depends on X86 | ||
334 | select GENERIC_TRACER | ||
335 | help | ||
336 | This tracer helps developers to analyze and optimize the kernel's | ||
337 | power management decisions, specifically the C-state and P-state | ||
338 | behavior. | ||
339 | |||
340 | config KSYM_TRACER | 331 | config KSYM_TRACER |
341 | bool "Trace read and write access on kernel memory locations" | 332 | bool "Trace read and write access on kernel memory locations" |
342 | depends on HAVE_HW_BREAKPOINT | 333 | depends on HAVE_HW_BREAKPOINT |
@@ -449,7 +440,7 @@ config BLK_DEV_IO_TRACE | |||
449 | 440 | ||
450 | config KPROBE_EVENT | 441 | config KPROBE_EVENT |
451 | depends on KPROBES | 442 | depends on KPROBES |
452 | depends on X86 | 443 | depends on HAVE_REGS_AND_STACK_ACCESS_API |
453 | bool "Enable kprobes-based dynamic events" | 444 | bool "Enable kprobes-based dynamic events" |
454 | select TRACING | 445 | select TRACING |
455 | default y | 446 | default y |
diff --git a/kernel/trace/Makefile b/kernel/trace/Makefile index cd9ecd89ec77..d00c6fe23f54 100644 --- a/kernel/trace/Makefile +++ b/kernel/trace/Makefile | |||
@@ -51,7 +51,9 @@ endif | |||
51 | obj-$(CONFIG_EVENT_TRACING) += trace_events.o | 51 | obj-$(CONFIG_EVENT_TRACING) += trace_events.o |
52 | obj-$(CONFIG_EVENT_TRACING) += trace_export.o | 52 | obj-$(CONFIG_EVENT_TRACING) += trace_export.o |
53 | obj-$(CONFIG_FTRACE_SYSCALLS) += trace_syscalls.o | 53 | obj-$(CONFIG_FTRACE_SYSCALLS) += trace_syscalls.o |
54 | obj-$(CONFIG_EVENT_PROFILE) += trace_event_profile.o | 54 | ifeq ($(CONFIG_PERF_EVENTS),y) |
55 | obj-$(CONFIG_EVENT_TRACING) += trace_event_profile.o | ||
56 | endif | ||
55 | obj-$(CONFIG_EVENT_TRACING) += trace_events_filter.o | 57 | obj-$(CONFIG_EVENT_TRACING) += trace_events_filter.o |
56 | obj-$(CONFIG_KPROBE_EVENT) += trace_kprobe.o | 58 | obj-$(CONFIG_KPROBE_EVENT) += trace_kprobe.o |
57 | obj-$(CONFIG_KSYM_TRACER) += trace_ksym.o | 59 | obj-$(CONFIG_KSYM_TRACER) += trace_ksym.o |
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 1e6640f80454..83783579378f 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c | |||
@@ -22,7 +22,6 @@ | |||
22 | #include <linux/hardirq.h> | 22 | #include <linux/hardirq.h> |
23 | #include <linux/kthread.h> | 23 | #include <linux/kthread.h> |
24 | #include <linux/uaccess.h> | 24 | #include <linux/uaccess.h> |
25 | #include <linux/kprobes.h> | ||
26 | #include <linux/ftrace.h> | 25 | #include <linux/ftrace.h> |
27 | #include <linux/sysctl.h> | 26 | #include <linux/sysctl.h> |
28 | #include <linux/ctype.h> | 27 | #include <linux/ctype.h> |
@@ -898,36 +897,6 @@ static struct dyn_ftrace *ftrace_free_records; | |||
898 | } \ | 897 | } \ |
899 | } | 898 | } |
900 | 899 | ||
901 | #ifdef CONFIG_KPROBES | ||
902 | |||
903 | static int frozen_record_count; | ||
904 | |||
905 | static inline void freeze_record(struct dyn_ftrace *rec) | ||
906 | { | ||
907 | if (!(rec->flags & FTRACE_FL_FROZEN)) { | ||
908 | rec->flags |= FTRACE_FL_FROZEN; | ||
909 | frozen_record_count++; | ||
910 | } | ||
911 | } | ||
912 | |||
913 | static inline void unfreeze_record(struct dyn_ftrace *rec) | ||
914 | { | ||
915 | if (rec->flags & FTRACE_FL_FROZEN) { | ||
916 | rec->flags &= ~FTRACE_FL_FROZEN; | ||
917 | frozen_record_count--; | ||
918 | } | ||
919 | } | ||
920 | |||
921 | static inline int record_frozen(struct dyn_ftrace *rec) | ||
922 | { | ||
923 | return rec->flags & FTRACE_FL_FROZEN; | ||
924 | } | ||
925 | #else | ||
926 | # define freeze_record(rec) ({ 0; }) | ||
927 | # define unfreeze_record(rec) ({ 0; }) | ||
928 | # define record_frozen(rec) ({ 0; }) | ||
929 | #endif /* CONFIG_KPROBES */ | ||
930 | |||
931 | static void ftrace_free_rec(struct dyn_ftrace *rec) | 900 | static void ftrace_free_rec(struct dyn_ftrace *rec) |
932 | { | 901 | { |
933 | rec->freelist = ftrace_free_records; | 902 | rec->freelist = ftrace_free_records; |
@@ -1025,6 +994,21 @@ static void ftrace_bug(int failed, unsigned long ip) | |||
1025 | } | 994 | } |
1026 | 995 | ||
1027 | 996 | ||
997 | /* Return 1 if the address range is reserved for ftrace */ | ||
998 | int ftrace_text_reserved(void *start, void *end) | ||
999 | { | ||
1000 | struct dyn_ftrace *rec; | ||
1001 | struct ftrace_page *pg; | ||
1002 | |||
1003 | do_for_each_ftrace_rec(pg, rec) { | ||
1004 | if (rec->ip <= (unsigned long)end && | ||
1005 | rec->ip + MCOUNT_INSN_SIZE > (unsigned long)start) | ||
1006 | return 1; | ||
1007 | } while_for_each_ftrace_rec(); | ||
1008 | return 0; | ||
1009 | } | ||
1010 | |||
1011 | |||
1028 | static int | 1012 | static int |
1029 | __ftrace_replace_code(struct dyn_ftrace *rec, int enable) | 1013 | __ftrace_replace_code(struct dyn_ftrace *rec, int enable) |
1030 | { | 1014 | { |
@@ -1076,14 +1060,6 @@ static void ftrace_replace_code(int enable) | |||
1076 | !(rec->flags & FTRACE_FL_CONVERTED)) | 1060 | !(rec->flags & FTRACE_FL_CONVERTED)) |
1077 | continue; | 1061 | continue; |
1078 | 1062 | ||
1079 | /* ignore updates to this record's mcount site */ | ||
1080 | if (get_kprobe((void *)rec->ip)) { | ||
1081 | freeze_record(rec); | ||
1082 | continue; | ||
1083 | } else { | ||
1084 | unfreeze_record(rec); | ||
1085 | } | ||
1086 | |||
1087 | failed = __ftrace_replace_code(rec, enable); | 1063 | failed = __ftrace_replace_code(rec, enable); |
1088 | if (failed) { | 1064 | if (failed) { |
1089 | rec->flags |= FTRACE_FL_FAILED; | 1065 | rec->flags |= FTRACE_FL_FAILED; |
@@ -2426,6 +2402,7 @@ static const struct file_operations ftrace_notrace_fops = { | |||
2426 | static DEFINE_MUTEX(graph_lock); | 2402 | static DEFINE_MUTEX(graph_lock); |
2427 | 2403 | ||
2428 | int ftrace_graph_count; | 2404 | int ftrace_graph_count; |
2405 | int ftrace_graph_filter_enabled; | ||
2429 | unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly; | 2406 | unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly; |
2430 | 2407 | ||
2431 | static void * | 2408 | static void * |
@@ -2448,7 +2425,7 @@ static void *g_start(struct seq_file *m, loff_t *pos) | |||
2448 | mutex_lock(&graph_lock); | 2425 | mutex_lock(&graph_lock); |
2449 | 2426 | ||
2450 | /* Nothing, tell g_show to print all functions are enabled */ | 2427 | /* Nothing, tell g_show to print all functions are enabled */ |
2451 | if (!ftrace_graph_count && !*pos) | 2428 | if (!ftrace_graph_filter_enabled && !*pos) |
2452 | return (void *)1; | 2429 | return (void *)1; |
2453 | 2430 | ||
2454 | return __g_next(m, pos); | 2431 | return __g_next(m, pos); |
@@ -2494,6 +2471,7 @@ ftrace_graph_open(struct inode *inode, struct file *file) | |||
2494 | mutex_lock(&graph_lock); | 2471 | mutex_lock(&graph_lock); |
2495 | if ((file->f_mode & FMODE_WRITE) && | 2472 | if ((file->f_mode & FMODE_WRITE) && |
2496 | (file->f_flags & O_TRUNC)) { | 2473 | (file->f_flags & O_TRUNC)) { |
2474 | ftrace_graph_filter_enabled = 0; | ||
2497 | ftrace_graph_count = 0; | 2475 | ftrace_graph_count = 0; |
2498 | memset(ftrace_graph_funcs, 0, sizeof(ftrace_graph_funcs)); | 2476 | memset(ftrace_graph_funcs, 0, sizeof(ftrace_graph_funcs)); |
2499 | } | 2477 | } |
@@ -2519,7 +2497,7 @@ ftrace_set_func(unsigned long *array, int *idx, char *buffer) | |||
2519 | struct dyn_ftrace *rec; | 2497 | struct dyn_ftrace *rec; |
2520 | struct ftrace_page *pg; | 2498 | struct ftrace_page *pg; |
2521 | int search_len; | 2499 | int search_len; |
2522 | int found = 0; | 2500 | int fail = 1; |
2523 | int type, not; | 2501 | int type, not; |
2524 | char *search; | 2502 | char *search; |
2525 | bool exists; | 2503 | bool exists; |
@@ -2530,37 +2508,51 @@ ftrace_set_func(unsigned long *array, int *idx, char *buffer) | |||
2530 | 2508 | ||
2531 | /* decode regex */ | 2509 | /* decode regex */ |
2532 | type = filter_parse_regex(buffer, strlen(buffer), &search, ¬); | 2510 | type = filter_parse_regex(buffer, strlen(buffer), &search, ¬); |
2533 | if (not) | 2511 | if (!not && *idx >= FTRACE_GRAPH_MAX_FUNCS) |
2534 | return -EINVAL; | 2512 | return -EBUSY; |
2535 | 2513 | ||
2536 | search_len = strlen(search); | 2514 | search_len = strlen(search); |
2537 | 2515 | ||
2538 | mutex_lock(&ftrace_lock); | 2516 | mutex_lock(&ftrace_lock); |
2539 | do_for_each_ftrace_rec(pg, rec) { | 2517 | do_for_each_ftrace_rec(pg, rec) { |
2540 | 2518 | ||
2541 | if (*idx >= FTRACE_GRAPH_MAX_FUNCS) | ||
2542 | break; | ||
2543 | |||
2544 | if (rec->flags & (FTRACE_FL_FAILED | FTRACE_FL_FREE)) | 2519 | if (rec->flags & (FTRACE_FL_FAILED | FTRACE_FL_FREE)) |
2545 | continue; | 2520 | continue; |
2546 | 2521 | ||
2547 | if (ftrace_match_record(rec, search, search_len, type)) { | 2522 | if (ftrace_match_record(rec, search, search_len, type)) { |
2548 | /* ensure it is not already in the array */ | 2523 | /* if it is in the array */ |
2549 | exists = false; | 2524 | exists = false; |
2550 | for (i = 0; i < *idx; i++) | 2525 | for (i = 0; i < *idx; i++) { |
2551 | if (array[i] == rec->ip) { | 2526 | if (array[i] == rec->ip) { |
2552 | exists = true; | 2527 | exists = true; |
2553 | break; | 2528 | break; |
2554 | } | 2529 | } |
2555 | if (!exists) | 2530 | } |
2556 | array[(*idx)++] = rec->ip; | 2531 | |
2557 | found = 1; | 2532 | if (!not) { |
2533 | fail = 0; | ||
2534 | if (!exists) { | ||
2535 | array[(*idx)++] = rec->ip; | ||
2536 | if (*idx >= FTRACE_GRAPH_MAX_FUNCS) | ||
2537 | goto out; | ||
2538 | } | ||
2539 | } else { | ||
2540 | if (exists) { | ||
2541 | array[i] = array[--(*idx)]; | ||
2542 | array[*idx] = 0; | ||
2543 | fail = 0; | ||
2544 | } | ||
2545 | } | ||
2558 | } | 2546 | } |
2559 | } while_for_each_ftrace_rec(); | 2547 | } while_for_each_ftrace_rec(); |
2560 | 2548 | out: | |
2561 | mutex_unlock(&ftrace_lock); | 2549 | mutex_unlock(&ftrace_lock); |
2562 | 2550 | ||
2563 | return found ? 0 : -EINVAL; | 2551 | if (fail) |
2552 | return -EINVAL; | ||
2553 | |||
2554 | ftrace_graph_filter_enabled = 1; | ||
2555 | return 0; | ||
2564 | } | 2556 | } |
2565 | 2557 | ||
2566 | static ssize_t | 2558 | static ssize_t |
@@ -2570,16 +2562,11 @@ ftrace_graph_write(struct file *file, const char __user *ubuf, | |||
2570 | struct trace_parser parser; | 2562 | struct trace_parser parser; |
2571 | ssize_t read, ret; | 2563 | ssize_t read, ret; |
2572 | 2564 | ||
2573 | if (!cnt || cnt < 0) | 2565 | if (!cnt) |
2574 | return 0; | 2566 | return 0; |
2575 | 2567 | ||
2576 | mutex_lock(&graph_lock); | 2568 | mutex_lock(&graph_lock); |
2577 | 2569 | ||
2578 | if (ftrace_graph_count >= FTRACE_GRAPH_MAX_FUNCS) { | ||
2579 | ret = -EBUSY; | ||
2580 | goto out_unlock; | ||
2581 | } | ||
2582 | |||
2583 | if (trace_parser_get_init(&parser, FTRACE_BUFF_MAX)) { | 2570 | if (trace_parser_get_init(&parser, FTRACE_BUFF_MAX)) { |
2584 | ret = -ENOMEM; | 2571 | ret = -ENOMEM; |
2585 | goto out_unlock; | 2572 | goto out_unlock; |
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index eac6875cb990..032c57ca6502 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c | |||
@@ -32,6 +32,7 @@ | |||
32 | #include <linux/splice.h> | 32 | #include <linux/splice.h> |
33 | #include <linux/kdebug.h> | 33 | #include <linux/kdebug.h> |
34 | #include <linux/string.h> | 34 | #include <linux/string.h> |
35 | #include <linux/rwsem.h> | ||
35 | #include <linux/ctype.h> | 36 | #include <linux/ctype.h> |
36 | #include <linux/init.h> | 37 | #include <linux/init.h> |
37 | #include <linux/poll.h> | 38 | #include <linux/poll.h> |
@@ -102,9 +103,6 @@ static inline void ftrace_enable_cpu(void) | |||
102 | 103 | ||
103 | static cpumask_var_t __read_mostly tracing_buffer_mask; | 104 | static cpumask_var_t __read_mostly tracing_buffer_mask; |
104 | 105 | ||
105 | /* Define which cpu buffers are currently read in trace_pipe */ | ||
106 | static cpumask_var_t tracing_reader_cpumask; | ||
107 | |||
108 | #define for_each_tracing_cpu(cpu) \ | 106 | #define for_each_tracing_cpu(cpu) \ |
109 | for_each_cpu(cpu, tracing_buffer_mask) | 107 | for_each_cpu(cpu, tracing_buffer_mask) |
110 | 108 | ||
@@ -243,12 +241,91 @@ static struct tracer *current_trace __read_mostly; | |||
243 | 241 | ||
244 | /* | 242 | /* |
245 | * trace_types_lock is used to protect the trace_types list. | 243 | * trace_types_lock is used to protect the trace_types list. |
246 | * This lock is also used to keep user access serialized. | ||
247 | * Accesses from userspace will grab this lock while userspace | ||
248 | * activities happen inside the kernel. | ||
249 | */ | 244 | */ |
250 | static DEFINE_MUTEX(trace_types_lock); | 245 | static DEFINE_MUTEX(trace_types_lock); |
251 | 246 | ||
247 | /* | ||
248 | * serialize the access of the ring buffer | ||
249 | * | ||
250 | * ring buffer serializes readers, but it is low level protection. | ||
251 | * The validity of the events (which returns by ring_buffer_peek() ..etc) | ||
252 | * are not protected by ring buffer. | ||
253 | * | ||
254 | * The content of events may become garbage if we allow other process consumes | ||
255 | * these events concurrently: | ||
256 | * A) the page of the consumed events may become a normal page | ||
257 | * (not reader page) in ring buffer, and this page will be rewrited | ||
258 | * by events producer. | ||
259 | * B) The page of the consumed events may become a page for splice_read, | ||
260 | * and this page will be returned to system. | ||
261 | * | ||
262 | * These primitives allow multi process access to different cpu ring buffer | ||
263 | * concurrently. | ||
264 | * | ||
265 | * These primitives don't distinguish read-only and read-consume access. | ||
266 | * Multi read-only access are also serialized. | ||
267 | */ | ||
268 | |||
269 | #ifdef CONFIG_SMP | ||
270 | static DECLARE_RWSEM(all_cpu_access_lock); | ||
271 | static DEFINE_PER_CPU(struct mutex, cpu_access_lock); | ||
272 | |||
273 | static inline void trace_access_lock(int cpu) | ||
274 | { | ||
275 | if (cpu == TRACE_PIPE_ALL_CPU) { | ||
276 | /* gain it for accessing the whole ring buffer. */ | ||
277 | down_write(&all_cpu_access_lock); | ||
278 | } else { | ||
279 | /* gain it for accessing a cpu ring buffer. */ | ||
280 | |||
281 | /* Firstly block other trace_access_lock(TRACE_PIPE_ALL_CPU). */ | ||
282 | down_read(&all_cpu_access_lock); | ||
283 | |||
284 | /* Secondly block other access to this @cpu ring buffer. */ | ||
285 | mutex_lock(&per_cpu(cpu_access_lock, cpu)); | ||
286 | } | ||
287 | } | ||
288 | |||
289 | static inline void trace_access_unlock(int cpu) | ||
290 | { | ||
291 | if (cpu == TRACE_PIPE_ALL_CPU) { | ||
292 | up_write(&all_cpu_access_lock); | ||
293 | } else { | ||
294 | mutex_unlock(&per_cpu(cpu_access_lock, cpu)); | ||
295 | up_read(&all_cpu_access_lock); | ||
296 | } | ||
297 | } | ||
298 | |||
299 | static inline void trace_access_lock_init(void) | ||
300 | { | ||
301 | int cpu; | ||
302 | |||
303 | for_each_possible_cpu(cpu) | ||
304 | mutex_init(&per_cpu(cpu_access_lock, cpu)); | ||
305 | } | ||
306 | |||
307 | #else | ||
308 | |||
309 | static DEFINE_MUTEX(access_lock); | ||
310 | |||
311 | static inline void trace_access_lock(int cpu) | ||
312 | { | ||
313 | (void)cpu; | ||
314 | mutex_lock(&access_lock); | ||
315 | } | ||
316 | |||
317 | static inline void trace_access_unlock(int cpu) | ||
318 | { | ||
319 | (void)cpu; | ||
320 | mutex_unlock(&access_lock); | ||
321 | } | ||
322 | |||
323 | static inline void trace_access_lock_init(void) | ||
324 | { | ||
325 | } | ||
326 | |||
327 | #endif | ||
328 | |||
252 | /* trace_wait is a waitqueue for tasks blocked on trace_poll */ | 329 | /* trace_wait is a waitqueue for tasks blocked on trace_poll */ |
253 | static DECLARE_WAIT_QUEUE_HEAD(trace_wait); | 330 | static DECLARE_WAIT_QUEUE_HEAD(trace_wait); |
254 | 331 | ||
@@ -1320,8 +1397,10 @@ int trace_vbprintk(unsigned long ip, const char *fmt, va_list args) | |||
1320 | entry->fmt = fmt; | 1397 | entry->fmt = fmt; |
1321 | 1398 | ||
1322 | memcpy(entry->buf, trace_buf, sizeof(u32) * len); | 1399 | memcpy(entry->buf, trace_buf, sizeof(u32) * len); |
1323 | if (!filter_check_discard(call, entry, buffer, event)) | 1400 | if (!filter_check_discard(call, entry, buffer, event)) { |
1324 | ring_buffer_unlock_commit(buffer, event); | 1401 | ring_buffer_unlock_commit(buffer, event); |
1402 | ftrace_trace_stack(buffer, flags, 6, pc); | ||
1403 | } | ||
1325 | 1404 | ||
1326 | out_unlock: | 1405 | out_unlock: |
1327 | arch_spin_unlock(&trace_buf_lock); | 1406 | arch_spin_unlock(&trace_buf_lock); |
@@ -1394,8 +1473,10 @@ int trace_array_vprintk(struct trace_array *tr, | |||
1394 | 1473 | ||
1395 | memcpy(&entry->buf, trace_buf, len); | 1474 | memcpy(&entry->buf, trace_buf, len); |
1396 | entry->buf[len] = '\0'; | 1475 | entry->buf[len] = '\0'; |
1397 | if (!filter_check_discard(call, entry, buffer, event)) | 1476 | if (!filter_check_discard(call, entry, buffer, event)) { |
1398 | ring_buffer_unlock_commit(buffer, event); | 1477 | ring_buffer_unlock_commit(buffer, event); |
1478 | ftrace_trace_stack(buffer, irq_flags, 6, pc); | ||
1479 | } | ||
1399 | 1480 | ||
1400 | out_unlock: | 1481 | out_unlock: |
1401 | arch_spin_unlock(&trace_buf_lock); | 1482 | arch_spin_unlock(&trace_buf_lock); |
@@ -1585,12 +1666,6 @@ static void tracing_iter_reset(struct trace_iterator *iter, int cpu) | |||
1585 | } | 1666 | } |
1586 | 1667 | ||
1587 | /* | 1668 | /* |
1588 | * No necessary locking here. The worst thing which can | ||
1589 | * happen is loosing events consumed at the same time | ||
1590 | * by a trace_pipe reader. | ||
1591 | * Other than that, we don't risk to crash the ring buffer | ||
1592 | * because it serializes the readers. | ||
1593 | * | ||
1594 | * The current tracer is copied to avoid a global locking | 1669 | * The current tracer is copied to avoid a global locking |
1595 | * all around. | 1670 | * all around. |
1596 | */ | 1671 | */ |
@@ -1645,12 +1720,16 @@ static void *s_start(struct seq_file *m, loff_t *pos) | |||
1645 | } | 1720 | } |
1646 | 1721 | ||
1647 | trace_event_read_lock(); | 1722 | trace_event_read_lock(); |
1723 | trace_access_lock(cpu_file); | ||
1648 | return p; | 1724 | return p; |
1649 | } | 1725 | } |
1650 | 1726 | ||
1651 | static void s_stop(struct seq_file *m, void *p) | 1727 | static void s_stop(struct seq_file *m, void *p) |
1652 | { | 1728 | { |
1729 | struct trace_iterator *iter = m->private; | ||
1730 | |||
1653 | atomic_dec(&trace_record_cmdline_disabled); | 1731 | atomic_dec(&trace_record_cmdline_disabled); |
1732 | trace_access_unlock(iter->cpu_file); | ||
1654 | trace_event_read_unlock(); | 1733 | trace_event_read_unlock(); |
1655 | } | 1734 | } |
1656 | 1735 | ||
@@ -2841,22 +2920,6 @@ static int tracing_open_pipe(struct inode *inode, struct file *filp) | |||
2841 | 2920 | ||
2842 | mutex_lock(&trace_types_lock); | 2921 | mutex_lock(&trace_types_lock); |
2843 | 2922 | ||
2844 | /* We only allow one reader per cpu */ | ||
2845 | if (cpu_file == TRACE_PIPE_ALL_CPU) { | ||
2846 | if (!cpumask_empty(tracing_reader_cpumask)) { | ||
2847 | ret = -EBUSY; | ||
2848 | goto out; | ||
2849 | } | ||
2850 | cpumask_setall(tracing_reader_cpumask); | ||
2851 | } else { | ||
2852 | if (!cpumask_test_cpu(cpu_file, tracing_reader_cpumask)) | ||
2853 | cpumask_set_cpu(cpu_file, tracing_reader_cpumask); | ||
2854 | else { | ||
2855 | ret = -EBUSY; | ||
2856 | goto out; | ||
2857 | } | ||
2858 | } | ||
2859 | |||
2860 | /* create a buffer to store the information to pass to userspace */ | 2923 | /* create a buffer to store the information to pass to userspace */ |
2861 | iter = kzalloc(sizeof(*iter), GFP_KERNEL); | 2924 | iter = kzalloc(sizeof(*iter), GFP_KERNEL); |
2862 | if (!iter) { | 2925 | if (!iter) { |
@@ -2912,12 +2975,6 @@ static int tracing_release_pipe(struct inode *inode, struct file *file) | |||
2912 | 2975 | ||
2913 | mutex_lock(&trace_types_lock); | 2976 | mutex_lock(&trace_types_lock); |
2914 | 2977 | ||
2915 | if (iter->cpu_file == TRACE_PIPE_ALL_CPU) | ||
2916 | cpumask_clear(tracing_reader_cpumask); | ||
2917 | else | ||
2918 | cpumask_clear_cpu(iter->cpu_file, tracing_reader_cpumask); | ||
2919 | |||
2920 | |||
2921 | if (iter->trace->pipe_close) | 2978 | if (iter->trace->pipe_close) |
2922 | iter->trace->pipe_close(iter); | 2979 | iter->trace->pipe_close(iter); |
2923 | 2980 | ||
@@ -3079,6 +3136,7 @@ waitagain: | |||
3079 | iter->pos = -1; | 3136 | iter->pos = -1; |
3080 | 3137 | ||
3081 | trace_event_read_lock(); | 3138 | trace_event_read_lock(); |
3139 | trace_access_lock(iter->cpu_file); | ||
3082 | while (find_next_entry_inc(iter) != NULL) { | 3140 | while (find_next_entry_inc(iter) != NULL) { |
3083 | enum print_line_t ret; | 3141 | enum print_line_t ret; |
3084 | int len = iter->seq.len; | 3142 | int len = iter->seq.len; |
@@ -3095,6 +3153,7 @@ waitagain: | |||
3095 | if (iter->seq.len >= cnt) | 3153 | if (iter->seq.len >= cnt) |
3096 | break; | 3154 | break; |
3097 | } | 3155 | } |
3156 | trace_access_unlock(iter->cpu_file); | ||
3098 | trace_event_read_unlock(); | 3157 | trace_event_read_unlock(); |
3099 | 3158 | ||
3100 | /* Now copy what we have to the user */ | 3159 | /* Now copy what we have to the user */ |
@@ -3220,6 +3279,7 @@ static ssize_t tracing_splice_read_pipe(struct file *filp, | |||
3220 | } | 3279 | } |
3221 | 3280 | ||
3222 | trace_event_read_lock(); | 3281 | trace_event_read_lock(); |
3282 | trace_access_lock(iter->cpu_file); | ||
3223 | 3283 | ||
3224 | /* Fill as many pages as possible. */ | 3284 | /* Fill as many pages as possible. */ |
3225 | for (i = 0, rem = len; i < PIPE_BUFFERS && rem; i++) { | 3285 | for (i = 0, rem = len; i < PIPE_BUFFERS && rem; i++) { |
@@ -3243,6 +3303,7 @@ static ssize_t tracing_splice_read_pipe(struct file *filp, | |||
3243 | trace_seq_init(&iter->seq); | 3303 | trace_seq_init(&iter->seq); |
3244 | } | 3304 | } |
3245 | 3305 | ||
3306 | trace_access_unlock(iter->cpu_file); | ||
3246 | trace_event_read_unlock(); | 3307 | trace_event_read_unlock(); |
3247 | mutex_unlock(&iter->mutex); | 3308 | mutex_unlock(&iter->mutex); |
3248 | 3309 | ||
@@ -3544,10 +3605,12 @@ tracing_buffers_read(struct file *filp, char __user *ubuf, | |||
3544 | 3605 | ||
3545 | info->read = 0; | 3606 | info->read = 0; |
3546 | 3607 | ||
3608 | trace_access_lock(info->cpu); | ||
3547 | ret = ring_buffer_read_page(info->tr->buffer, | 3609 | ret = ring_buffer_read_page(info->tr->buffer, |
3548 | &info->spare, | 3610 | &info->spare, |
3549 | count, | 3611 | count, |
3550 | info->cpu, 0); | 3612 | info->cpu, 0); |
3613 | trace_access_unlock(info->cpu); | ||
3551 | if (ret < 0) | 3614 | if (ret < 0) |
3552 | return 0; | 3615 | return 0; |
3553 | 3616 | ||
@@ -3675,6 +3738,7 @@ tracing_buffers_splice_read(struct file *file, loff_t *ppos, | |||
3675 | len &= PAGE_MASK; | 3738 | len &= PAGE_MASK; |
3676 | } | 3739 | } |
3677 | 3740 | ||
3741 | trace_access_lock(info->cpu); | ||
3678 | entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu); | 3742 | entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu); |
3679 | 3743 | ||
3680 | for (i = 0; i < PIPE_BUFFERS && len && entries; i++, len -= PAGE_SIZE) { | 3744 | for (i = 0; i < PIPE_BUFFERS && len && entries; i++, len -= PAGE_SIZE) { |
@@ -3722,6 +3786,7 @@ tracing_buffers_splice_read(struct file *file, loff_t *ppos, | |||
3722 | entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu); | 3786 | entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu); |
3723 | } | 3787 | } |
3724 | 3788 | ||
3789 | trace_access_unlock(info->cpu); | ||
3725 | spd.nr_pages = i; | 3790 | spd.nr_pages = i; |
3726 | 3791 | ||
3727 | /* did we read anything? */ | 3792 | /* did we read anything? */ |
@@ -4158,6 +4223,8 @@ static __init int tracer_init_debugfs(void) | |||
4158 | struct dentry *d_tracer; | 4223 | struct dentry *d_tracer; |
4159 | int cpu; | 4224 | int cpu; |
4160 | 4225 | ||
4226 | trace_access_lock_init(); | ||
4227 | |||
4161 | d_tracer = tracing_init_dentry(); | 4228 | d_tracer = tracing_init_dentry(); |
4162 | 4229 | ||
4163 | trace_create_file("tracing_enabled", 0644, d_tracer, | 4230 | trace_create_file("tracing_enabled", 0644, d_tracer, |
@@ -4392,9 +4459,6 @@ __init static int tracer_alloc_buffers(void) | |||
4392 | if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL)) | 4459 | if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL)) |
4393 | goto out_free_buffer_mask; | 4460 | goto out_free_buffer_mask; |
4394 | 4461 | ||
4395 | if (!zalloc_cpumask_var(&tracing_reader_cpumask, GFP_KERNEL)) | ||
4396 | goto out_free_tracing_cpumask; | ||
4397 | |||
4398 | /* To save memory, keep the ring buffer size to its minimum */ | 4462 | /* To save memory, keep the ring buffer size to its minimum */ |
4399 | if (ring_buffer_expanded) | 4463 | if (ring_buffer_expanded) |
4400 | ring_buf_size = trace_buf_size; | 4464 | ring_buf_size = trace_buf_size; |
@@ -4452,8 +4516,6 @@ __init static int tracer_alloc_buffers(void) | |||
4452 | return 0; | 4516 | return 0; |
4453 | 4517 | ||
4454 | out_free_cpumask: | 4518 | out_free_cpumask: |
4455 | free_cpumask_var(tracing_reader_cpumask); | ||
4456 | out_free_tracing_cpumask: | ||
4457 | free_cpumask_var(tracing_cpumask); | 4519 | free_cpumask_var(tracing_cpumask); |
4458 | out_free_buffer_mask: | 4520 | out_free_buffer_mask: |
4459 | free_cpumask_var(tracing_buffer_mask); | 4521 | free_cpumask_var(tracing_buffer_mask); |
diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index 4df6a77eb196..fd05bcaf91b0 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h | |||
@@ -497,6 +497,7 @@ trace_print_graph_duration(unsigned long long duration, struct trace_seq *s); | |||
497 | #ifdef CONFIG_DYNAMIC_FTRACE | 497 | #ifdef CONFIG_DYNAMIC_FTRACE |
498 | /* TODO: make this variable */ | 498 | /* TODO: make this variable */ |
499 | #define FTRACE_GRAPH_MAX_FUNCS 32 | 499 | #define FTRACE_GRAPH_MAX_FUNCS 32 |
500 | extern int ftrace_graph_filter_enabled; | ||
500 | extern int ftrace_graph_count; | 501 | extern int ftrace_graph_count; |
501 | extern unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS]; | 502 | extern unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS]; |
502 | 503 | ||
@@ -504,7 +505,7 @@ static inline int ftrace_graph_addr(unsigned long addr) | |||
504 | { | 505 | { |
505 | int i; | 506 | int i; |
506 | 507 | ||
507 | if (!ftrace_graph_count || test_tsk_trace_graph(current)) | 508 | if (!ftrace_graph_filter_enabled) |
508 | return 1; | 509 | return 1; |
509 | 510 | ||
510 | for (i = 0; i < ftrace_graph_count; i++) { | 511 | for (i = 0; i < ftrace_graph_count; i++) { |
@@ -791,7 +792,8 @@ extern const char *__stop___trace_bprintk_fmt[]; | |||
791 | 792 | ||
792 | #undef FTRACE_ENTRY | 793 | #undef FTRACE_ENTRY |
793 | #define FTRACE_ENTRY(call, struct_name, id, tstruct, print) \ | 794 | #define FTRACE_ENTRY(call, struct_name, id, tstruct, print) \ |
794 | extern struct ftrace_event_call event_##call; | 795 | extern struct ftrace_event_call \ |
796 | __attribute__((__aligned__(4))) event_##call; | ||
795 | #undef FTRACE_ENTRY_DUP | 797 | #undef FTRACE_ENTRY_DUP |
796 | #define FTRACE_ENTRY_DUP(call, struct_name, id, tstruct, print) \ | 798 | #define FTRACE_ENTRY_DUP(call, struct_name, id, tstruct, print) \ |
797 | FTRACE_ENTRY(call, struct_name, id, PARAMS(tstruct), PARAMS(print)) | 799 | FTRACE_ENTRY(call, struct_name, id, PARAMS(tstruct), PARAMS(print)) |
diff --git a/kernel/trace/trace_branch.c b/kernel/trace/trace_branch.c index 4a194f08f88c..b9bc4d470177 100644 --- a/kernel/trace/trace_branch.c +++ b/kernel/trace/trace_branch.c | |||
@@ -307,8 +307,23 @@ static int annotated_branch_stat_cmp(void *p1, void *p2) | |||
307 | return -1; | 307 | return -1; |
308 | if (percent_a > percent_b) | 308 | if (percent_a > percent_b) |
309 | return 1; | 309 | return 1; |
310 | else | 310 | |
311 | return 0; | 311 | if (a->incorrect < b->incorrect) |
312 | return -1; | ||
313 | if (a->incorrect > b->incorrect) | ||
314 | return 1; | ||
315 | |||
316 | /* | ||
317 | * Since the above shows worse (incorrect) cases | ||
318 | * first, we continue that by showing best (correct) | ||
319 | * cases last. | ||
320 | */ | ||
321 | if (a->correct > b->correct) | ||
322 | return -1; | ||
323 | if (a->correct < b->correct) | ||
324 | return 1; | ||
325 | |||
326 | return 0; | ||
312 | } | 327 | } |
313 | 328 | ||
314 | static struct tracer_stat annotated_branch_stats = { | 329 | static struct tracer_stat annotated_branch_stats = { |
diff --git a/kernel/trace/trace_event_profile.c b/kernel/trace/trace_event_profile.c index 9e25573242cf..f0d693005075 100644 --- a/kernel/trace/trace_event_profile.c +++ b/kernel/trace/trace_event_profile.c | |||
@@ -6,14 +6,12 @@ | |||
6 | */ | 6 | */ |
7 | 7 | ||
8 | #include <linux/module.h> | 8 | #include <linux/module.h> |
9 | #include <linux/kprobes.h> | ||
9 | #include "trace.h" | 10 | #include "trace.h" |
10 | 11 | ||
11 | 12 | ||
12 | char *perf_trace_buf; | 13 | static char *perf_trace_buf; |
13 | EXPORT_SYMBOL_GPL(perf_trace_buf); | 14 | static char *perf_trace_buf_nmi; |
14 | |||
15 | char *perf_trace_buf_nmi; | ||
16 | EXPORT_SYMBOL_GPL(perf_trace_buf_nmi); | ||
17 | 15 | ||
18 | typedef typeof(char [FTRACE_MAX_PROFILE_SIZE]) perf_trace_t ; | 16 | typedef typeof(char [FTRACE_MAX_PROFILE_SIZE]) perf_trace_t ; |
19 | 17 | ||
@@ -120,3 +118,47 @@ void ftrace_profile_disable(int event_id) | |||
120 | } | 118 | } |
121 | mutex_unlock(&event_mutex); | 119 | mutex_unlock(&event_mutex); |
122 | } | 120 | } |
121 | |||
122 | __kprobes void *ftrace_perf_buf_prepare(int size, unsigned short type, | ||
123 | int *rctxp, unsigned long *irq_flags) | ||
124 | { | ||
125 | struct trace_entry *entry; | ||
126 | char *trace_buf, *raw_data; | ||
127 | int pc, cpu; | ||
128 | |||
129 | pc = preempt_count(); | ||
130 | |||
131 | /* Protect the per cpu buffer, begin the rcu read side */ | ||
132 | local_irq_save(*irq_flags); | ||
133 | |||
134 | *rctxp = perf_swevent_get_recursion_context(); | ||
135 | if (*rctxp < 0) | ||
136 | goto err_recursion; | ||
137 | |||
138 | cpu = smp_processor_id(); | ||
139 | |||
140 | if (in_nmi()) | ||
141 | trace_buf = rcu_dereference(perf_trace_buf_nmi); | ||
142 | else | ||
143 | trace_buf = rcu_dereference(perf_trace_buf); | ||
144 | |||
145 | if (!trace_buf) | ||
146 | goto err; | ||
147 | |||
148 | raw_data = per_cpu_ptr(trace_buf, cpu); | ||
149 | |||
150 | /* zero the dead bytes from align to not leak stack to user */ | ||
151 | *(u64 *)(&raw_data[size - sizeof(u64)]) = 0ULL; | ||
152 | |||
153 | entry = (struct trace_entry *)raw_data; | ||
154 | tracing_generic_entry_update(entry, *irq_flags, pc); | ||
155 | entry->type = type; | ||
156 | |||
157 | return raw_data; | ||
158 | err: | ||
159 | perf_swevent_put_recursion_context(*rctxp); | ||
160 | err_recursion: | ||
161 | local_irq_restore(*irq_flags); | ||
162 | return NULL; | ||
163 | } | ||
164 | EXPORT_SYMBOL_GPL(ftrace_perf_buf_prepare); | ||
diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c index 189b09baf4fb..3f972ad98d04 100644 --- a/kernel/trace/trace_events.c +++ b/kernel/trace/trace_events.c | |||
@@ -60,10 +60,8 @@ int trace_define_field(struct ftrace_event_call *call, const char *type, | |||
60 | return 0; | 60 | return 0; |
61 | 61 | ||
62 | err: | 62 | err: |
63 | if (field) { | 63 | if (field) |
64 | kfree(field->name); | 64 | kfree(field->name); |
65 | kfree(field->type); | ||
66 | } | ||
67 | kfree(field); | 65 | kfree(field); |
68 | 66 | ||
69 | return -ENOMEM; | 67 | return -ENOMEM; |
@@ -520,41 +518,16 @@ out: | |||
520 | return ret; | 518 | return ret; |
521 | } | 519 | } |
522 | 520 | ||
523 | extern char *__bad_type_size(void); | ||
524 | |||
525 | #undef FIELD | ||
526 | #define FIELD(type, name) \ | ||
527 | sizeof(type) != sizeof(field.name) ? __bad_type_size() : \ | ||
528 | #type, "common_" #name, offsetof(typeof(field), name), \ | ||
529 | sizeof(field.name), is_signed_type(type) | ||
530 | |||
531 | static int trace_write_header(struct trace_seq *s) | ||
532 | { | ||
533 | struct trace_entry field; | ||
534 | |||
535 | /* struct trace_entry */ | ||
536 | return trace_seq_printf(s, | ||
537 | "\tfield:%s %s;\toffset:%zu;\tsize:%zu;\tsigned:%u;\n" | ||
538 | "\tfield:%s %s;\toffset:%zu;\tsize:%zu;\tsigned:%u;\n" | ||
539 | "\tfield:%s %s;\toffset:%zu;\tsize:%zu;\tsigned:%u;\n" | ||
540 | "\tfield:%s %s;\toffset:%zu;\tsize:%zu;\tsigned:%u;\n" | ||
541 | "\tfield:%s %s;\toffset:%zu;\tsize:%zu;\tsigned:%u;\n" | ||
542 | "\n", | ||
543 | FIELD(unsigned short, type), | ||
544 | FIELD(unsigned char, flags), | ||
545 | FIELD(unsigned char, preempt_count), | ||
546 | FIELD(int, pid), | ||
547 | FIELD(int, lock_depth)); | ||
548 | } | ||
549 | |||
550 | static ssize_t | 521 | static ssize_t |
551 | event_format_read(struct file *filp, char __user *ubuf, size_t cnt, | 522 | event_format_read(struct file *filp, char __user *ubuf, size_t cnt, |
552 | loff_t *ppos) | 523 | loff_t *ppos) |
553 | { | 524 | { |
554 | struct ftrace_event_call *call = filp->private_data; | 525 | struct ftrace_event_call *call = filp->private_data; |
526 | struct ftrace_event_field *field; | ||
555 | struct trace_seq *s; | 527 | struct trace_seq *s; |
528 | int common_field_count = 5; | ||
556 | char *buf; | 529 | char *buf; |
557 | int r; | 530 | int r = 0; |
558 | 531 | ||
559 | if (*ppos) | 532 | if (*ppos) |
560 | return 0; | 533 | return 0; |
@@ -565,14 +538,48 @@ event_format_read(struct file *filp, char __user *ubuf, size_t cnt, | |||
565 | 538 | ||
566 | trace_seq_init(s); | 539 | trace_seq_init(s); |
567 | 540 | ||
568 | /* If any of the first writes fail, so will the show_format. */ | ||
569 | |||
570 | trace_seq_printf(s, "name: %s\n", call->name); | 541 | trace_seq_printf(s, "name: %s\n", call->name); |
571 | trace_seq_printf(s, "ID: %d\n", call->id); | 542 | trace_seq_printf(s, "ID: %d\n", call->id); |
572 | trace_seq_printf(s, "format:\n"); | 543 | trace_seq_printf(s, "format:\n"); |
573 | trace_write_header(s); | ||
574 | 544 | ||
575 | r = call->show_format(call, s); | 545 | list_for_each_entry_reverse(field, &call->fields, link) { |
546 | /* | ||
547 | * Smartly shows the array type(except dynamic array). | ||
548 | * Normal: | ||
549 | * field:TYPE VAR | ||
550 | * If TYPE := TYPE[LEN], it is shown: | ||
551 | * field:TYPE VAR[LEN] | ||
552 | */ | ||
553 | const char *array_descriptor = strchr(field->type, '['); | ||
554 | |||
555 | if (!strncmp(field->type, "__data_loc", 10)) | ||
556 | array_descriptor = NULL; | ||
557 | |||
558 | if (!array_descriptor) { | ||
559 | r = trace_seq_printf(s, "\tfield:%s %s;\toffset:%u;" | ||
560 | "\tsize:%u;\tsigned:%d;\n", | ||
561 | field->type, field->name, field->offset, | ||
562 | field->size, !!field->is_signed); | ||
563 | } else { | ||
564 | r = trace_seq_printf(s, "\tfield:%.*s %s%s;\toffset:%u;" | ||
565 | "\tsize:%u;\tsigned:%d;\n", | ||
566 | (int)(array_descriptor - field->type), | ||
567 | field->type, field->name, | ||
568 | array_descriptor, field->offset, | ||
569 | field->size, !!field->is_signed); | ||
570 | } | ||
571 | |||
572 | if (--common_field_count == 0) | ||
573 | r = trace_seq_printf(s, "\n"); | ||
574 | |||
575 | if (!r) | ||
576 | break; | ||
577 | } | ||
578 | |||
579 | if (r) | ||
580 | r = trace_seq_printf(s, "\nprint fmt: %s\n", | ||
581 | call->print_fmt); | ||
582 | |||
576 | if (!r) { | 583 | if (!r) { |
577 | /* | 584 | /* |
578 | * ug! The format output is bigger than a PAGE!! | 585 | * ug! The format output is bigger than a PAGE!! |
@@ -948,10 +955,6 @@ event_create_dir(struct ftrace_event_call *call, struct dentry *d_events, | |||
948 | filter); | 955 | filter); |
949 | } | 956 | } |
950 | 957 | ||
951 | /* A trace may not want to export its format */ | ||
952 | if (!call->show_format) | ||
953 | return 0; | ||
954 | |||
955 | trace_create_file("format", 0444, call->dir, call, | 958 | trace_create_file("format", 0444, call->dir, call, |
956 | format); | 959 | format); |
957 | 960 | ||
diff --git a/kernel/trace/trace_events_filter.c b/kernel/trace/trace_events_filter.c index e42af9aad69f..4615f62a04f1 100644 --- a/kernel/trace/trace_events_filter.c +++ b/kernel/trace/trace_events_filter.c | |||
@@ -1371,7 +1371,7 @@ out_unlock: | |||
1371 | return err; | 1371 | return err; |
1372 | } | 1372 | } |
1373 | 1373 | ||
1374 | #ifdef CONFIG_EVENT_PROFILE | 1374 | #ifdef CONFIG_PERF_EVENTS |
1375 | 1375 | ||
1376 | void ftrace_profile_free_filter(struct perf_event *event) | 1376 | void ftrace_profile_free_filter(struct perf_event *event) |
1377 | { | 1377 | { |
@@ -1439,5 +1439,5 @@ out_unlock: | |||
1439 | return err; | 1439 | return err; |
1440 | } | 1440 | } |
1441 | 1441 | ||
1442 | #endif /* CONFIG_EVENT_PROFILE */ | 1442 | #endif /* CONFIG_PERF_EVENTS */ |
1443 | 1443 | ||
diff --git a/kernel/trace/trace_export.c b/kernel/trace/trace_export.c index d4fa5dc1ee4e..e091f64ba6ce 100644 --- a/kernel/trace/trace_export.c +++ b/kernel/trace/trace_export.c | |||
@@ -62,78 +62,6 @@ static void __always_unused ____ftrace_check_##name(void) \ | |||
62 | 62 | ||
63 | #include "trace_entries.h" | 63 | #include "trace_entries.h" |
64 | 64 | ||
65 | |||
66 | #undef __field | ||
67 | #define __field(type, item) \ | ||
68 | ret = trace_seq_printf(s, "\tfield:" #type " " #item ";\t" \ | ||
69 | "offset:%zu;\tsize:%zu;\tsigned:%u;\n", \ | ||
70 | offsetof(typeof(field), item), \ | ||
71 | sizeof(field.item), is_signed_type(type)); \ | ||
72 | if (!ret) \ | ||
73 | return 0; | ||
74 | |||
75 | #undef __field_desc | ||
76 | #define __field_desc(type, container, item) \ | ||
77 | ret = trace_seq_printf(s, "\tfield:" #type " " #item ";\t" \ | ||
78 | "offset:%zu;\tsize:%zu;\tsigned:%u;\n", \ | ||
79 | offsetof(typeof(field), container.item), \ | ||
80 | sizeof(field.container.item), \ | ||
81 | is_signed_type(type)); \ | ||
82 | if (!ret) \ | ||
83 | return 0; | ||
84 | |||
85 | #undef __array | ||
86 | #define __array(type, item, len) \ | ||
87 | ret = trace_seq_printf(s, "\tfield:" #type " " #item "[" #len "];\t" \ | ||
88 | "offset:%zu;\tsize:%zu;\tsigned:%u;\n", \ | ||
89 | offsetof(typeof(field), item), \ | ||
90 | sizeof(field.item), is_signed_type(type)); \ | ||
91 | if (!ret) \ | ||
92 | return 0; | ||
93 | |||
94 | #undef __array_desc | ||
95 | #define __array_desc(type, container, item, len) \ | ||
96 | ret = trace_seq_printf(s, "\tfield:" #type " " #item "[" #len "];\t" \ | ||
97 | "offset:%zu;\tsize:%zu;\tsigned:%u;\n", \ | ||
98 | offsetof(typeof(field), container.item), \ | ||
99 | sizeof(field.container.item), \ | ||
100 | is_signed_type(type)); \ | ||
101 | if (!ret) \ | ||
102 | return 0; | ||
103 | |||
104 | #undef __dynamic_array | ||
105 | #define __dynamic_array(type, item) \ | ||
106 | ret = trace_seq_printf(s, "\tfield:" #type " " #item ";\t" \ | ||
107 | "offset:%zu;\tsize:0;\tsigned:%u;\n", \ | ||
108 | offsetof(typeof(field), item), \ | ||
109 | is_signed_type(type)); \ | ||
110 | if (!ret) \ | ||
111 | return 0; | ||
112 | |||
113 | #undef F_printk | ||
114 | #define F_printk(fmt, args...) "%s, %s\n", #fmt, __stringify(args) | ||
115 | |||
116 | #undef __entry | ||
117 | #define __entry REC | ||
118 | |||
119 | #undef FTRACE_ENTRY | ||
120 | #define FTRACE_ENTRY(name, struct_name, id, tstruct, print) \ | ||
121 | static int \ | ||
122 | ftrace_format_##name(struct ftrace_event_call *unused, \ | ||
123 | struct trace_seq *s) \ | ||
124 | { \ | ||
125 | struct struct_name field __attribute__((unused)); \ | ||
126 | int ret = 0; \ | ||
127 | \ | ||
128 | tstruct; \ | ||
129 | \ | ||
130 | trace_seq_printf(s, "\nprint fmt: " print); \ | ||
131 | \ | ||
132 | return ret; \ | ||
133 | } | ||
134 | |||
135 | #include "trace_entries.h" | ||
136 | |||
137 | #undef __field | 65 | #undef __field |
138 | #define __field(type, item) \ | 66 | #define __field(type, item) \ |
139 | ret = trace_define_field(event_call, #type, #item, \ | 67 | ret = trace_define_field(event_call, #type, #item, \ |
@@ -175,7 +103,12 @@ ftrace_format_##name(struct ftrace_event_call *unused, \ | |||
175 | return ret; | 103 | return ret; |
176 | 104 | ||
177 | #undef __dynamic_array | 105 | #undef __dynamic_array |
178 | #define __dynamic_array(type, item) | 106 | #define __dynamic_array(type, item) \ |
107 | ret = trace_define_field(event_call, #type, #item, \ | ||
108 | offsetof(typeof(field), item), \ | ||
109 | 0, is_signed_type(type), FILTER_OTHER);\ | ||
110 | if (ret) \ | ||
111 | return ret; | ||
179 | 112 | ||
180 | #undef FTRACE_ENTRY | 113 | #undef FTRACE_ENTRY |
181 | #define FTRACE_ENTRY(name, struct_name, id, tstruct, print) \ | 114 | #define FTRACE_ENTRY(name, struct_name, id, tstruct, print) \ |
@@ -198,6 +131,9 @@ static int ftrace_raw_init_event(struct ftrace_event_call *call) | |||
198 | return 0; | 131 | return 0; |
199 | } | 132 | } |
200 | 133 | ||
134 | #undef __entry | ||
135 | #define __entry REC | ||
136 | |||
201 | #undef __field | 137 | #undef __field |
202 | #define __field(type, item) | 138 | #define __field(type, item) |
203 | 139 | ||
@@ -213,6 +149,9 @@ static int ftrace_raw_init_event(struct ftrace_event_call *call) | |||
213 | #undef __dynamic_array | 149 | #undef __dynamic_array |
214 | #define __dynamic_array(type, item) | 150 | #define __dynamic_array(type, item) |
215 | 151 | ||
152 | #undef F_printk | ||
153 | #define F_printk(fmt, args...) #fmt ", " __stringify(args) | ||
154 | |||
216 | #undef FTRACE_ENTRY | 155 | #undef FTRACE_ENTRY |
217 | #define FTRACE_ENTRY(call, struct_name, type, tstruct, print) \ | 156 | #define FTRACE_ENTRY(call, struct_name, type, tstruct, print) \ |
218 | \ | 157 | \ |
@@ -223,7 +162,7 @@ __attribute__((section("_ftrace_events"))) event_##call = { \ | |||
223 | .id = type, \ | 162 | .id = type, \ |
224 | .system = __stringify(TRACE_SYSTEM), \ | 163 | .system = __stringify(TRACE_SYSTEM), \ |
225 | .raw_init = ftrace_raw_init_event, \ | 164 | .raw_init = ftrace_raw_init_event, \ |
226 | .show_format = ftrace_format_##call, \ | 165 | .print_fmt = print, \ |
227 | .define_fields = ftrace_define_fields_##call, \ | 166 | .define_fields = ftrace_define_fields_##call, \ |
228 | }; \ | 167 | }; \ |
229 | 168 | ||
diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c index b1342c5d37cf..e998a824e9db 100644 --- a/kernel/trace/trace_functions_graph.c +++ b/kernel/trace/trace_functions_graph.c | |||
@@ -18,6 +18,7 @@ struct fgraph_cpu_data { | |||
18 | pid_t last_pid; | 18 | pid_t last_pid; |
19 | int depth; | 19 | int depth; |
20 | int ignore; | 20 | int ignore; |
21 | unsigned long enter_funcs[FTRACE_RETFUNC_DEPTH]; | ||
21 | }; | 22 | }; |
22 | 23 | ||
23 | struct fgraph_data { | 24 | struct fgraph_data { |
@@ -212,13 +213,11 @@ int trace_graph_entry(struct ftrace_graph_ent *trace) | |||
212 | int cpu; | 213 | int cpu; |
213 | int pc; | 214 | int pc; |
214 | 215 | ||
215 | if (unlikely(!tr)) | ||
216 | return 0; | ||
217 | |||
218 | if (!ftrace_trace_task(current)) | 216 | if (!ftrace_trace_task(current)) |
219 | return 0; | 217 | return 0; |
220 | 218 | ||
221 | if (!ftrace_graph_addr(trace->func)) | 219 | /* trace it when it is-nested-in or is a function enabled. */ |
220 | if (!(trace->depth || ftrace_graph_addr(trace->func))) | ||
222 | return 0; | 221 | return 0; |
223 | 222 | ||
224 | local_irq_save(flags); | 223 | local_irq_save(flags); |
@@ -231,9 +230,6 @@ int trace_graph_entry(struct ftrace_graph_ent *trace) | |||
231 | } else { | 230 | } else { |
232 | ret = 0; | 231 | ret = 0; |
233 | } | 232 | } |
234 | /* Only do the atomic if it is not already set */ | ||
235 | if (!test_tsk_trace_graph(current)) | ||
236 | set_tsk_trace_graph(current); | ||
237 | 233 | ||
238 | atomic_dec(&data->disabled); | 234 | atomic_dec(&data->disabled); |
239 | local_irq_restore(flags); | 235 | local_irq_restore(flags); |
@@ -281,17 +277,24 @@ void trace_graph_return(struct ftrace_graph_ret *trace) | |||
281 | pc = preempt_count(); | 277 | pc = preempt_count(); |
282 | __trace_graph_return(tr, trace, flags, pc); | 278 | __trace_graph_return(tr, trace, flags, pc); |
283 | } | 279 | } |
284 | if (!trace->depth) | ||
285 | clear_tsk_trace_graph(current); | ||
286 | atomic_dec(&data->disabled); | 280 | atomic_dec(&data->disabled); |
287 | local_irq_restore(flags); | 281 | local_irq_restore(flags); |
288 | } | 282 | } |
289 | 283 | ||
284 | void set_graph_array(struct trace_array *tr) | ||
285 | { | ||
286 | graph_array = tr; | ||
287 | |||
288 | /* Make graph_array visible before we start tracing */ | ||
289 | |||
290 | smp_mb(); | ||
291 | } | ||
292 | |||
290 | static int graph_trace_init(struct trace_array *tr) | 293 | static int graph_trace_init(struct trace_array *tr) |
291 | { | 294 | { |
292 | int ret; | 295 | int ret; |
293 | 296 | ||
294 | graph_array = tr; | 297 | set_graph_array(tr); |
295 | ret = register_ftrace_graph(&trace_graph_return, | 298 | ret = register_ftrace_graph(&trace_graph_return, |
296 | &trace_graph_entry); | 299 | &trace_graph_entry); |
297 | if (ret) | 300 | if (ret) |
@@ -301,11 +304,6 @@ static int graph_trace_init(struct trace_array *tr) | |||
301 | return 0; | 304 | return 0; |
302 | } | 305 | } |
303 | 306 | ||
304 | void set_graph_array(struct trace_array *tr) | ||
305 | { | ||
306 | graph_array = tr; | ||
307 | } | ||
308 | |||
309 | static void graph_trace_reset(struct trace_array *tr) | 307 | static void graph_trace_reset(struct trace_array *tr) |
310 | { | 308 | { |
311 | tracing_stop_cmdline_record(); | 309 | tracing_stop_cmdline_record(); |
@@ -673,15 +671,21 @@ print_graph_entry_leaf(struct trace_iterator *iter, | |||
673 | duration = graph_ret->rettime - graph_ret->calltime; | 671 | duration = graph_ret->rettime - graph_ret->calltime; |
674 | 672 | ||
675 | if (data) { | 673 | if (data) { |
674 | struct fgraph_cpu_data *cpu_data; | ||
676 | int cpu = iter->cpu; | 675 | int cpu = iter->cpu; |
677 | int *depth = &(per_cpu_ptr(data->cpu_data, cpu)->depth); | 676 | |
677 | cpu_data = per_cpu_ptr(data->cpu_data, cpu); | ||
678 | 678 | ||
679 | /* | 679 | /* |
680 | * Comments display at + 1 to depth. Since | 680 | * Comments display at + 1 to depth. Since |
681 | * this is a leaf function, keep the comments | 681 | * this is a leaf function, keep the comments |
682 | * equal to this depth. | 682 | * equal to this depth. |
683 | */ | 683 | */ |
684 | *depth = call->depth - 1; | 684 | cpu_data->depth = call->depth - 1; |
685 | |||
686 | /* No need to keep this function around for this depth */ | ||
687 | if (call->depth < FTRACE_RETFUNC_DEPTH) | ||
688 | cpu_data->enter_funcs[call->depth] = 0; | ||
685 | } | 689 | } |
686 | 690 | ||
687 | /* Overhead */ | 691 | /* Overhead */ |
@@ -721,10 +725,15 @@ print_graph_entry_nested(struct trace_iterator *iter, | |||
721 | int i; | 725 | int i; |
722 | 726 | ||
723 | if (data) { | 727 | if (data) { |
728 | struct fgraph_cpu_data *cpu_data; | ||
724 | int cpu = iter->cpu; | 729 | int cpu = iter->cpu; |
725 | int *depth = &(per_cpu_ptr(data->cpu_data, cpu)->depth); | ||
726 | 730 | ||
727 | *depth = call->depth; | 731 | cpu_data = per_cpu_ptr(data->cpu_data, cpu); |
732 | cpu_data->depth = call->depth; | ||
733 | |||
734 | /* Save this function pointer to see if the exit matches */ | ||
735 | if (call->depth < FTRACE_RETFUNC_DEPTH) | ||
736 | cpu_data->enter_funcs[call->depth] = call->func; | ||
728 | } | 737 | } |
729 | 738 | ||
730 | /* No overhead */ | 739 | /* No overhead */ |
@@ -854,19 +863,28 @@ print_graph_return(struct ftrace_graph_ret *trace, struct trace_seq *s, | |||
854 | struct fgraph_data *data = iter->private; | 863 | struct fgraph_data *data = iter->private; |
855 | pid_t pid = ent->pid; | 864 | pid_t pid = ent->pid; |
856 | int cpu = iter->cpu; | 865 | int cpu = iter->cpu; |
866 | int func_match = 1; | ||
857 | int ret; | 867 | int ret; |
858 | int i; | 868 | int i; |
859 | 869 | ||
860 | if (data) { | 870 | if (data) { |
871 | struct fgraph_cpu_data *cpu_data; | ||
861 | int cpu = iter->cpu; | 872 | int cpu = iter->cpu; |
862 | int *depth = &(per_cpu_ptr(data->cpu_data, cpu)->depth); | 873 | |
874 | cpu_data = per_cpu_ptr(data->cpu_data, cpu); | ||
863 | 875 | ||
864 | /* | 876 | /* |
865 | * Comments display at + 1 to depth. This is the | 877 | * Comments display at + 1 to depth. This is the |
866 | * return from a function, we now want the comments | 878 | * return from a function, we now want the comments |
867 | * to display at the same level of the bracket. | 879 | * to display at the same level of the bracket. |
868 | */ | 880 | */ |
869 | *depth = trace->depth - 1; | 881 | cpu_data->depth = trace->depth - 1; |
882 | |||
883 | if (trace->depth < FTRACE_RETFUNC_DEPTH) { | ||
884 | if (cpu_data->enter_funcs[trace->depth] != trace->func) | ||
885 | func_match = 0; | ||
886 | cpu_data->enter_funcs[trace->depth] = 0; | ||
887 | } | ||
870 | } | 888 | } |
871 | 889 | ||
872 | if (print_graph_prologue(iter, s, 0, 0)) | 890 | if (print_graph_prologue(iter, s, 0, 0)) |
@@ -891,9 +909,21 @@ print_graph_return(struct ftrace_graph_ret *trace, struct trace_seq *s, | |||
891 | return TRACE_TYPE_PARTIAL_LINE; | 909 | return TRACE_TYPE_PARTIAL_LINE; |
892 | } | 910 | } |
893 | 911 | ||
894 | ret = trace_seq_printf(s, "}\n"); | 912 | /* |
895 | if (!ret) | 913 | * If the return function does not have a matching entry, |
896 | return TRACE_TYPE_PARTIAL_LINE; | 914 | * then the entry was lost. Instead of just printing |
915 | * the '}' and letting the user guess what function this | ||
916 | * belongs to, write out the function name. | ||
917 | */ | ||
918 | if (func_match) { | ||
919 | ret = trace_seq_printf(s, "}\n"); | ||
920 | if (!ret) | ||
921 | return TRACE_TYPE_PARTIAL_LINE; | ||
922 | } else { | ||
923 | ret = trace_seq_printf(s, "} (%ps)\n", (void *)trace->func); | ||
924 | if (!ret) | ||
925 | return TRACE_TYPE_PARTIAL_LINE; | ||
926 | } | ||
897 | 927 | ||
898 | /* Overrun */ | 928 | /* Overrun */ |
899 | if (tracer_flags.val & TRACE_GRAPH_PRINT_OVERRUN) { | 929 | if (tracer_flags.val & TRACE_GRAPH_PRINT_OVERRUN) { |
diff --git a/kernel/trace/trace_kprobe.c b/kernel/trace/trace_kprobe.c index 50b1b8239806..505c92273b1a 100644 --- a/kernel/trace/trace_kprobe.c +++ b/kernel/trace/trace_kprobe.c | |||
@@ -91,11 +91,6 @@ static __kprobes unsigned long fetch_memory(struct pt_regs *regs, void *addr) | |||
91 | return retval; | 91 | return retval; |
92 | } | 92 | } |
93 | 93 | ||
94 | static __kprobes unsigned long fetch_argument(struct pt_regs *regs, void *num) | ||
95 | { | ||
96 | return regs_get_argument_nth(regs, (unsigned int)((unsigned long)num)); | ||
97 | } | ||
98 | |||
99 | static __kprobes unsigned long fetch_retvalue(struct pt_regs *regs, | 94 | static __kprobes unsigned long fetch_retvalue(struct pt_regs *regs, |
100 | void *dummy) | 95 | void *dummy) |
101 | { | 96 | { |
@@ -231,9 +226,7 @@ static int probe_arg_string(char *buf, size_t n, struct fetch_func *ff) | |||
231 | { | 226 | { |
232 | int ret = -EINVAL; | 227 | int ret = -EINVAL; |
233 | 228 | ||
234 | if (ff->func == fetch_argument) | 229 | if (ff->func == fetch_register) { |
235 | ret = snprintf(buf, n, "$arg%lu", (unsigned long)ff->data); | ||
236 | else if (ff->func == fetch_register) { | ||
237 | const char *name; | 230 | const char *name; |
238 | name = regs_query_register_name((unsigned int)((long)ff->data)); | 231 | name = regs_query_register_name((unsigned int)((long)ff->data)); |
239 | ret = snprintf(buf, n, "%%%s", name); | 232 | ret = snprintf(buf, n, "%%%s", name); |
@@ -489,14 +482,6 @@ static int parse_probe_vars(char *arg, struct fetch_func *ff, int is_return) | |||
489 | } | 482 | } |
490 | } else | 483 | } else |
491 | ret = -EINVAL; | 484 | ret = -EINVAL; |
492 | } else if (strncmp(arg, "arg", 3) == 0 && isdigit(arg[3])) { | ||
493 | ret = strict_strtoul(arg + 3, 10, ¶m); | ||
494 | if (ret || param > PARAM_MAX_ARGS) | ||
495 | ret = -EINVAL; | ||
496 | else { | ||
497 | ff->func = fetch_argument; | ||
498 | ff->data = (void *)param; | ||
499 | } | ||
500 | } else | 485 | } else |
501 | ret = -EINVAL; | 486 | ret = -EINVAL; |
502 | return ret; | 487 | return ret; |
@@ -611,7 +596,6 @@ static int create_trace_probe(int argc, char **argv) | |||
611 | * - Add kprobe: p[:[GRP/]EVENT] KSYM[+OFFS]|KADDR [FETCHARGS] | 596 | * - Add kprobe: p[:[GRP/]EVENT] KSYM[+OFFS]|KADDR [FETCHARGS] |
612 | * - Add kretprobe: r[:[GRP/]EVENT] KSYM[+0] [FETCHARGS] | 597 | * - Add kretprobe: r[:[GRP/]EVENT] KSYM[+0] [FETCHARGS] |
613 | * Fetch args: | 598 | * Fetch args: |
614 | * $argN : fetch Nth of function argument. (N:0-) | ||
615 | * $retval : fetch return value | 599 | * $retval : fetch return value |
616 | * $stack : fetch stack address | 600 | * $stack : fetch stack address |
617 | * $stackN : fetch Nth of stack (N:0-) | 601 | * $stackN : fetch Nth of stack (N:0-) |
@@ -651,12 +635,12 @@ static int create_trace_probe(int argc, char **argv) | |||
651 | event = strchr(group, '/') + 1; | 635 | event = strchr(group, '/') + 1; |
652 | event[-1] = '\0'; | 636 | event[-1] = '\0'; |
653 | if (strlen(group) == 0) { | 637 | if (strlen(group) == 0) { |
654 | pr_info("Group name is not specifiled\n"); | 638 | pr_info("Group name is not specified\n"); |
655 | return -EINVAL; | 639 | return -EINVAL; |
656 | } | 640 | } |
657 | } | 641 | } |
658 | if (strlen(event) == 0) { | 642 | if (strlen(event) == 0) { |
659 | pr_info("Event name is not specifiled\n"); | 643 | pr_info("Event name is not specified\n"); |
660 | return -EINVAL; | 644 | return -EINVAL; |
661 | } | 645 | } |
662 | } | 646 | } |
@@ -958,7 +942,7 @@ static const struct file_operations kprobe_profile_ops = { | |||
958 | }; | 942 | }; |
959 | 943 | ||
960 | /* Kprobe handler */ | 944 | /* Kprobe handler */ |
961 | static __kprobes int kprobe_trace_func(struct kprobe *kp, struct pt_regs *regs) | 945 | static __kprobes void kprobe_trace_func(struct kprobe *kp, struct pt_regs *regs) |
962 | { | 946 | { |
963 | struct trace_probe *tp = container_of(kp, struct trace_probe, rp.kp); | 947 | struct trace_probe *tp = container_of(kp, struct trace_probe, rp.kp); |
964 | struct kprobe_trace_entry *entry; | 948 | struct kprobe_trace_entry *entry; |
@@ -978,7 +962,7 @@ static __kprobes int kprobe_trace_func(struct kprobe *kp, struct pt_regs *regs) | |||
978 | event = trace_current_buffer_lock_reserve(&buffer, call->id, size, | 962 | event = trace_current_buffer_lock_reserve(&buffer, call->id, size, |
979 | irq_flags, pc); | 963 | irq_flags, pc); |
980 | if (!event) | 964 | if (!event) |
981 | return 0; | 965 | return; |
982 | 966 | ||
983 | entry = ring_buffer_event_data(event); | 967 | entry = ring_buffer_event_data(event); |
984 | entry->nargs = tp->nr_args; | 968 | entry->nargs = tp->nr_args; |
@@ -988,11 +972,10 @@ static __kprobes int kprobe_trace_func(struct kprobe *kp, struct pt_regs *regs) | |||
988 | 972 | ||
989 | if (!filter_current_check_discard(buffer, call, entry, event)) | 973 | if (!filter_current_check_discard(buffer, call, entry, event)) |
990 | trace_nowake_buffer_unlock_commit(buffer, event, irq_flags, pc); | 974 | trace_nowake_buffer_unlock_commit(buffer, event, irq_flags, pc); |
991 | return 0; | ||
992 | } | 975 | } |
993 | 976 | ||
994 | /* Kretprobe handler */ | 977 | /* Kretprobe handler */ |
995 | static __kprobes int kretprobe_trace_func(struct kretprobe_instance *ri, | 978 | static __kprobes void kretprobe_trace_func(struct kretprobe_instance *ri, |
996 | struct pt_regs *regs) | 979 | struct pt_regs *regs) |
997 | { | 980 | { |
998 | struct trace_probe *tp = container_of(ri->rp, struct trace_probe, rp); | 981 | struct trace_probe *tp = container_of(ri->rp, struct trace_probe, rp); |
@@ -1011,7 +994,7 @@ static __kprobes int kretprobe_trace_func(struct kretprobe_instance *ri, | |||
1011 | event = trace_current_buffer_lock_reserve(&buffer, call->id, size, | 994 | event = trace_current_buffer_lock_reserve(&buffer, call->id, size, |
1012 | irq_flags, pc); | 995 | irq_flags, pc); |
1013 | if (!event) | 996 | if (!event) |
1014 | return 0; | 997 | return; |
1015 | 998 | ||
1016 | entry = ring_buffer_event_data(event); | 999 | entry = ring_buffer_event_data(event); |
1017 | entry->nargs = tp->nr_args; | 1000 | entry->nargs = tp->nr_args; |
@@ -1022,8 +1005,6 @@ static __kprobes int kretprobe_trace_func(struct kretprobe_instance *ri, | |||
1022 | 1005 | ||
1023 | if (!filter_current_check_discard(buffer, call, entry, event)) | 1006 | if (!filter_current_check_discard(buffer, call, entry, event)) |
1024 | trace_nowake_buffer_unlock_commit(buffer, event, irq_flags, pc); | 1007 | trace_nowake_buffer_unlock_commit(buffer, event, irq_flags, pc); |
1025 | |||
1026 | return 0; | ||
1027 | } | 1008 | } |
1028 | 1009 | ||
1029 | /* Event entry printers */ | 1010 | /* Event entry printers */ |
@@ -1174,213 +1155,123 @@ static int kretprobe_event_define_fields(struct ftrace_event_call *event_call) | |||
1174 | return 0; | 1155 | return 0; |
1175 | } | 1156 | } |
1176 | 1157 | ||
1177 | static int __probe_event_show_format(struct trace_seq *s, | 1158 | static int __set_print_fmt(struct trace_probe *tp, char *buf, int len) |
1178 | struct trace_probe *tp, const char *fmt, | ||
1179 | const char *arg) | ||
1180 | { | 1159 | { |
1181 | int i; | 1160 | int i; |
1161 | int pos = 0; | ||
1182 | 1162 | ||
1183 | /* Show format */ | 1163 | const char *fmt, *arg; |
1184 | if (!trace_seq_printf(s, "\nprint fmt: \"%s", fmt)) | ||
1185 | return 0; | ||
1186 | 1164 | ||
1187 | for (i = 0; i < tp->nr_args; i++) | 1165 | if (!probe_is_return(tp)) { |
1188 | if (!trace_seq_printf(s, " %s=%%lx", tp->args[i].name)) | 1166 | fmt = "(%lx)"; |
1189 | return 0; | 1167 | arg = "REC->" FIELD_STRING_IP; |
1168 | } else { | ||
1169 | fmt = "(%lx <- %lx)"; | ||
1170 | arg = "REC->" FIELD_STRING_FUNC ", REC->" FIELD_STRING_RETIP; | ||
1171 | } | ||
1190 | 1172 | ||
1191 | if (!trace_seq_printf(s, "\", %s", arg)) | 1173 | /* When len=0, we just calculate the needed length */ |
1192 | return 0; | 1174 | #define LEN_OR_ZERO (len ? len - pos : 0) |
1193 | 1175 | ||
1194 | for (i = 0; i < tp->nr_args; i++) | 1176 | pos += snprintf(buf + pos, LEN_OR_ZERO, "\"%s", fmt); |
1195 | if (!trace_seq_printf(s, ", REC->%s", tp->args[i].name)) | ||
1196 | return 0; | ||
1197 | |||
1198 | return trace_seq_puts(s, "\n"); | ||
1199 | } | ||
1200 | 1177 | ||
1201 | #undef SHOW_FIELD | 1178 | for (i = 0; i < tp->nr_args; i++) { |
1202 | #define SHOW_FIELD(type, item, name) \ | 1179 | pos += snprintf(buf + pos, LEN_OR_ZERO, " %s=%%lx", |
1203 | do { \ | 1180 | tp->args[i].name); |
1204 | ret = trace_seq_printf(s, "\tfield:" #type " %s;\t" \ | 1181 | } |
1205 | "offset:%u;\tsize:%u;\tsigned:%d;\n", name,\ | ||
1206 | (unsigned int)offsetof(typeof(field), item),\ | ||
1207 | (unsigned int)sizeof(type), \ | ||
1208 | is_signed_type(type)); \ | ||
1209 | if (!ret) \ | ||
1210 | return 0; \ | ||
1211 | } while (0) | ||
1212 | 1182 | ||
1213 | static int kprobe_event_show_format(struct ftrace_event_call *call, | 1183 | pos += snprintf(buf + pos, LEN_OR_ZERO, "\", %s", arg); |
1214 | struct trace_seq *s) | ||
1215 | { | ||
1216 | struct kprobe_trace_entry field __attribute__((unused)); | ||
1217 | int ret, i; | ||
1218 | struct trace_probe *tp = (struct trace_probe *)call->data; | ||
1219 | 1184 | ||
1220 | SHOW_FIELD(unsigned long, ip, FIELD_STRING_IP); | 1185 | for (i = 0; i < tp->nr_args; i++) { |
1221 | SHOW_FIELD(int, nargs, FIELD_STRING_NARGS); | 1186 | pos += snprintf(buf + pos, LEN_OR_ZERO, ", REC->%s", |
1187 | tp->args[i].name); | ||
1188 | } | ||
1222 | 1189 | ||
1223 | /* Show fields */ | 1190 | #undef LEN_OR_ZERO |
1224 | for (i = 0; i < tp->nr_args; i++) | ||
1225 | SHOW_FIELD(unsigned long, args[i], tp->args[i].name); | ||
1226 | trace_seq_puts(s, "\n"); | ||
1227 | 1191 | ||
1228 | return __probe_event_show_format(s, tp, "(%lx)", | 1192 | /* return the length of print_fmt */ |
1229 | "REC->" FIELD_STRING_IP); | 1193 | return pos; |
1230 | } | 1194 | } |
1231 | 1195 | ||
1232 | static int kretprobe_event_show_format(struct ftrace_event_call *call, | 1196 | static int set_print_fmt(struct trace_probe *tp) |
1233 | struct trace_seq *s) | ||
1234 | { | 1197 | { |
1235 | struct kretprobe_trace_entry field __attribute__((unused)); | 1198 | int len; |
1236 | int ret, i; | 1199 | char *print_fmt; |
1237 | struct trace_probe *tp = (struct trace_probe *)call->data; | ||
1238 | 1200 | ||
1239 | SHOW_FIELD(unsigned long, func, FIELD_STRING_FUNC); | 1201 | /* First: called with 0 length to calculate the needed length */ |
1240 | SHOW_FIELD(unsigned long, ret_ip, FIELD_STRING_RETIP); | 1202 | len = __set_print_fmt(tp, NULL, 0); |
1241 | SHOW_FIELD(int, nargs, FIELD_STRING_NARGS); | 1203 | print_fmt = kmalloc(len + 1, GFP_KERNEL); |
1204 | if (!print_fmt) | ||
1205 | return -ENOMEM; | ||
1242 | 1206 | ||
1243 | /* Show fields */ | 1207 | /* Second: actually write the @print_fmt */ |
1244 | for (i = 0; i < tp->nr_args; i++) | 1208 | __set_print_fmt(tp, print_fmt, len + 1); |
1245 | SHOW_FIELD(unsigned long, args[i], tp->args[i].name); | 1209 | tp->call.print_fmt = print_fmt; |
1246 | trace_seq_puts(s, "\n"); | ||
1247 | 1210 | ||
1248 | return __probe_event_show_format(s, tp, "(%lx <- %lx)", | 1211 | return 0; |
1249 | "REC->" FIELD_STRING_FUNC | ||
1250 | ", REC->" FIELD_STRING_RETIP); | ||
1251 | } | 1212 | } |
1252 | 1213 | ||
1253 | #ifdef CONFIG_EVENT_PROFILE | 1214 | #ifdef CONFIG_PERF_EVENTS |
1254 | 1215 | ||
1255 | /* Kprobe profile handler */ | 1216 | /* Kprobe profile handler */ |
1256 | static __kprobes int kprobe_profile_func(struct kprobe *kp, | 1217 | static __kprobes void kprobe_profile_func(struct kprobe *kp, |
1257 | struct pt_regs *regs) | 1218 | struct pt_regs *regs) |
1258 | { | 1219 | { |
1259 | struct trace_probe *tp = container_of(kp, struct trace_probe, rp.kp); | 1220 | struct trace_probe *tp = container_of(kp, struct trace_probe, rp.kp); |
1260 | struct ftrace_event_call *call = &tp->call; | 1221 | struct ftrace_event_call *call = &tp->call; |
1261 | struct kprobe_trace_entry *entry; | 1222 | struct kprobe_trace_entry *entry; |
1262 | struct trace_entry *ent; | 1223 | int size, __size, i; |
1263 | int size, __size, i, pc, __cpu; | ||
1264 | unsigned long irq_flags; | 1224 | unsigned long irq_flags; |
1265 | char *trace_buf; | ||
1266 | char *raw_data; | ||
1267 | int rctx; | 1225 | int rctx; |
1268 | 1226 | ||
1269 | pc = preempt_count(); | ||
1270 | __size = SIZEOF_KPROBE_TRACE_ENTRY(tp->nr_args); | 1227 | __size = SIZEOF_KPROBE_TRACE_ENTRY(tp->nr_args); |
1271 | size = ALIGN(__size + sizeof(u32), sizeof(u64)); | 1228 | size = ALIGN(__size + sizeof(u32), sizeof(u64)); |
1272 | size -= sizeof(u32); | 1229 | size -= sizeof(u32); |
1273 | if (WARN_ONCE(size > FTRACE_MAX_PROFILE_SIZE, | 1230 | if (WARN_ONCE(size > FTRACE_MAX_PROFILE_SIZE, |
1274 | "profile buffer not large enough")) | 1231 | "profile buffer not large enough")) |
1275 | return 0; | 1232 | return; |
1276 | |||
1277 | /* | ||
1278 | * Protect the non nmi buffer | ||
1279 | * This also protects the rcu read side | ||
1280 | */ | ||
1281 | local_irq_save(irq_flags); | ||
1282 | |||
1283 | rctx = perf_swevent_get_recursion_context(); | ||
1284 | if (rctx < 0) | ||
1285 | goto end_recursion; | ||
1286 | |||
1287 | __cpu = smp_processor_id(); | ||
1288 | |||
1289 | if (in_nmi()) | ||
1290 | trace_buf = rcu_dereference(perf_trace_buf_nmi); | ||
1291 | else | ||
1292 | trace_buf = rcu_dereference(perf_trace_buf); | ||
1293 | 1233 | ||
1294 | if (!trace_buf) | 1234 | entry = ftrace_perf_buf_prepare(size, call->id, &rctx, &irq_flags); |
1295 | goto end; | 1235 | if (!entry) |
1296 | 1236 | return; | |
1297 | raw_data = per_cpu_ptr(trace_buf, __cpu); | ||
1298 | |||
1299 | /* Zero dead bytes from alignment to avoid buffer leak to userspace */ | ||
1300 | *(u64 *)(&raw_data[size - sizeof(u64)]) = 0ULL; | ||
1301 | entry = (struct kprobe_trace_entry *)raw_data; | ||
1302 | ent = &entry->ent; | ||
1303 | 1237 | ||
1304 | tracing_generic_entry_update(ent, irq_flags, pc); | ||
1305 | ent->type = call->id; | ||
1306 | entry->nargs = tp->nr_args; | 1238 | entry->nargs = tp->nr_args; |
1307 | entry->ip = (unsigned long)kp->addr; | 1239 | entry->ip = (unsigned long)kp->addr; |
1308 | for (i = 0; i < tp->nr_args; i++) | 1240 | for (i = 0; i < tp->nr_args; i++) |
1309 | entry->args[i] = call_fetch(&tp->args[i].fetch, regs); | 1241 | entry->args[i] = call_fetch(&tp->args[i].fetch, regs); |
1310 | perf_tp_event(call->id, entry->ip, 1, entry, size); | ||
1311 | |||
1312 | end: | ||
1313 | perf_swevent_put_recursion_context(rctx); | ||
1314 | end_recursion: | ||
1315 | local_irq_restore(irq_flags); | ||
1316 | 1242 | ||
1317 | return 0; | 1243 | ftrace_perf_buf_submit(entry, size, rctx, entry->ip, 1, irq_flags); |
1318 | } | 1244 | } |
1319 | 1245 | ||
1320 | /* Kretprobe profile handler */ | 1246 | /* Kretprobe profile handler */ |
1321 | static __kprobes int kretprobe_profile_func(struct kretprobe_instance *ri, | 1247 | static __kprobes void kretprobe_profile_func(struct kretprobe_instance *ri, |
1322 | struct pt_regs *regs) | 1248 | struct pt_regs *regs) |
1323 | { | 1249 | { |
1324 | struct trace_probe *tp = container_of(ri->rp, struct trace_probe, rp); | 1250 | struct trace_probe *tp = container_of(ri->rp, struct trace_probe, rp); |
1325 | struct ftrace_event_call *call = &tp->call; | 1251 | struct ftrace_event_call *call = &tp->call; |
1326 | struct kretprobe_trace_entry *entry; | 1252 | struct kretprobe_trace_entry *entry; |
1327 | struct trace_entry *ent; | 1253 | int size, __size, i; |
1328 | int size, __size, i, pc, __cpu; | ||
1329 | unsigned long irq_flags; | 1254 | unsigned long irq_flags; |
1330 | char *trace_buf; | ||
1331 | char *raw_data; | ||
1332 | int rctx; | 1255 | int rctx; |
1333 | 1256 | ||
1334 | pc = preempt_count(); | ||
1335 | __size = SIZEOF_KRETPROBE_TRACE_ENTRY(tp->nr_args); | 1257 | __size = SIZEOF_KRETPROBE_TRACE_ENTRY(tp->nr_args); |
1336 | size = ALIGN(__size + sizeof(u32), sizeof(u64)); | 1258 | size = ALIGN(__size + sizeof(u32), sizeof(u64)); |
1337 | size -= sizeof(u32); | 1259 | size -= sizeof(u32); |
1338 | if (WARN_ONCE(size > FTRACE_MAX_PROFILE_SIZE, | 1260 | if (WARN_ONCE(size > FTRACE_MAX_PROFILE_SIZE, |
1339 | "profile buffer not large enough")) | 1261 | "profile buffer not large enough")) |
1340 | return 0; | 1262 | return; |
1341 | |||
1342 | /* | ||
1343 | * Protect the non nmi buffer | ||
1344 | * This also protects the rcu read side | ||
1345 | */ | ||
1346 | local_irq_save(irq_flags); | ||
1347 | |||
1348 | rctx = perf_swevent_get_recursion_context(); | ||
1349 | if (rctx < 0) | ||
1350 | goto end_recursion; | ||
1351 | |||
1352 | __cpu = smp_processor_id(); | ||
1353 | 1263 | ||
1354 | if (in_nmi()) | 1264 | entry = ftrace_perf_buf_prepare(size, call->id, &rctx, &irq_flags); |
1355 | trace_buf = rcu_dereference(perf_trace_buf_nmi); | 1265 | if (!entry) |
1356 | else | 1266 | return; |
1357 | trace_buf = rcu_dereference(perf_trace_buf); | ||
1358 | |||
1359 | if (!trace_buf) | ||
1360 | goto end; | ||
1361 | |||
1362 | raw_data = per_cpu_ptr(trace_buf, __cpu); | ||
1363 | |||
1364 | /* Zero dead bytes from alignment to avoid buffer leak to userspace */ | ||
1365 | *(u64 *)(&raw_data[size - sizeof(u64)]) = 0ULL; | ||
1366 | entry = (struct kretprobe_trace_entry *)raw_data; | ||
1367 | ent = &entry->ent; | ||
1368 | 1267 | ||
1369 | tracing_generic_entry_update(ent, irq_flags, pc); | ||
1370 | ent->type = call->id; | ||
1371 | entry->nargs = tp->nr_args; | 1268 | entry->nargs = tp->nr_args; |
1372 | entry->func = (unsigned long)tp->rp.kp.addr; | 1269 | entry->func = (unsigned long)tp->rp.kp.addr; |
1373 | entry->ret_ip = (unsigned long)ri->ret_addr; | 1270 | entry->ret_ip = (unsigned long)ri->ret_addr; |
1374 | for (i = 0; i < tp->nr_args; i++) | 1271 | for (i = 0; i < tp->nr_args; i++) |
1375 | entry->args[i] = call_fetch(&tp->args[i].fetch, regs); | 1272 | entry->args[i] = call_fetch(&tp->args[i].fetch, regs); |
1376 | perf_tp_event(call->id, entry->ret_ip, 1, entry, size); | ||
1377 | |||
1378 | end: | ||
1379 | perf_swevent_put_recursion_context(rctx); | ||
1380 | end_recursion: | ||
1381 | local_irq_restore(irq_flags); | ||
1382 | 1273 | ||
1383 | return 0; | 1274 | ftrace_perf_buf_submit(entry, size, rctx, entry->ret_ip, 1, irq_flags); |
1384 | } | 1275 | } |
1385 | 1276 | ||
1386 | static int probe_profile_enable(struct ftrace_event_call *call) | 1277 | static int probe_profile_enable(struct ftrace_event_call *call) |
@@ -1408,7 +1299,7 @@ static void probe_profile_disable(struct ftrace_event_call *call) | |||
1408 | disable_kprobe(&tp->rp.kp); | 1299 | disable_kprobe(&tp->rp.kp); |
1409 | } | 1300 | } |
1410 | } | 1301 | } |
1411 | #endif /* CONFIG_EVENT_PROFILE */ | 1302 | #endif /* CONFIG_PERF_EVENTS */ |
1412 | 1303 | ||
1413 | 1304 | ||
1414 | static __kprobes | 1305 | static __kprobes |
@@ -1418,10 +1309,10 @@ int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs) | |||
1418 | 1309 | ||
1419 | if (tp->flags & TP_FLAG_TRACE) | 1310 | if (tp->flags & TP_FLAG_TRACE) |
1420 | kprobe_trace_func(kp, regs); | 1311 | kprobe_trace_func(kp, regs); |
1421 | #ifdef CONFIG_EVENT_PROFILE | 1312 | #ifdef CONFIG_PERF_EVENTS |
1422 | if (tp->flags & TP_FLAG_PROFILE) | 1313 | if (tp->flags & TP_FLAG_PROFILE) |
1423 | kprobe_profile_func(kp, regs); | 1314 | kprobe_profile_func(kp, regs); |
1424 | #endif /* CONFIG_EVENT_PROFILE */ | 1315 | #endif |
1425 | return 0; /* We don't tweek kernel, so just return 0 */ | 1316 | return 0; /* We don't tweek kernel, so just return 0 */ |
1426 | } | 1317 | } |
1427 | 1318 | ||
@@ -1432,10 +1323,10 @@ int kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs) | |||
1432 | 1323 | ||
1433 | if (tp->flags & TP_FLAG_TRACE) | 1324 | if (tp->flags & TP_FLAG_TRACE) |
1434 | kretprobe_trace_func(ri, regs); | 1325 | kretprobe_trace_func(ri, regs); |
1435 | #ifdef CONFIG_EVENT_PROFILE | 1326 | #ifdef CONFIG_PERF_EVENTS |
1436 | if (tp->flags & TP_FLAG_PROFILE) | 1327 | if (tp->flags & TP_FLAG_PROFILE) |
1437 | kretprobe_profile_func(ri, regs); | 1328 | kretprobe_profile_func(ri, regs); |
1438 | #endif /* CONFIG_EVENT_PROFILE */ | 1329 | #endif |
1439 | return 0; /* We don't tweek kernel, so just return 0 */ | 1330 | return 0; /* We don't tweek kernel, so just return 0 */ |
1440 | } | 1331 | } |
1441 | 1332 | ||
@@ -1448,23 +1339,25 @@ static int register_probe_event(struct trace_probe *tp) | |||
1448 | if (probe_is_return(tp)) { | 1339 | if (probe_is_return(tp)) { |
1449 | tp->event.trace = print_kretprobe_event; | 1340 | tp->event.trace = print_kretprobe_event; |
1450 | call->raw_init = probe_event_raw_init; | 1341 | call->raw_init = probe_event_raw_init; |
1451 | call->show_format = kretprobe_event_show_format; | ||
1452 | call->define_fields = kretprobe_event_define_fields; | 1342 | call->define_fields = kretprobe_event_define_fields; |
1453 | } else { | 1343 | } else { |
1454 | tp->event.trace = print_kprobe_event; | 1344 | tp->event.trace = print_kprobe_event; |
1455 | call->raw_init = probe_event_raw_init; | 1345 | call->raw_init = probe_event_raw_init; |
1456 | call->show_format = kprobe_event_show_format; | ||
1457 | call->define_fields = kprobe_event_define_fields; | 1346 | call->define_fields = kprobe_event_define_fields; |
1458 | } | 1347 | } |
1348 | if (set_print_fmt(tp) < 0) | ||
1349 | return -ENOMEM; | ||
1459 | call->event = &tp->event; | 1350 | call->event = &tp->event; |
1460 | call->id = register_ftrace_event(&tp->event); | 1351 | call->id = register_ftrace_event(&tp->event); |
1461 | if (!call->id) | 1352 | if (!call->id) { |
1353 | kfree(call->print_fmt); | ||
1462 | return -ENODEV; | 1354 | return -ENODEV; |
1355 | } | ||
1463 | call->enabled = 0; | 1356 | call->enabled = 0; |
1464 | call->regfunc = probe_event_enable; | 1357 | call->regfunc = probe_event_enable; |
1465 | call->unregfunc = probe_event_disable; | 1358 | call->unregfunc = probe_event_disable; |
1466 | 1359 | ||
1467 | #ifdef CONFIG_EVENT_PROFILE | 1360 | #ifdef CONFIG_PERF_EVENTS |
1468 | call->profile_enable = probe_profile_enable; | 1361 | call->profile_enable = probe_profile_enable; |
1469 | call->profile_disable = probe_profile_disable; | 1362 | call->profile_disable = probe_profile_disable; |
1470 | #endif | 1363 | #endif |
@@ -1472,6 +1365,7 @@ static int register_probe_event(struct trace_probe *tp) | |||
1472 | ret = trace_add_event_call(call); | 1365 | ret = trace_add_event_call(call); |
1473 | if (ret) { | 1366 | if (ret) { |
1474 | pr_info("Failed to register kprobe event: %s\n", call->name); | 1367 | pr_info("Failed to register kprobe event: %s\n", call->name); |
1368 | kfree(call->print_fmt); | ||
1475 | unregister_ftrace_event(&tp->event); | 1369 | unregister_ftrace_event(&tp->event); |
1476 | } | 1370 | } |
1477 | return ret; | 1371 | return ret; |
@@ -1481,6 +1375,7 @@ static void unregister_probe_event(struct trace_probe *tp) | |||
1481 | { | 1375 | { |
1482 | /* tp->event is unregistered in trace_remove_event_call() */ | 1376 | /* tp->event is unregistered in trace_remove_event_call() */ |
1483 | trace_remove_event_call(&tp->call); | 1377 | trace_remove_event_call(&tp->call); |
1378 | kfree(tp->call.print_fmt); | ||
1484 | } | 1379 | } |
1485 | 1380 | ||
1486 | /* Make a debugfs interface for controling probe points */ | 1381 | /* Make a debugfs interface for controling probe points */ |
@@ -1523,28 +1418,67 @@ static int kprobe_trace_selftest_target(int a1, int a2, int a3, | |||
1523 | 1418 | ||
1524 | static __init int kprobe_trace_self_tests_init(void) | 1419 | static __init int kprobe_trace_self_tests_init(void) |
1525 | { | 1420 | { |
1526 | int ret; | 1421 | int ret, warn = 0; |
1527 | int (*target)(int, int, int, int, int, int); | 1422 | int (*target)(int, int, int, int, int, int); |
1423 | struct trace_probe *tp; | ||
1528 | 1424 | ||
1529 | target = kprobe_trace_selftest_target; | 1425 | target = kprobe_trace_selftest_target; |
1530 | 1426 | ||
1531 | pr_info("Testing kprobe tracing: "); | 1427 | pr_info("Testing kprobe tracing: "); |
1532 | 1428 | ||
1533 | ret = command_trace_probe("p:testprobe kprobe_trace_selftest_target " | 1429 | ret = command_trace_probe("p:testprobe kprobe_trace_selftest_target " |
1534 | "$arg1 $arg2 $arg3 $arg4 $stack $stack0"); | 1430 | "$stack $stack0 +0($stack)"); |
1535 | if (WARN_ON_ONCE(ret)) | 1431 | if (WARN_ON_ONCE(ret)) { |
1536 | pr_warning("error enabling function entry\n"); | 1432 | pr_warning("error on probing function entry.\n"); |
1433 | warn++; | ||
1434 | } else { | ||
1435 | /* Enable trace point */ | ||
1436 | tp = find_probe_event("testprobe", KPROBE_EVENT_SYSTEM); | ||
1437 | if (WARN_ON_ONCE(tp == NULL)) { | ||
1438 | pr_warning("error on getting new probe.\n"); | ||
1439 | warn++; | ||
1440 | } else | ||
1441 | probe_event_enable(&tp->call); | ||
1442 | } | ||
1537 | 1443 | ||
1538 | ret = command_trace_probe("r:testprobe2 kprobe_trace_selftest_target " | 1444 | ret = command_trace_probe("r:testprobe2 kprobe_trace_selftest_target " |
1539 | "$retval"); | 1445 | "$retval"); |
1540 | if (WARN_ON_ONCE(ret)) | 1446 | if (WARN_ON_ONCE(ret)) { |
1541 | pr_warning("error enabling function return\n"); | 1447 | pr_warning("error on probing function return.\n"); |
1448 | warn++; | ||
1449 | } else { | ||
1450 | /* Enable trace point */ | ||
1451 | tp = find_probe_event("testprobe2", KPROBE_EVENT_SYSTEM); | ||
1452 | if (WARN_ON_ONCE(tp == NULL)) { | ||
1453 | pr_warning("error on getting new probe.\n"); | ||
1454 | warn++; | ||
1455 | } else | ||
1456 | probe_event_enable(&tp->call); | ||
1457 | } | ||
1458 | |||
1459 | if (warn) | ||
1460 | goto end; | ||
1542 | 1461 | ||
1543 | ret = target(1, 2, 3, 4, 5, 6); | 1462 | ret = target(1, 2, 3, 4, 5, 6); |
1544 | 1463 | ||
1545 | cleanup_all_probes(); | 1464 | ret = command_trace_probe("-:testprobe"); |
1465 | if (WARN_ON_ONCE(ret)) { | ||
1466 | pr_warning("error on deleting a probe.\n"); | ||
1467 | warn++; | ||
1468 | } | ||
1546 | 1469 | ||
1547 | pr_cont("OK\n"); | 1470 | ret = command_trace_probe("-:testprobe2"); |
1471 | if (WARN_ON_ONCE(ret)) { | ||
1472 | pr_warning("error on deleting a probe.\n"); | ||
1473 | warn++; | ||
1474 | } | ||
1475 | |||
1476 | end: | ||
1477 | cleanup_all_probes(); | ||
1478 | if (warn) | ||
1479 | pr_cont("NG: Some tests are failed. Please check them.\n"); | ||
1480 | else | ||
1481 | pr_cont("OK\n"); | ||
1548 | return 0; | 1482 | return 0; |
1549 | } | 1483 | } |
1550 | 1484 | ||
diff --git a/kernel/trace/trace_syscalls.c b/kernel/trace/trace_syscalls.c index 75289f372dd2..cba47d7935cc 100644 --- a/kernel/trace/trace_syscalls.c +++ b/kernel/trace/trace_syscalls.c | |||
@@ -143,70 +143,65 @@ extern char *__bad_type_size(void); | |||
143 | #type, #name, offsetof(typeof(trace), name), \ | 143 | #type, #name, offsetof(typeof(trace), name), \ |
144 | sizeof(trace.name), is_signed_type(type) | 144 | sizeof(trace.name), is_signed_type(type) |
145 | 145 | ||
146 | int syscall_enter_format(struct ftrace_event_call *call, struct trace_seq *s) | 146 | static |
147 | int __set_enter_print_fmt(struct syscall_metadata *entry, char *buf, int len) | ||
147 | { | 148 | { |
148 | int i; | 149 | int i; |
149 | int ret; | 150 | int pos = 0; |
150 | struct syscall_metadata *entry = call->data; | ||
151 | struct syscall_trace_enter trace; | ||
152 | int offset = offsetof(struct syscall_trace_enter, args); | ||
153 | 151 | ||
154 | ret = trace_seq_printf(s, "\tfield:%s %s;\toffset:%zu;\tsize:%zu;" | 152 | /* When len=0, we just calculate the needed length */ |
155 | "\tsigned:%u;\n", | 153 | #define LEN_OR_ZERO (len ? len - pos : 0) |
156 | SYSCALL_FIELD(int, nr)); | ||
157 | if (!ret) | ||
158 | return 0; | ||
159 | 154 | ||
155 | pos += snprintf(buf + pos, LEN_OR_ZERO, "\""); | ||
160 | for (i = 0; i < entry->nb_args; i++) { | 156 | for (i = 0; i < entry->nb_args; i++) { |
161 | ret = trace_seq_printf(s, "\tfield:%s %s;", entry->types[i], | 157 | pos += snprintf(buf + pos, LEN_OR_ZERO, "%s: 0x%%0%zulx%s", |
162 | entry->args[i]); | 158 | entry->args[i], sizeof(unsigned long), |
163 | if (!ret) | 159 | i == entry->nb_args - 1 ? "" : ", "); |
164 | return 0; | ||
165 | ret = trace_seq_printf(s, "\toffset:%d;\tsize:%zu;" | ||
166 | "\tsigned:%u;\n", offset, | ||
167 | sizeof(unsigned long), | ||
168 | is_signed_type(unsigned long)); | ||
169 | if (!ret) | ||
170 | return 0; | ||
171 | offset += sizeof(unsigned long); | ||
172 | } | 160 | } |
161 | pos += snprintf(buf + pos, LEN_OR_ZERO, "\""); | ||
173 | 162 | ||
174 | trace_seq_puts(s, "\nprint fmt: \""); | ||
175 | for (i = 0; i < entry->nb_args; i++) { | 163 | for (i = 0; i < entry->nb_args; i++) { |
176 | ret = trace_seq_printf(s, "%s: 0x%%0%zulx%s", entry->args[i], | 164 | pos += snprintf(buf + pos, LEN_OR_ZERO, |
177 | sizeof(unsigned long), | 165 | ", ((unsigned long)(REC->%s))", entry->args[i]); |
178 | i == entry->nb_args - 1 ? "" : ", "); | ||
179 | if (!ret) | ||
180 | return 0; | ||
181 | } | 166 | } |
182 | trace_seq_putc(s, '"'); | ||
183 | 167 | ||
184 | for (i = 0; i < entry->nb_args; i++) { | 168 | #undef LEN_OR_ZERO |
185 | ret = trace_seq_printf(s, ", ((unsigned long)(REC->%s))", | ||
186 | entry->args[i]); | ||
187 | if (!ret) | ||
188 | return 0; | ||
189 | } | ||
190 | 169 | ||
191 | return trace_seq_putc(s, '\n'); | 170 | /* return the length of print_fmt */ |
171 | return pos; | ||
192 | } | 172 | } |
193 | 173 | ||
194 | int syscall_exit_format(struct ftrace_event_call *call, struct trace_seq *s) | 174 | static int set_syscall_print_fmt(struct ftrace_event_call *call) |
195 | { | 175 | { |
196 | int ret; | 176 | char *print_fmt; |
197 | struct syscall_trace_exit trace; | 177 | int len; |
178 | struct syscall_metadata *entry = call->data; | ||
198 | 179 | ||
199 | ret = trace_seq_printf(s, | 180 | if (entry->enter_event != call) { |
200 | "\tfield:%s %s;\toffset:%zu;\tsize:%zu;" | 181 | call->print_fmt = "\"0x%lx\", REC->ret"; |
201 | "\tsigned:%u;\n" | ||
202 | "\tfield:%s %s;\toffset:%zu;\tsize:%zu;" | ||
203 | "\tsigned:%u;\n", | ||
204 | SYSCALL_FIELD(int, nr), | ||
205 | SYSCALL_FIELD(long, ret)); | ||
206 | if (!ret) | ||
207 | return 0; | 182 | return 0; |
183 | } | ||
184 | |||
185 | /* First: called with 0 length to calculate the needed length */ | ||
186 | len = __set_enter_print_fmt(entry, NULL, 0); | ||
187 | |||
188 | print_fmt = kmalloc(len + 1, GFP_KERNEL); | ||
189 | if (!print_fmt) | ||
190 | return -ENOMEM; | ||
191 | |||
192 | /* Second: actually write the @print_fmt */ | ||
193 | __set_enter_print_fmt(entry, print_fmt, len + 1); | ||
194 | call->print_fmt = print_fmt; | ||
208 | 195 | ||
209 | return trace_seq_printf(s, "\nprint fmt: \"0x%%lx\", REC->ret\n"); | 196 | return 0; |
197 | } | ||
198 | |||
199 | static void free_syscall_print_fmt(struct ftrace_event_call *call) | ||
200 | { | ||
201 | struct syscall_metadata *entry = call->data; | ||
202 | |||
203 | if (entry->enter_event == call) | ||
204 | kfree(call->print_fmt); | ||
210 | } | 205 | } |
211 | 206 | ||
212 | int syscall_enter_define_fields(struct ftrace_event_call *call) | 207 | int syscall_enter_define_fields(struct ftrace_event_call *call) |
@@ -386,12 +381,22 @@ int init_syscall_trace(struct ftrace_event_call *call) | |||
386 | { | 381 | { |
387 | int id; | 382 | int id; |
388 | 383 | ||
389 | id = register_ftrace_event(call->event); | 384 | if (set_syscall_print_fmt(call) < 0) |
390 | if (!id) | 385 | return -ENOMEM; |
391 | return -ENODEV; | 386 | |
392 | call->id = id; | 387 | id = trace_event_raw_init(call); |
393 | INIT_LIST_HEAD(&call->fields); | 388 | |
394 | return 0; | 389 | if (id < 0) { |
390 | free_syscall_print_fmt(call); | ||
391 | return id; | ||
392 | } | ||
393 | |||
394 | return id; | ||
395 | } | ||
396 | |||
397 | unsigned long __init arch_syscall_addr(int nr) | ||
398 | { | ||
399 | return (unsigned long)sys_call_table[nr]; | ||
395 | } | 400 | } |
396 | 401 | ||
397 | int __init init_ftrace_syscalls(void) | 402 | int __init init_ftrace_syscalls(void) |
@@ -421,7 +426,7 @@ int __init init_ftrace_syscalls(void) | |||
421 | } | 426 | } |
422 | core_initcall(init_ftrace_syscalls); | 427 | core_initcall(init_ftrace_syscalls); |
423 | 428 | ||
424 | #ifdef CONFIG_EVENT_PROFILE | 429 | #ifdef CONFIG_PERF_EVENTS |
425 | 430 | ||
426 | static DECLARE_BITMAP(enabled_prof_enter_syscalls, NR_syscalls); | 431 | static DECLARE_BITMAP(enabled_prof_enter_syscalls, NR_syscalls); |
427 | static DECLARE_BITMAP(enabled_prof_exit_syscalls, NR_syscalls); | 432 | static DECLARE_BITMAP(enabled_prof_exit_syscalls, NR_syscalls); |
@@ -433,12 +438,9 @@ static void prof_syscall_enter(struct pt_regs *regs, long id) | |||
433 | struct syscall_metadata *sys_data; | 438 | struct syscall_metadata *sys_data; |
434 | struct syscall_trace_enter *rec; | 439 | struct syscall_trace_enter *rec; |
435 | unsigned long flags; | 440 | unsigned long flags; |
436 | char *trace_buf; | ||
437 | char *raw_data; | ||
438 | int syscall_nr; | 441 | int syscall_nr; |
439 | int rctx; | 442 | int rctx; |
440 | int size; | 443 | int size; |
441 | int cpu; | ||
442 | 444 | ||
443 | syscall_nr = syscall_get_nr(current, regs); | 445 | syscall_nr = syscall_get_nr(current, regs); |
444 | if (!test_bit(syscall_nr, enabled_prof_enter_syscalls)) | 446 | if (!test_bit(syscall_nr, enabled_prof_enter_syscalls)) |
@@ -457,37 +459,15 @@ static void prof_syscall_enter(struct pt_regs *regs, long id) | |||
457 | "profile buffer not large enough")) | 459 | "profile buffer not large enough")) |
458 | return; | 460 | return; |
459 | 461 | ||
460 | /* Protect the per cpu buffer, begin the rcu read side */ | 462 | rec = (struct syscall_trace_enter *)ftrace_perf_buf_prepare(size, |
461 | local_irq_save(flags); | 463 | sys_data->enter_event->id, &rctx, &flags); |
462 | 464 | if (!rec) | |
463 | rctx = perf_swevent_get_recursion_context(); | 465 | return; |
464 | if (rctx < 0) | ||
465 | goto end_recursion; | ||
466 | |||
467 | cpu = smp_processor_id(); | ||
468 | |||
469 | trace_buf = rcu_dereference(perf_trace_buf); | ||
470 | |||
471 | if (!trace_buf) | ||
472 | goto end; | ||
473 | |||
474 | raw_data = per_cpu_ptr(trace_buf, cpu); | ||
475 | |||
476 | /* zero the dead bytes from align to not leak stack to user */ | ||
477 | *(u64 *)(&raw_data[size - sizeof(u64)]) = 0ULL; | ||
478 | 466 | ||
479 | rec = (struct syscall_trace_enter *) raw_data; | ||
480 | tracing_generic_entry_update(&rec->ent, 0, 0); | ||
481 | rec->ent.type = sys_data->enter_event->id; | ||
482 | rec->nr = syscall_nr; | 467 | rec->nr = syscall_nr; |
483 | syscall_get_arguments(current, regs, 0, sys_data->nb_args, | 468 | syscall_get_arguments(current, regs, 0, sys_data->nb_args, |
484 | (unsigned long *)&rec->args); | 469 | (unsigned long *)&rec->args); |
485 | perf_tp_event(sys_data->enter_event->id, 0, 1, rec, size); | 470 | ftrace_perf_buf_submit(rec, size, rctx, 0, 1, flags); |
486 | |||
487 | end: | ||
488 | perf_swevent_put_recursion_context(rctx); | ||
489 | end_recursion: | ||
490 | local_irq_restore(flags); | ||
491 | } | 471 | } |
492 | 472 | ||
493 | int prof_sysenter_enable(struct ftrace_event_call *call) | 473 | int prof_sysenter_enable(struct ftrace_event_call *call) |
@@ -531,11 +511,8 @@ static void prof_syscall_exit(struct pt_regs *regs, long ret) | |||
531 | struct syscall_trace_exit *rec; | 511 | struct syscall_trace_exit *rec; |
532 | unsigned long flags; | 512 | unsigned long flags; |
533 | int syscall_nr; | 513 | int syscall_nr; |
534 | char *trace_buf; | ||
535 | char *raw_data; | ||
536 | int rctx; | 514 | int rctx; |
537 | int size; | 515 | int size; |
538 | int cpu; | ||
539 | 516 | ||
540 | syscall_nr = syscall_get_nr(current, regs); | 517 | syscall_nr = syscall_get_nr(current, regs); |
541 | if (!test_bit(syscall_nr, enabled_prof_exit_syscalls)) | 518 | if (!test_bit(syscall_nr, enabled_prof_exit_syscalls)) |
@@ -557,38 +534,15 @@ static void prof_syscall_exit(struct pt_regs *regs, long ret) | |||
557 | "exit event has grown above profile buffer size")) | 534 | "exit event has grown above profile buffer size")) |
558 | return; | 535 | return; |
559 | 536 | ||
560 | /* Protect the per cpu buffer, begin the rcu read side */ | 537 | rec = (struct syscall_trace_exit *)ftrace_perf_buf_prepare(size, |
561 | local_irq_save(flags); | 538 | sys_data->exit_event->id, &rctx, &flags); |
562 | 539 | if (!rec) | |
563 | rctx = perf_swevent_get_recursion_context(); | 540 | return; |
564 | if (rctx < 0) | ||
565 | goto end_recursion; | ||
566 | |||
567 | cpu = smp_processor_id(); | ||
568 | |||
569 | trace_buf = rcu_dereference(perf_trace_buf); | ||
570 | |||
571 | if (!trace_buf) | ||
572 | goto end; | ||
573 | |||
574 | raw_data = per_cpu_ptr(trace_buf, cpu); | ||
575 | |||
576 | /* zero the dead bytes from align to not leak stack to user */ | ||
577 | *(u64 *)(&raw_data[size - sizeof(u64)]) = 0ULL; | ||
578 | |||
579 | rec = (struct syscall_trace_exit *)raw_data; | ||
580 | 541 | ||
581 | tracing_generic_entry_update(&rec->ent, 0, 0); | ||
582 | rec->ent.type = sys_data->exit_event->id; | ||
583 | rec->nr = syscall_nr; | 542 | rec->nr = syscall_nr; |
584 | rec->ret = syscall_get_return_value(current, regs); | 543 | rec->ret = syscall_get_return_value(current, regs); |
585 | 544 | ||
586 | perf_tp_event(sys_data->exit_event->id, 0, 1, rec, size); | 545 | ftrace_perf_buf_submit(rec, size, rctx, 0, 1, flags); |
587 | |||
588 | end: | ||
589 | perf_swevent_put_recursion_context(rctx); | ||
590 | end_recursion: | ||
591 | local_irq_restore(flags); | ||
592 | } | 546 | } |
593 | 547 | ||
594 | int prof_sysexit_enable(struct ftrace_event_call *call) | 548 | int prof_sysexit_enable(struct ftrace_event_call *call) |
@@ -603,7 +557,7 @@ int prof_sysexit_enable(struct ftrace_event_call *call) | |||
603 | ret = register_trace_sys_exit(prof_syscall_exit); | 557 | ret = register_trace_sys_exit(prof_syscall_exit); |
604 | if (ret) { | 558 | if (ret) { |
605 | pr_info("event trace: Could not activate" | 559 | pr_info("event trace: Could not activate" |
606 | "syscall entry trace point"); | 560 | "syscall exit trace point"); |
607 | } else { | 561 | } else { |
608 | set_bit(num, enabled_prof_exit_syscalls); | 562 | set_bit(num, enabled_prof_exit_syscalls); |
609 | sys_prof_refcount_exit++; | 563 | sys_prof_refcount_exit++; |
@@ -626,6 +580,5 @@ void prof_sysexit_disable(struct ftrace_event_call *call) | |||
626 | mutex_unlock(&syscall_trace_lock); | 580 | mutex_unlock(&syscall_trace_lock); |
627 | } | 581 | } |
628 | 582 | ||
629 | #endif | 583 | #endif /* CONFIG_PERF_EVENTS */ |
630 | |||
631 | 584 | ||