aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/trace/trace.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/trace/trace.c')
-rw-r--r--kernel/trace/trace.c3060
1 files changed, 1711 insertions, 1349 deletions
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index 17bb88d86ac2..a884c09006c4 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -11,32 +11,34 @@
11 * Copyright (C) 2004-2006 Ingo Molnar 11 * Copyright (C) 2004-2006 Ingo Molnar
12 * Copyright (C) 2004 William Lee Irwin III 12 * Copyright (C) 2004 William Lee Irwin III
13 */ 13 */
14#include <linux/ring_buffer.h>
14#include <linux/utsrelease.h> 15#include <linux/utsrelease.h>
16#include <linux/stacktrace.h>
17#include <linux/writeback.h>
15#include <linux/kallsyms.h> 18#include <linux/kallsyms.h>
16#include <linux/seq_file.h> 19#include <linux/seq_file.h>
17#include <linux/notifier.h> 20#include <linux/notifier.h>
21#include <linux/irqflags.h>
18#include <linux/debugfs.h> 22#include <linux/debugfs.h>
19#include <linux/pagemap.h> 23#include <linux/pagemap.h>
20#include <linux/hardirq.h> 24#include <linux/hardirq.h>
21#include <linux/linkage.h> 25#include <linux/linkage.h>
22#include <linux/uaccess.h> 26#include <linux/uaccess.h>
27#include <linux/kprobes.h>
23#include <linux/ftrace.h> 28#include <linux/ftrace.h>
24#include <linux/module.h> 29#include <linux/module.h>
25#include <linux/percpu.h> 30#include <linux/percpu.h>
31#include <linux/splice.h>
26#include <linux/kdebug.h> 32#include <linux/kdebug.h>
33#include <linux/string.h>
27#include <linux/ctype.h> 34#include <linux/ctype.h>
28#include <linux/init.h> 35#include <linux/init.h>
29#include <linux/poll.h> 36#include <linux/poll.h>
30#include <linux/gfp.h> 37#include <linux/gfp.h>
31#include <linux/fs.h> 38#include <linux/fs.h>
32#include <linux/kprobes.h>
33#include <linux/writeback.h>
34
35#include <linux/stacktrace.h>
36#include <linux/ring_buffer.h>
37#include <linux/irqflags.h>
38 39
39#include "trace.h" 40#include "trace.h"
41#include "trace_output.h"
40 42
41#define TRACE_BUFFER_FLAGS (RB_FL_OVERWRITE) 43#define TRACE_BUFFER_FLAGS (RB_FL_OVERWRITE)
42 44
@@ -44,14 +46,25 @@ unsigned long __read_mostly tracing_max_latency;
44unsigned long __read_mostly tracing_thresh; 46unsigned long __read_mostly tracing_thresh;
45 47
46/* 48/*
49 * On boot up, the ring buffer is set to the minimum size, so that
50 * we do not waste memory on systems that are not using tracing.
51 */
52static int ring_buffer_expanded;
53
54/*
47 * We need to change this state when a selftest is running. 55 * We need to change this state when a selftest is running.
48 * A selftest will lurk into the ring-buffer to count the 56 * A selftest will lurk into the ring-buffer to count the
49 * entries inserted during the selftest although some concurrent 57 * entries inserted during the selftest although some concurrent
50 * insertions into the ring-buffer such as ftrace_printk could occurred 58 * insertions into the ring-buffer such as trace_printk could occurred
51 * at the same time, giving false positive or negative results. 59 * at the same time, giving false positive or negative results.
52 */ 60 */
53static bool __read_mostly tracing_selftest_running; 61static bool __read_mostly tracing_selftest_running;
54 62
63/*
64 * If a tracer is running, we do not want to run SELFTEST.
65 */
66static bool __read_mostly tracing_selftest_disabled;
67
55/* For tracers that don't implement custom flags */ 68/* For tracers that don't implement custom flags */
56static struct tracer_opt dummy_tracer_opt[] = { 69static struct tracer_opt dummy_tracer_opt[] = {
57 { } 70 { }
@@ -73,7 +86,7 @@ static int dummy_set_flag(u32 old_flags, u32 bit, int set)
73 * of the tracer is successful. But that is the only place that sets 86 * of the tracer is successful. But that is the only place that sets
74 * this back to zero. 87 * this back to zero.
75 */ 88 */
76int tracing_disabled = 1; 89static int tracing_disabled = 1;
77 90
78static DEFINE_PER_CPU(local_t, ftrace_cpu_disabled); 91static DEFINE_PER_CPU(local_t, ftrace_cpu_disabled);
79 92
@@ -91,6 +104,9 @@ static inline void ftrace_enable_cpu(void)
91 104
92static cpumask_var_t __read_mostly tracing_buffer_mask; 105static cpumask_var_t __read_mostly tracing_buffer_mask;
93 106
107/* Define which cpu buffers are currently read in trace_pipe */
108static cpumask_var_t tracing_reader_cpumask;
109
94#define for_each_tracing_cpu(cpu) \ 110#define for_each_tracing_cpu(cpu) \
95 for_each_cpu(cpu, tracing_buffer_mask) 111 for_each_cpu(cpu, tracing_buffer_mask)
96 112
@@ -109,14 +125,21 @@ static cpumask_var_t __read_mostly tracing_buffer_mask;
109 */ 125 */
110int ftrace_dump_on_oops; 126int ftrace_dump_on_oops;
111 127
112static int tracing_set_tracer(char *buf); 128static int tracing_set_tracer(const char *buf);
129
130#define BOOTUP_TRACER_SIZE 100
131static char bootup_tracer_buf[BOOTUP_TRACER_SIZE] __initdata;
132static char *default_bootup_tracer;
113 133
114static int __init set_ftrace(char *str) 134static int __init set_ftrace(char *str)
115{ 135{
116 tracing_set_tracer(str); 136 strncpy(bootup_tracer_buf, str, BOOTUP_TRACER_SIZE);
137 default_bootup_tracer = bootup_tracer_buf;
138 /* We are using ftrace early, expand it */
139 ring_buffer_expanded = 1;
117 return 1; 140 return 1;
118} 141}
119__setup("ftrace", set_ftrace); 142__setup("ftrace=", set_ftrace);
120 143
121static int __init set_ftrace_dump_on_oops(char *str) 144static int __init set_ftrace_dump_on_oops(char *str)
122{ 145{
@@ -125,21 +148,13 @@ static int __init set_ftrace_dump_on_oops(char *str)
125} 148}
126__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops); 149__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
127 150
128long 151unsigned long long ns2usecs(cycle_t nsec)
129ns2usecs(cycle_t nsec)
130{ 152{
131 nsec += 500; 153 nsec += 500;
132 do_div(nsec, 1000); 154 do_div(nsec, 1000);
133 return nsec; 155 return nsec;
134} 156}
135 157
136cycle_t ftrace_now(int cpu)
137{
138 u64 ts = ring_buffer_time_stamp(cpu);
139 ring_buffer_normalize_time_stamp(cpu, &ts);
140 return ts;
141}
142
143/* 158/*
144 * The global_trace is the descriptor that holds the tracing 159 * The global_trace is the descriptor that holds the tracing
145 * buffers for the live tracing. For each CPU, it contains 160 * buffers for the live tracing. For each CPU, it contains
@@ -156,6 +171,20 @@ static struct trace_array global_trace;
156 171
157static DEFINE_PER_CPU(struct trace_array_cpu, global_trace_cpu); 172static DEFINE_PER_CPU(struct trace_array_cpu, global_trace_cpu);
158 173
174cycle_t ftrace_now(int cpu)
175{
176 u64 ts;
177
178 /* Early boot up does not have a buffer yet */
179 if (!global_trace.buffer)
180 return trace_clock_local();
181
182 ts = ring_buffer_time_stamp(global_trace.buffer, cpu);
183 ring_buffer_normalize_time_stamp(global_trace.buffer, cpu, &ts);
184
185 return ts;
186}
187
159/* 188/*
160 * The max_tr is used to snapshot the global_trace when a maximum 189 * The max_tr is used to snapshot the global_trace when a maximum
161 * latency is reached. Some tracers will use this to store a maximum 190 * latency is reached. Some tracers will use this to store a maximum
@@ -186,9 +215,6 @@ int tracing_is_enabled(void)
186 return tracer_enabled; 215 return tracer_enabled;
187} 216}
188 217
189/* function tracing enabled */
190int ftrace_function_enabled;
191
192/* 218/*
193 * trace_buf_size is the size in bytes that is allocated 219 * trace_buf_size is the size in bytes that is allocated
194 * for a buffer. Note, the number of bytes is always rounded 220 * for a buffer. Note, the number of bytes is always rounded
@@ -229,7 +255,7 @@ static DECLARE_WAIT_QUEUE_HEAD(trace_wait);
229 255
230/* trace_flags holds trace_options default values */ 256/* trace_flags holds trace_options default values */
231unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK | 257unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
232 TRACE_ITER_ANNOTATE; 258 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME;
233 259
234/** 260/**
235 * trace_wake_up - wake up tasks waiting for trace input 261 * trace_wake_up - wake up tasks waiting for trace input
@@ -280,13 +306,17 @@ static const char *trace_options[] = {
280 "block", 306 "block",
281 "stacktrace", 307 "stacktrace",
282 "sched-tree", 308 "sched-tree",
283 "ftrace_printk", 309 "trace_printk",
284 "ftrace_preempt", 310 "ftrace_preempt",
285 "branch", 311 "branch",
286 "annotate", 312 "annotate",
287 "userstacktrace", 313 "userstacktrace",
288 "sym-userobj", 314 "sym-userobj",
289 "printk-msg-only", 315 "printk-msg-only",
316 "context-info",
317 "latency-format",
318 "global-clock",
319 "sleep-time",
290 NULL 320 NULL
291}; 321};
292 322
@@ -326,146 +356,37 @@ __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
326 data->rt_priority = tsk->rt_priority; 356 data->rt_priority = tsk->rt_priority;
327 357
328 /* record this tasks comm */ 358 /* record this tasks comm */
329 tracing_record_cmdline(current); 359 tracing_record_cmdline(tsk);
330} 360}
331 361
332/** 362ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
333 * trace_seq_printf - sequence printing of trace information
334 * @s: trace sequence descriptor
335 * @fmt: printf format string
336 *
337 * The tracer may use either sequence operations or its own
338 * copy to user routines. To simplify formating of a trace
339 * trace_seq_printf is used to store strings into a special
340 * buffer (@s). Then the output may be either used by
341 * the sequencer or pulled into another buffer.
342 */
343int
344trace_seq_printf(struct trace_seq *s, const char *fmt, ...)
345{ 363{
346 int len = (PAGE_SIZE - 1) - s->len; 364 int len;
347 va_list ap;
348 int ret; 365 int ret;
349 366
350 if (!len) 367 if (!cnt)
351 return 0;
352
353 va_start(ap, fmt);
354 ret = vsnprintf(s->buffer + s->len, len, fmt, ap);
355 va_end(ap);
356
357 /* If we can't write it all, don't bother writing anything */
358 if (ret >= len)
359 return 0;
360
361 s->len += ret;
362
363 return len;
364}
365
366/**
367 * trace_seq_puts - trace sequence printing of simple string
368 * @s: trace sequence descriptor
369 * @str: simple string to record
370 *
371 * The tracer may use either the sequence operations or its own
372 * copy to user routines. This function records a simple string
373 * into a special buffer (@s) for later retrieval by a sequencer
374 * or other mechanism.
375 */
376static int
377trace_seq_puts(struct trace_seq *s, const char *str)
378{
379 int len = strlen(str);
380
381 if (len > ((PAGE_SIZE - 1) - s->len))
382 return 0;
383
384 memcpy(s->buffer + s->len, str, len);
385 s->len += len;
386
387 return len;
388}
389
390static int
391trace_seq_putc(struct trace_seq *s, unsigned char c)
392{
393 if (s->len >= (PAGE_SIZE - 1))
394 return 0;
395
396 s->buffer[s->len++] = c;
397
398 return 1;
399}
400
401static int
402trace_seq_putmem(struct trace_seq *s, void *mem, size_t len)
403{
404 if (len > ((PAGE_SIZE - 1) - s->len))
405 return 0; 368 return 0;
406 369
407 memcpy(s->buffer + s->len, mem, len); 370 if (s->len <= s->readpos)
408 s->len += len; 371 return -EBUSY;
409
410 return len;
411}
412
413#define MAX_MEMHEX_BYTES 8
414#define HEX_CHARS (MAX_MEMHEX_BYTES*2 + 1)
415
416static int
417trace_seq_putmem_hex(struct trace_seq *s, void *mem, size_t len)
418{
419 unsigned char hex[HEX_CHARS];
420 unsigned char *data = mem;
421 int i, j;
422
423#ifdef __BIG_ENDIAN
424 for (i = 0, j = 0; i < len; i++) {
425#else
426 for (i = len-1, j = 0; i >= 0; i--) {
427#endif
428 hex[j++] = hex_asc_hi(data[i]);
429 hex[j++] = hex_asc_lo(data[i]);
430 }
431 hex[j++] = ' ';
432
433 return trace_seq_putmem(s, hex, j);
434}
435
436static int
437trace_seq_path(struct trace_seq *s, struct path *path)
438{
439 unsigned char *p;
440 372
441 if (s->len >= (PAGE_SIZE - 1)) 373 len = s->len - s->readpos;
442 return 0; 374 if (cnt > len)
443 p = d_path(path, s->buffer + s->len, PAGE_SIZE - s->len); 375 cnt = len;
444 if (!IS_ERR(p)) { 376 ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
445 p = mangle_path(s->buffer + s->len, p, "\n"); 377 if (ret == cnt)
446 if (p) { 378 return -EFAULT;
447 s->len = p - s->buffer;
448 return 1;
449 }
450 } else {
451 s->buffer[s->len++] = '?';
452 return 1;
453 }
454 379
455 return 0; 380 cnt -= ret;
456}
457 381
458static void 382 s->readpos += cnt;
459trace_seq_reset(struct trace_seq *s) 383 return cnt;
460{
461 s->len = 0;
462 s->readpos = 0;
463} 384}
464 385
465ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt) 386static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
466{ 387{
467 int len; 388 int len;
468 int ret; 389 void *ret;
469 390
470 if (s->len <= s->readpos) 391 if (s->len <= s->readpos)
471 return -EBUSY; 392 return -EBUSY;
@@ -473,11 +394,11 @@ ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
473 len = s->len - s->readpos; 394 len = s->len - s->readpos;
474 if (cnt > len) 395 if (cnt > len)
475 cnt = len; 396 cnt = len;
476 ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt); 397 ret = memcpy(buf, s->buffer + s->readpos, cnt);
477 if (ret) 398 if (!ret)
478 return -EFAULT; 399 return -EFAULT;
479 400
480 s->readpos += len; 401 s->readpos += cnt;
481 return cnt; 402 return cnt;
482} 403}
483 404
@@ -489,7 +410,7 @@ trace_print_seq(struct seq_file *m, struct trace_seq *s)
489 s->buffer[len] = 0; 410 s->buffer[len] = 0;
490 seq_puts(m, s->buffer); 411 seq_puts(m, s->buffer);
491 412
492 trace_seq_reset(s); 413 trace_seq_init(s);
493} 414}
494 415
495/** 416/**
@@ -543,7 +464,7 @@ update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
543 464
544 ftrace_enable_cpu(); 465 ftrace_enable_cpu();
545 466
546 WARN_ON_ONCE(ret); 467 WARN_ON_ONCE(ret && ret != -EAGAIN);
547 468
548 __update_max_tr(tr, tsk, cpu); 469 __update_max_tr(tr, tsk, cpu);
549 __raw_spin_unlock(&ftrace_max_lock); 470 __raw_spin_unlock(&ftrace_max_lock);
@@ -556,6 +477,8 @@ update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
556 * Register a new plugin tracer. 477 * Register a new plugin tracer.
557 */ 478 */
558int register_tracer(struct tracer *type) 479int register_tracer(struct tracer *type)
480__releases(kernel_lock)
481__acquires(kernel_lock)
559{ 482{
560 struct tracer *t; 483 struct tracer *t;
561 int len; 484 int len;
@@ -594,9 +517,12 @@ int register_tracer(struct tracer *type)
594 else 517 else
595 if (!type->flags->opts) 518 if (!type->flags->opts)
596 type->flags->opts = dummy_tracer_opt; 519 type->flags->opts = dummy_tracer_opt;
520 if (!type->wait_pipe)
521 type->wait_pipe = default_wait_pipe;
522
597 523
598#ifdef CONFIG_FTRACE_STARTUP_TEST 524#ifdef CONFIG_FTRACE_STARTUP_TEST
599 if (type->selftest) { 525 if (type->selftest && !tracing_selftest_disabled) {
600 struct tracer *saved_tracer = current_trace; 526 struct tracer *saved_tracer = current_trace;
601 struct trace_array *tr = &global_trace; 527 struct trace_array *tr = &global_trace;
602 int i; 528 int i;
@@ -638,8 +564,26 @@ int register_tracer(struct tracer *type)
638 out: 564 out:
639 tracing_selftest_running = false; 565 tracing_selftest_running = false;
640 mutex_unlock(&trace_types_lock); 566 mutex_unlock(&trace_types_lock);
641 lock_kernel();
642 567
568 if (ret || !default_bootup_tracer)
569 goto out_unlock;
570
571 if (strncmp(default_bootup_tracer, type->name, BOOTUP_TRACER_SIZE))
572 goto out_unlock;
573
574 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
575 /* Do we want this tracer to start on bootup? */
576 tracing_set_tracer(type->name);
577 default_bootup_tracer = NULL;
578 /* disable other selftests, since this will break it. */
579 tracing_selftest_disabled = 1;
580#ifdef CONFIG_FTRACE_STARTUP_TEST
581 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
582 type->name);
583#endif
584
585 out_unlock:
586 lock_kernel();
643 return ret; 587 return ret;
644} 588}
645 589
@@ -658,6 +602,15 @@ void unregister_tracer(struct tracer *type)
658 602
659 found: 603 found:
660 *t = (*t)->next; 604 *t = (*t)->next;
605
606 if (type == current_trace && tracer_enabled) {
607 tracer_enabled = 0;
608 tracing_stop();
609 if (current_trace->stop)
610 current_trace->stop(&global_trace);
611 current_trace = &nop_trace;
612 }
613
661 if (strlen(type->name) != max_tracer_type_len) 614 if (strlen(type->name) != max_tracer_type_len)
662 goto out; 615 goto out;
663 616
@@ -689,19 +642,20 @@ void tracing_reset_online_cpus(struct trace_array *tr)
689} 642}
690 643
691#define SAVED_CMDLINES 128 644#define SAVED_CMDLINES 128
645#define NO_CMDLINE_MAP UINT_MAX
692static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1]; 646static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
693static unsigned map_cmdline_to_pid[SAVED_CMDLINES]; 647static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
694static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN]; 648static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
695static int cmdline_idx; 649static int cmdline_idx;
696static DEFINE_SPINLOCK(trace_cmdline_lock); 650static raw_spinlock_t trace_cmdline_lock = __RAW_SPIN_LOCK_UNLOCKED;
697 651
698/* temporary disable recording */ 652/* temporary disable recording */
699atomic_t trace_record_cmdline_disabled __read_mostly; 653static atomic_t trace_record_cmdline_disabled __read_mostly;
700 654
701static void trace_init_cmdlines(void) 655static void trace_init_cmdlines(void)
702{ 656{
703 memset(&map_pid_to_cmdline, -1, sizeof(map_pid_to_cmdline)); 657 memset(&map_pid_to_cmdline, NO_CMDLINE_MAP, sizeof(map_pid_to_cmdline));
704 memset(&map_cmdline_to_pid, -1, sizeof(map_cmdline_to_pid)); 658 memset(&map_cmdline_to_pid, NO_CMDLINE_MAP, sizeof(map_cmdline_to_pid));
705 cmdline_idx = 0; 659 cmdline_idx = 0;
706} 660}
707 661
@@ -738,13 +692,12 @@ void tracing_start(void)
738 return; 692 return;
739 693
740 spin_lock_irqsave(&tracing_start_lock, flags); 694 spin_lock_irqsave(&tracing_start_lock, flags);
741 if (--trace_stop_count) 695 if (--trace_stop_count) {
742 goto out; 696 if (trace_stop_count < 0) {
743 697 /* Someone screwed up their debugging */
744 if (trace_stop_count < 0) { 698 WARN_ON_ONCE(1);
745 /* Someone screwed up their debugging */ 699 trace_stop_count = 0;
746 WARN_ON_ONCE(1); 700 }
747 trace_stop_count = 0;
748 goto out; 701 goto out;
749 } 702 }
750 703
@@ -794,8 +747,7 @@ void trace_stop_cmdline_recording(void);
794 747
795static void trace_save_cmdline(struct task_struct *tsk) 748static void trace_save_cmdline(struct task_struct *tsk)
796{ 749{
797 unsigned map; 750 unsigned pid, idx;
798 unsigned idx;
799 751
800 if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT)) 752 if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
801 return; 753 return;
@@ -806,17 +758,24 @@ static void trace_save_cmdline(struct task_struct *tsk)
806 * nor do we want to disable interrupts, 758 * nor do we want to disable interrupts,
807 * so if we miss here, then better luck next time. 759 * so if we miss here, then better luck next time.
808 */ 760 */
809 if (!spin_trylock(&trace_cmdline_lock)) 761 if (!__raw_spin_trylock(&trace_cmdline_lock))
810 return; 762 return;
811 763
812 idx = map_pid_to_cmdline[tsk->pid]; 764 idx = map_pid_to_cmdline[tsk->pid];
813 if (idx >= SAVED_CMDLINES) { 765 if (idx == NO_CMDLINE_MAP) {
814 idx = (cmdline_idx + 1) % SAVED_CMDLINES; 766 idx = (cmdline_idx + 1) % SAVED_CMDLINES;
815 767
816 map = map_cmdline_to_pid[idx]; 768 /*
817 if (map <= PID_MAX_DEFAULT) 769 * Check whether the cmdline buffer at idx has a pid
818 map_pid_to_cmdline[map] = (unsigned)-1; 770 * mapped. We are going to overwrite that entry so we
771 * need to clear the map_pid_to_cmdline. Otherwise we
772 * would read the new comm for the old pid.
773 */
774 pid = map_cmdline_to_pid[idx];
775 if (pid != NO_CMDLINE_MAP)
776 map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
819 777
778 map_cmdline_to_pid[idx] = tsk->pid;
820 map_pid_to_cmdline[tsk->pid] = idx; 779 map_pid_to_cmdline[tsk->pid] = idx;
821 780
822 cmdline_idx = idx; 781 cmdline_idx = idx;
@@ -824,33 +783,37 @@ static void trace_save_cmdline(struct task_struct *tsk)
824 783
825 memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN); 784 memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
826 785
827 spin_unlock(&trace_cmdline_lock); 786 __raw_spin_unlock(&trace_cmdline_lock);
828} 787}
829 788
830char *trace_find_cmdline(int pid) 789void trace_find_cmdline(int pid, char comm[])
831{ 790{
832 char *cmdline = "<...>";
833 unsigned map; 791 unsigned map;
834 792
835 if (!pid) 793 if (!pid) {
836 return "<idle>"; 794 strcpy(comm, "<idle>");
795 return;
796 }
837 797
838 if (pid > PID_MAX_DEFAULT) 798 if (pid > PID_MAX_DEFAULT) {
839 goto out; 799 strcpy(comm, "<...>");
800 return;
801 }
840 802
803 __raw_spin_lock(&trace_cmdline_lock);
841 map = map_pid_to_cmdline[pid]; 804 map = map_pid_to_cmdline[pid];
842 if (map >= SAVED_CMDLINES) 805 if (map != NO_CMDLINE_MAP)
843 goto out; 806 strcpy(comm, saved_cmdlines[map]);
844 807 else
845 cmdline = saved_cmdlines[map]; 808 strcpy(comm, "<...>");
846 809
847 out: 810 __raw_spin_unlock(&trace_cmdline_lock);
848 return cmdline;
849} 811}
850 812
851void tracing_record_cmdline(struct task_struct *tsk) 813void tracing_record_cmdline(struct task_struct *tsk)
852{ 814{
853 if (atomic_read(&trace_record_cmdline_disabled)) 815 if (atomic_read(&trace_record_cmdline_disabled) || !tracer_enabled ||
816 !tracing_is_on())
854 return; 817 return;
855 818
856 trace_save_cmdline(tsk); 819 trace_save_cmdline(tsk);
@@ -864,7 +827,7 @@ tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
864 827
865 entry->preempt_count = pc & 0xff; 828 entry->preempt_count = pc & 0xff;
866 entry->pid = (tsk) ? tsk->pid : 0; 829 entry->pid = (tsk) ? tsk->pid : 0;
867 entry->tgid = (tsk) ? tsk->tgid : 0; 830 entry->tgid = (tsk) ? tsk->tgid : 0;
868 entry->flags = 831 entry->flags =
869#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT 832#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
870 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) | 833 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
@@ -876,78 +839,132 @@ tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
876 (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0); 839 (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
877} 840}
878 841
842struct ring_buffer_event *trace_buffer_lock_reserve(struct trace_array *tr,
843 unsigned char type,
844 unsigned long len,
845 unsigned long flags, int pc)
846{
847 struct ring_buffer_event *event;
848
849 event = ring_buffer_lock_reserve(tr->buffer, len);
850 if (event != NULL) {
851 struct trace_entry *ent = ring_buffer_event_data(event);
852
853 tracing_generic_entry_update(ent, flags, pc);
854 ent->type = type;
855 }
856
857 return event;
858}
859static void ftrace_trace_stack(struct trace_array *tr,
860 unsigned long flags, int skip, int pc);
861static void ftrace_trace_userstack(struct trace_array *tr,
862 unsigned long flags, int pc);
863
864static inline void __trace_buffer_unlock_commit(struct trace_array *tr,
865 struct ring_buffer_event *event,
866 unsigned long flags, int pc,
867 int wake)
868{
869 ring_buffer_unlock_commit(tr->buffer, event);
870
871 ftrace_trace_stack(tr, flags, 6, pc);
872 ftrace_trace_userstack(tr, flags, pc);
873
874 if (wake)
875 trace_wake_up();
876}
877
878void trace_buffer_unlock_commit(struct trace_array *tr,
879 struct ring_buffer_event *event,
880 unsigned long flags, int pc)
881{
882 __trace_buffer_unlock_commit(tr, event, flags, pc, 1);
883}
884
885struct ring_buffer_event *
886trace_current_buffer_lock_reserve(unsigned char type, unsigned long len,
887 unsigned long flags, int pc)
888{
889 return trace_buffer_lock_reserve(&global_trace,
890 type, len, flags, pc);
891}
892
893void trace_current_buffer_unlock_commit(struct ring_buffer_event *event,
894 unsigned long flags, int pc)
895{
896 return __trace_buffer_unlock_commit(&global_trace, event, flags, pc, 1);
897}
898
899void trace_nowake_buffer_unlock_commit(struct ring_buffer_event *event,
900 unsigned long flags, int pc)
901{
902 return __trace_buffer_unlock_commit(&global_trace, event, flags, pc, 0);
903}
904
879void 905void
880trace_function(struct trace_array *tr, struct trace_array_cpu *data, 906trace_function(struct trace_array *tr,
881 unsigned long ip, unsigned long parent_ip, unsigned long flags, 907 unsigned long ip, unsigned long parent_ip, unsigned long flags,
882 int pc) 908 int pc)
883{ 909{
884 struct ring_buffer_event *event; 910 struct ring_buffer_event *event;
885 struct ftrace_entry *entry; 911 struct ftrace_entry *entry;
886 unsigned long irq_flags;
887 912
888 /* If we are reading the ring buffer, don't trace */ 913 /* If we are reading the ring buffer, don't trace */
889 if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled)))) 914 if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled))))
890 return; 915 return;
891 916
892 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry), 917 event = trace_buffer_lock_reserve(tr, TRACE_FN, sizeof(*entry),
893 &irq_flags); 918 flags, pc);
894 if (!event) 919 if (!event)
895 return; 920 return;
896 entry = ring_buffer_event_data(event); 921 entry = ring_buffer_event_data(event);
897 tracing_generic_entry_update(&entry->ent, flags, pc);
898 entry->ent.type = TRACE_FN;
899 entry->ip = ip; 922 entry->ip = ip;
900 entry->parent_ip = parent_ip; 923 entry->parent_ip = parent_ip;
901 ring_buffer_unlock_commit(tr->buffer, event, irq_flags); 924 ring_buffer_unlock_commit(tr->buffer, event);
902} 925}
903 926
904#ifdef CONFIG_FUNCTION_GRAPH_TRACER 927#ifdef CONFIG_FUNCTION_GRAPH_TRACER
905static void __trace_graph_entry(struct trace_array *tr, 928static int __trace_graph_entry(struct trace_array *tr,
906 struct trace_array_cpu *data,
907 struct ftrace_graph_ent *trace, 929 struct ftrace_graph_ent *trace,
908 unsigned long flags, 930 unsigned long flags,
909 int pc) 931 int pc)
910{ 932{
911 struct ring_buffer_event *event; 933 struct ring_buffer_event *event;
912 struct ftrace_graph_ent_entry *entry; 934 struct ftrace_graph_ent_entry *entry;
913 unsigned long irq_flags;
914 935
915 if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled)))) 936 if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled))))
916 return; 937 return 0;
917 938
918 event = ring_buffer_lock_reserve(global_trace.buffer, sizeof(*entry), 939 event = trace_buffer_lock_reserve(&global_trace, TRACE_GRAPH_ENT,
919 &irq_flags); 940 sizeof(*entry), flags, pc);
920 if (!event) 941 if (!event)
921 return; 942 return 0;
922 entry = ring_buffer_event_data(event); 943 entry = ring_buffer_event_data(event);
923 tracing_generic_entry_update(&entry->ent, flags, pc);
924 entry->ent.type = TRACE_GRAPH_ENT;
925 entry->graph_ent = *trace; 944 entry->graph_ent = *trace;
926 ring_buffer_unlock_commit(global_trace.buffer, event, irq_flags); 945 ring_buffer_unlock_commit(global_trace.buffer, event);
946
947 return 1;
927} 948}
928 949
929static void __trace_graph_return(struct trace_array *tr, 950static void __trace_graph_return(struct trace_array *tr,
930 struct trace_array_cpu *data,
931 struct ftrace_graph_ret *trace, 951 struct ftrace_graph_ret *trace,
932 unsigned long flags, 952 unsigned long flags,
933 int pc) 953 int pc)
934{ 954{
935 struct ring_buffer_event *event; 955 struct ring_buffer_event *event;
936 struct ftrace_graph_ret_entry *entry; 956 struct ftrace_graph_ret_entry *entry;
937 unsigned long irq_flags;
938 957
939 if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled)))) 958 if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled))))
940 return; 959 return;
941 960
942 event = ring_buffer_lock_reserve(global_trace.buffer, sizeof(*entry), 961 event = trace_buffer_lock_reserve(&global_trace, TRACE_GRAPH_RET,
943 &irq_flags); 962 sizeof(*entry), flags, pc);
944 if (!event) 963 if (!event)
945 return; 964 return;
946 entry = ring_buffer_event_data(event); 965 entry = ring_buffer_event_data(event);
947 tracing_generic_entry_update(&entry->ent, flags, pc);
948 entry->ent.type = TRACE_GRAPH_RET;
949 entry->ret = *trace; 966 entry->ret = *trace;
950 ring_buffer_unlock_commit(global_trace.buffer, event, irq_flags); 967 ring_buffer_unlock_commit(global_trace.buffer, event);
951} 968}
952#endif 969#endif
953 970
@@ -957,31 +974,23 @@ ftrace(struct trace_array *tr, struct trace_array_cpu *data,
957 int pc) 974 int pc)
958{ 975{
959 if (likely(!atomic_read(&data->disabled))) 976 if (likely(!atomic_read(&data->disabled)))
960 trace_function(tr, data, ip, parent_ip, flags, pc); 977 trace_function(tr, ip, parent_ip, flags, pc);
961} 978}
962 979
963static void ftrace_trace_stack(struct trace_array *tr, 980static void __ftrace_trace_stack(struct trace_array *tr,
964 struct trace_array_cpu *data, 981 unsigned long flags,
965 unsigned long flags, 982 int skip, int pc)
966 int skip, int pc)
967{ 983{
968#ifdef CONFIG_STACKTRACE 984#ifdef CONFIG_STACKTRACE
969 struct ring_buffer_event *event; 985 struct ring_buffer_event *event;
970 struct stack_entry *entry; 986 struct stack_entry *entry;
971 struct stack_trace trace; 987 struct stack_trace trace;
972 unsigned long irq_flags;
973 988
974 if (!(trace_flags & TRACE_ITER_STACKTRACE)) 989 event = trace_buffer_lock_reserve(tr, TRACE_STACK,
975 return; 990 sizeof(*entry), flags, pc);
976
977 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
978 &irq_flags);
979 if (!event) 991 if (!event)
980 return; 992 return;
981 entry = ring_buffer_event_data(event); 993 entry = ring_buffer_event_data(event);
982 tracing_generic_entry_update(&entry->ent, flags, pc);
983 entry->ent.type = TRACE_STACK;
984
985 memset(&entry->caller, 0, sizeof(entry->caller)); 994 memset(&entry->caller, 0, sizeof(entry->caller));
986 995
987 trace.nr_entries = 0; 996 trace.nr_entries = 0;
@@ -990,38 +999,43 @@ static void ftrace_trace_stack(struct trace_array *tr,
990 trace.entries = entry->caller; 999 trace.entries = entry->caller;
991 1000
992 save_stack_trace(&trace); 1001 save_stack_trace(&trace);
993 ring_buffer_unlock_commit(tr->buffer, event, irq_flags); 1002 ring_buffer_unlock_commit(tr->buffer, event);
994#endif 1003#endif
995} 1004}
996 1005
1006static void ftrace_trace_stack(struct trace_array *tr,
1007 unsigned long flags,
1008 int skip, int pc)
1009{
1010 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1011 return;
1012
1013 __ftrace_trace_stack(tr, flags, skip, pc);
1014}
1015
997void __trace_stack(struct trace_array *tr, 1016void __trace_stack(struct trace_array *tr,
998 struct trace_array_cpu *data,
999 unsigned long flags, 1017 unsigned long flags,
1000 int skip) 1018 int skip, int pc)
1001{ 1019{
1002 ftrace_trace_stack(tr, data, flags, skip, preempt_count()); 1020 __ftrace_trace_stack(tr, flags, skip, pc);
1003} 1021}
1004 1022
1005static void ftrace_trace_userstack(struct trace_array *tr, 1023static void ftrace_trace_userstack(struct trace_array *tr,
1006 struct trace_array_cpu *data, 1024 unsigned long flags, int pc)
1007 unsigned long flags, int pc)
1008{ 1025{
1009#ifdef CONFIG_STACKTRACE 1026#ifdef CONFIG_STACKTRACE
1010 struct ring_buffer_event *event; 1027 struct ring_buffer_event *event;
1011 struct userstack_entry *entry; 1028 struct userstack_entry *entry;
1012 struct stack_trace trace; 1029 struct stack_trace trace;
1013 unsigned long irq_flags;
1014 1030
1015 if (!(trace_flags & TRACE_ITER_USERSTACKTRACE)) 1031 if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1016 return; 1032 return;
1017 1033
1018 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry), 1034 event = trace_buffer_lock_reserve(tr, TRACE_USER_STACK,
1019 &irq_flags); 1035 sizeof(*entry), flags, pc);
1020 if (!event) 1036 if (!event)
1021 return; 1037 return;
1022 entry = ring_buffer_event_data(event); 1038 entry = ring_buffer_event_data(event);
1023 tracing_generic_entry_update(&entry->ent, flags, pc);
1024 entry->ent.type = TRACE_USER_STACK;
1025 1039
1026 memset(&entry->caller, 0, sizeof(entry->caller)); 1040 memset(&entry->caller, 0, sizeof(entry->caller));
1027 1041
@@ -1031,70 +1045,58 @@ static void ftrace_trace_userstack(struct trace_array *tr,
1031 trace.entries = entry->caller; 1045 trace.entries = entry->caller;
1032 1046
1033 save_stack_trace_user(&trace); 1047 save_stack_trace_user(&trace);
1034 ring_buffer_unlock_commit(tr->buffer, event, irq_flags); 1048 ring_buffer_unlock_commit(tr->buffer, event);
1035#endif 1049#endif
1036} 1050}
1037 1051
1038void __trace_userstack(struct trace_array *tr, 1052#ifdef UNUSED
1039 struct trace_array_cpu *data, 1053static void __trace_userstack(struct trace_array *tr, unsigned long flags)
1040 unsigned long flags)
1041{ 1054{
1042 ftrace_trace_userstack(tr, data, flags, preempt_count()); 1055 ftrace_trace_userstack(tr, flags, preempt_count());
1043} 1056}
1057#endif /* UNUSED */
1044 1058
1045static void 1059static void
1046ftrace_trace_special(void *__tr, void *__data, 1060ftrace_trace_special(void *__tr,
1047 unsigned long arg1, unsigned long arg2, unsigned long arg3, 1061 unsigned long arg1, unsigned long arg2, unsigned long arg3,
1048 int pc) 1062 int pc)
1049{ 1063{
1050 struct ring_buffer_event *event; 1064 struct ring_buffer_event *event;
1051 struct trace_array_cpu *data = __data;
1052 struct trace_array *tr = __tr; 1065 struct trace_array *tr = __tr;
1053 struct special_entry *entry; 1066 struct special_entry *entry;
1054 unsigned long irq_flags;
1055 1067
1056 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry), 1068 event = trace_buffer_lock_reserve(tr, TRACE_SPECIAL,
1057 &irq_flags); 1069 sizeof(*entry), 0, pc);
1058 if (!event) 1070 if (!event)
1059 return; 1071 return;
1060 entry = ring_buffer_event_data(event); 1072 entry = ring_buffer_event_data(event);
1061 tracing_generic_entry_update(&entry->ent, 0, pc);
1062 entry->ent.type = TRACE_SPECIAL;
1063 entry->arg1 = arg1; 1073 entry->arg1 = arg1;
1064 entry->arg2 = arg2; 1074 entry->arg2 = arg2;
1065 entry->arg3 = arg3; 1075 entry->arg3 = arg3;
1066 ring_buffer_unlock_commit(tr->buffer, event, irq_flags); 1076 trace_buffer_unlock_commit(tr, event, 0, pc);
1067 ftrace_trace_stack(tr, data, irq_flags, 4, pc);
1068 ftrace_trace_userstack(tr, data, irq_flags, pc);
1069
1070 trace_wake_up();
1071} 1077}
1072 1078
1073void 1079void
1074__trace_special(void *__tr, void *__data, 1080__trace_special(void *__tr, void *__data,
1075 unsigned long arg1, unsigned long arg2, unsigned long arg3) 1081 unsigned long arg1, unsigned long arg2, unsigned long arg3)
1076{ 1082{
1077 ftrace_trace_special(__tr, __data, arg1, arg2, arg3, preempt_count()); 1083 ftrace_trace_special(__tr, arg1, arg2, arg3, preempt_count());
1078} 1084}
1079 1085
1080void 1086void
1081tracing_sched_switch_trace(struct trace_array *tr, 1087tracing_sched_switch_trace(struct trace_array *tr,
1082 struct trace_array_cpu *data,
1083 struct task_struct *prev, 1088 struct task_struct *prev,
1084 struct task_struct *next, 1089 struct task_struct *next,
1085 unsigned long flags, int pc) 1090 unsigned long flags, int pc)
1086{ 1091{
1087 struct ring_buffer_event *event; 1092 struct ring_buffer_event *event;
1088 struct ctx_switch_entry *entry; 1093 struct ctx_switch_entry *entry;
1089 unsigned long irq_flags;
1090 1094
1091 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry), 1095 event = trace_buffer_lock_reserve(tr, TRACE_CTX,
1092 &irq_flags); 1096 sizeof(*entry), flags, pc);
1093 if (!event) 1097 if (!event)
1094 return; 1098 return;
1095 entry = ring_buffer_event_data(event); 1099 entry = ring_buffer_event_data(event);
1096 tracing_generic_entry_update(&entry->ent, flags, pc);
1097 entry->ent.type = TRACE_CTX;
1098 entry->prev_pid = prev->pid; 1100 entry->prev_pid = prev->pid;
1099 entry->prev_prio = prev->prio; 1101 entry->prev_prio = prev->prio;
1100 entry->prev_state = prev->state; 1102 entry->prev_state = prev->state;
@@ -1102,29 +1104,23 @@ tracing_sched_switch_trace(struct trace_array *tr,
1102 entry->next_prio = next->prio; 1104 entry->next_prio = next->prio;
1103 entry->next_state = next->state; 1105 entry->next_state = next->state;
1104 entry->next_cpu = task_cpu(next); 1106 entry->next_cpu = task_cpu(next);
1105 ring_buffer_unlock_commit(tr->buffer, event, irq_flags); 1107 trace_buffer_unlock_commit(tr, event, flags, pc);
1106 ftrace_trace_stack(tr, data, flags, 5, pc);
1107 ftrace_trace_userstack(tr, data, flags, pc);
1108} 1108}
1109 1109
1110void 1110void
1111tracing_sched_wakeup_trace(struct trace_array *tr, 1111tracing_sched_wakeup_trace(struct trace_array *tr,
1112 struct trace_array_cpu *data,
1113 struct task_struct *wakee, 1112 struct task_struct *wakee,
1114 struct task_struct *curr, 1113 struct task_struct *curr,
1115 unsigned long flags, int pc) 1114 unsigned long flags, int pc)
1116{ 1115{
1117 struct ring_buffer_event *event; 1116 struct ring_buffer_event *event;
1118 struct ctx_switch_entry *entry; 1117 struct ctx_switch_entry *entry;
1119 unsigned long irq_flags;
1120 1118
1121 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry), 1119 event = trace_buffer_lock_reserve(tr, TRACE_WAKE,
1122 &irq_flags); 1120 sizeof(*entry), flags, pc);
1123 if (!event) 1121 if (!event)
1124 return; 1122 return;
1125 entry = ring_buffer_event_data(event); 1123 entry = ring_buffer_event_data(event);
1126 tracing_generic_entry_update(&entry->ent, flags, pc);
1127 entry->ent.type = TRACE_WAKE;
1128 entry->prev_pid = curr->pid; 1124 entry->prev_pid = curr->pid;
1129 entry->prev_prio = curr->prio; 1125 entry->prev_prio = curr->prio;
1130 entry->prev_state = curr->state; 1126 entry->prev_state = curr->state;
@@ -1132,11 +1128,10 @@ tracing_sched_wakeup_trace(struct trace_array *tr,
1132 entry->next_prio = wakee->prio; 1128 entry->next_prio = wakee->prio;
1133 entry->next_state = wakee->state; 1129 entry->next_state = wakee->state;
1134 entry->next_cpu = task_cpu(wakee); 1130 entry->next_cpu = task_cpu(wakee);
1135 ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
1136 ftrace_trace_stack(tr, data, flags, 6, pc);
1137 ftrace_trace_userstack(tr, data, flags, pc);
1138 1131
1139 trace_wake_up(); 1132 ring_buffer_unlock_commit(tr->buffer, event);
1133 ftrace_trace_stack(tr, flags, 6, pc);
1134 ftrace_trace_userstack(tr, flags, pc);
1140} 1135}
1141 1136
1142void 1137void
@@ -1157,66 +1152,7 @@ ftrace_special(unsigned long arg1, unsigned long arg2, unsigned long arg3)
1157 data = tr->data[cpu]; 1152 data = tr->data[cpu];
1158 1153
1159 if (likely(atomic_inc_return(&data->disabled) == 1)) 1154 if (likely(atomic_inc_return(&data->disabled) == 1))
1160 ftrace_trace_special(tr, data, arg1, arg2, arg3, pc); 1155 ftrace_trace_special(tr, arg1, arg2, arg3, pc);
1161
1162 atomic_dec(&data->disabled);
1163 local_irq_restore(flags);
1164}
1165
1166#ifdef CONFIG_FUNCTION_TRACER
1167static void
1168function_trace_call_preempt_only(unsigned long ip, unsigned long parent_ip)
1169{
1170 struct trace_array *tr = &global_trace;
1171 struct trace_array_cpu *data;
1172 unsigned long flags;
1173 long disabled;
1174 int cpu, resched;
1175 int pc;
1176
1177 if (unlikely(!ftrace_function_enabled))
1178 return;
1179
1180 pc = preempt_count();
1181 resched = ftrace_preempt_disable();
1182 local_save_flags(flags);
1183 cpu = raw_smp_processor_id();
1184 data = tr->data[cpu];
1185 disabled = atomic_inc_return(&data->disabled);
1186
1187 if (likely(disabled == 1))
1188 trace_function(tr, data, ip, parent_ip, flags, pc);
1189
1190 atomic_dec(&data->disabled);
1191 ftrace_preempt_enable(resched);
1192}
1193
1194static void
1195function_trace_call(unsigned long ip, unsigned long parent_ip)
1196{
1197 struct trace_array *tr = &global_trace;
1198 struct trace_array_cpu *data;
1199 unsigned long flags;
1200 long disabled;
1201 int cpu;
1202 int pc;
1203
1204 if (unlikely(!ftrace_function_enabled))
1205 return;
1206
1207 /*
1208 * Need to use raw, since this must be called before the
1209 * recursive protection is performed.
1210 */
1211 local_irq_save(flags);
1212 cpu = raw_smp_processor_id();
1213 data = tr->data[cpu];
1214 disabled = atomic_inc_return(&data->disabled);
1215
1216 if (likely(disabled == 1)) {
1217 pc = preempt_count();
1218 trace_function(tr, data, ip, parent_ip, flags, pc);
1219 }
1220 1156
1221 atomic_dec(&data->disabled); 1157 atomic_dec(&data->disabled);
1222 local_irq_restore(flags); 1158 local_irq_restore(flags);
@@ -1229,6 +1165,7 @@ int trace_graph_entry(struct ftrace_graph_ent *trace)
1229 struct trace_array_cpu *data; 1165 struct trace_array_cpu *data;
1230 unsigned long flags; 1166 unsigned long flags;
1231 long disabled; 1167 long disabled;
1168 int ret;
1232 int cpu; 1169 int cpu;
1233 int pc; 1170 int pc;
1234 1171
@@ -1244,15 +1181,18 @@ int trace_graph_entry(struct ftrace_graph_ent *trace)
1244 disabled = atomic_inc_return(&data->disabled); 1181 disabled = atomic_inc_return(&data->disabled);
1245 if (likely(disabled == 1)) { 1182 if (likely(disabled == 1)) {
1246 pc = preempt_count(); 1183 pc = preempt_count();
1247 __trace_graph_entry(tr, data, trace, flags, pc); 1184 ret = __trace_graph_entry(tr, trace, flags, pc);
1185 } else {
1186 ret = 0;
1248 } 1187 }
1249 /* Only do the atomic if it is not already set */ 1188 /* Only do the atomic if it is not already set */
1250 if (!test_tsk_trace_graph(current)) 1189 if (!test_tsk_trace_graph(current))
1251 set_tsk_trace_graph(current); 1190 set_tsk_trace_graph(current);
1191
1252 atomic_dec(&data->disabled); 1192 atomic_dec(&data->disabled);
1253 local_irq_restore(flags); 1193 local_irq_restore(flags);
1254 1194
1255 return 1; 1195 return ret;
1256} 1196}
1257 1197
1258void trace_graph_return(struct ftrace_graph_ret *trace) 1198void trace_graph_return(struct ftrace_graph_ret *trace)
@@ -1270,7 +1210,7 @@ void trace_graph_return(struct ftrace_graph_ret *trace)
1270 disabled = atomic_inc_return(&data->disabled); 1210 disabled = atomic_inc_return(&data->disabled);
1271 if (likely(disabled == 1)) { 1211 if (likely(disabled == 1)) {
1272 pc = preempt_count(); 1212 pc = preempt_count();
1273 __trace_graph_return(tr, data, trace, flags, pc); 1213 __trace_graph_return(tr, trace, flags, pc);
1274 } 1214 }
1275 if (!trace->depth) 1215 if (!trace->depth)
1276 clear_tsk_trace_graph(current); 1216 clear_tsk_trace_graph(current);
@@ -1279,30 +1219,122 @@ void trace_graph_return(struct ftrace_graph_ret *trace)
1279} 1219}
1280#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 1220#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
1281 1221
1282static struct ftrace_ops trace_ops __read_mostly =
1283{
1284 .func = function_trace_call,
1285};
1286 1222
1287void tracing_start_function_trace(void) 1223/**
1224 * trace_vbprintk - write binary msg to tracing buffer
1225 *
1226 */
1227int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
1288{ 1228{
1289 ftrace_function_enabled = 0; 1229 static raw_spinlock_t trace_buf_lock =
1230 (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
1231 static u32 trace_buf[TRACE_BUF_SIZE];
1290 1232
1291 if (trace_flags & TRACE_ITER_PREEMPTONLY) 1233 struct ring_buffer_event *event;
1292 trace_ops.func = function_trace_call_preempt_only; 1234 struct trace_array *tr = &global_trace;
1293 else 1235 struct trace_array_cpu *data;
1294 trace_ops.func = function_trace_call; 1236 struct bprint_entry *entry;
1237 unsigned long flags;
1238 int resched;
1239 int cpu, len = 0, size, pc;
1240
1241 if (unlikely(tracing_selftest_running || tracing_disabled))
1242 return 0;
1243
1244 /* Don't pollute graph traces with trace_vprintk internals */
1245 pause_graph_tracing();
1246
1247 pc = preempt_count();
1248 resched = ftrace_preempt_disable();
1249 cpu = raw_smp_processor_id();
1250 data = tr->data[cpu];
1251
1252 if (unlikely(atomic_read(&data->disabled)))
1253 goto out;
1254
1255 /* Lockdep uses trace_printk for lock tracing */
1256 local_irq_save(flags);
1257 __raw_spin_lock(&trace_buf_lock);
1258 len = vbin_printf(trace_buf, TRACE_BUF_SIZE, fmt, args);
1259
1260 if (len > TRACE_BUF_SIZE || len < 0)
1261 goto out_unlock;
1262
1263 size = sizeof(*entry) + sizeof(u32) * len;
1264 event = trace_buffer_lock_reserve(tr, TRACE_BPRINT, size, flags, pc);
1265 if (!event)
1266 goto out_unlock;
1267 entry = ring_buffer_event_data(event);
1268 entry->ip = ip;
1269 entry->fmt = fmt;
1270
1271 memcpy(entry->buf, trace_buf, sizeof(u32) * len);
1272 ring_buffer_unlock_commit(tr->buffer, event);
1273
1274out_unlock:
1275 __raw_spin_unlock(&trace_buf_lock);
1276 local_irq_restore(flags);
1277
1278out:
1279 ftrace_preempt_enable(resched);
1280 unpause_graph_tracing();
1295 1281
1296 register_ftrace_function(&trace_ops); 1282 return len;
1297 ftrace_function_enabled = 1;
1298} 1283}
1284EXPORT_SYMBOL_GPL(trace_vbprintk);
1299 1285
1300void tracing_stop_function_trace(void) 1286int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
1301{ 1287{
1302 ftrace_function_enabled = 0; 1288 static raw_spinlock_t trace_buf_lock = __RAW_SPIN_LOCK_UNLOCKED;
1303 unregister_ftrace_function(&trace_ops); 1289 static char trace_buf[TRACE_BUF_SIZE];
1290
1291 struct ring_buffer_event *event;
1292 struct trace_array *tr = &global_trace;
1293 struct trace_array_cpu *data;
1294 int cpu, len = 0, size, pc;
1295 struct print_entry *entry;
1296 unsigned long irq_flags;
1297
1298 if (tracing_disabled || tracing_selftest_running)
1299 return 0;
1300
1301 pc = preempt_count();
1302 preempt_disable_notrace();
1303 cpu = raw_smp_processor_id();
1304 data = tr->data[cpu];
1305
1306 if (unlikely(atomic_read(&data->disabled)))
1307 goto out;
1308
1309 pause_graph_tracing();
1310 raw_local_irq_save(irq_flags);
1311 __raw_spin_lock(&trace_buf_lock);
1312 len = vsnprintf(trace_buf, TRACE_BUF_SIZE, fmt, args);
1313
1314 len = min(len, TRACE_BUF_SIZE-1);
1315 trace_buf[len] = 0;
1316
1317 size = sizeof(*entry) + len + 1;
1318 event = trace_buffer_lock_reserve(tr, TRACE_PRINT, size, irq_flags, pc);
1319 if (!event)
1320 goto out_unlock;
1321 entry = ring_buffer_event_data(event);
1322 entry->ip = ip;
1323
1324 memcpy(&entry->buf, trace_buf, len);
1325 entry->buf[len] = 0;
1326 ring_buffer_unlock_commit(tr->buffer, event);
1327
1328 out_unlock:
1329 __raw_spin_unlock(&trace_buf_lock);
1330 raw_local_irq_restore(irq_flags);
1331 unpause_graph_tracing();
1332 out:
1333 preempt_enable_notrace();
1334
1335 return len;
1304} 1336}
1305#endif 1337EXPORT_SYMBOL_GPL(trace_vprintk);
1306 1338
1307enum trace_file_type { 1339enum trace_file_type {
1308 TRACE_FILE_LAT_FMT = 1, 1340 TRACE_FILE_LAT_FMT = 1,
@@ -1345,10 +1377,25 @@ __find_next_entry(struct trace_iterator *iter, int *ent_cpu, u64 *ent_ts)
1345{ 1377{
1346 struct ring_buffer *buffer = iter->tr->buffer; 1378 struct ring_buffer *buffer = iter->tr->buffer;
1347 struct trace_entry *ent, *next = NULL; 1379 struct trace_entry *ent, *next = NULL;
1380 int cpu_file = iter->cpu_file;
1348 u64 next_ts = 0, ts; 1381 u64 next_ts = 0, ts;
1349 int next_cpu = -1; 1382 int next_cpu = -1;
1350 int cpu; 1383 int cpu;
1351 1384
1385 /*
1386 * If we are in a per_cpu trace file, don't bother by iterating over
1387 * all cpu and peek directly.
1388 */
1389 if (cpu_file > TRACE_PIPE_ALL_CPU) {
1390 if (ring_buffer_empty_cpu(buffer, cpu_file))
1391 return NULL;
1392 ent = peek_next_entry(iter, cpu_file, ent_ts);
1393 if (ent_cpu)
1394 *ent_cpu = cpu_file;
1395
1396 return ent;
1397 }
1398
1352 for_each_tracing_cpu(cpu) { 1399 for_each_tracing_cpu(cpu) {
1353 1400
1354 if (ring_buffer_empty_cpu(buffer, cpu)) 1401 if (ring_buffer_empty_cpu(buffer, cpu))
@@ -1376,8 +1423,8 @@ __find_next_entry(struct trace_iterator *iter, int *ent_cpu, u64 *ent_ts)
1376} 1423}
1377 1424
1378/* Find the next real entry, without updating the iterator itself */ 1425/* Find the next real entry, without updating the iterator itself */
1379static struct trace_entry * 1426struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
1380find_next_entry(struct trace_iterator *iter, int *ent_cpu, u64 *ent_ts) 1427 int *ent_cpu, u64 *ent_ts)
1381{ 1428{
1382 return __find_next_entry(iter, ent_cpu, ent_ts); 1429 return __find_next_entry(iter, ent_cpu, ent_ts);
1383} 1430}
@@ -1426,19 +1473,32 @@ static void *s_next(struct seq_file *m, void *v, loff_t *pos)
1426 return ent; 1473 return ent;
1427} 1474}
1428 1475
1476/*
1477 * No necessary locking here. The worst thing which can
1478 * happen is loosing events consumed at the same time
1479 * by a trace_pipe reader.
1480 * Other than that, we don't risk to crash the ring buffer
1481 * because it serializes the readers.
1482 *
1483 * The current tracer is copied to avoid a global locking
1484 * all around.
1485 */
1429static void *s_start(struct seq_file *m, loff_t *pos) 1486static void *s_start(struct seq_file *m, loff_t *pos)
1430{ 1487{
1431 struct trace_iterator *iter = m->private; 1488 struct trace_iterator *iter = m->private;
1489 static struct tracer *old_tracer;
1490 int cpu_file = iter->cpu_file;
1432 void *p = NULL; 1491 void *p = NULL;
1433 loff_t l = 0; 1492 loff_t l = 0;
1434 int cpu; 1493 int cpu;
1435 1494
1495 /* copy the tracer to avoid using a global lock all around */
1436 mutex_lock(&trace_types_lock); 1496 mutex_lock(&trace_types_lock);
1437 1497 if (unlikely(old_tracer != current_trace && current_trace)) {
1438 if (!current_trace || current_trace != iter->trace) { 1498 old_tracer = current_trace;
1439 mutex_unlock(&trace_types_lock); 1499 *iter->trace = *current_trace;
1440 return NULL;
1441 } 1500 }
1501 mutex_unlock(&trace_types_lock);
1442 1502
1443 atomic_inc(&trace_record_cmdline_disabled); 1503 atomic_inc(&trace_record_cmdline_disabled);
1444 1504
@@ -1449,9 +1509,12 @@ static void *s_start(struct seq_file *m, loff_t *pos)
1449 1509
1450 ftrace_disable_cpu(); 1510 ftrace_disable_cpu();
1451 1511
1452 for_each_tracing_cpu(cpu) { 1512 if (cpu_file == TRACE_PIPE_ALL_CPU) {
1453 ring_buffer_iter_reset(iter->buffer_iter[cpu]); 1513 for_each_tracing_cpu(cpu)
1454 } 1514 ring_buffer_iter_reset(iter->buffer_iter[cpu]);
1515 } else
1516 ring_buffer_iter_reset(iter->buffer_iter[cpu_file]);
1517
1455 1518
1456 ftrace_enable_cpu(); 1519 ftrace_enable_cpu();
1457 1520
@@ -1469,155 +1532,6 @@ static void *s_start(struct seq_file *m, loff_t *pos)
1469static void s_stop(struct seq_file *m, void *p) 1532static void s_stop(struct seq_file *m, void *p)
1470{ 1533{
1471 atomic_dec(&trace_record_cmdline_disabled); 1534 atomic_dec(&trace_record_cmdline_disabled);
1472 mutex_unlock(&trace_types_lock);
1473}
1474
1475#ifdef CONFIG_KRETPROBES
1476static inline const char *kretprobed(const char *name)
1477{
1478 static const char tramp_name[] = "kretprobe_trampoline";
1479 int size = sizeof(tramp_name);
1480
1481 if (strncmp(tramp_name, name, size) == 0)
1482 return "[unknown/kretprobe'd]";
1483 return name;
1484}
1485#else
1486static inline const char *kretprobed(const char *name)
1487{
1488 return name;
1489}
1490#endif /* CONFIG_KRETPROBES */
1491
1492static int
1493seq_print_sym_short(struct trace_seq *s, const char *fmt, unsigned long address)
1494{
1495#ifdef CONFIG_KALLSYMS
1496 char str[KSYM_SYMBOL_LEN];
1497 const char *name;
1498
1499 kallsyms_lookup(address, NULL, NULL, NULL, str);
1500
1501 name = kretprobed(str);
1502
1503 return trace_seq_printf(s, fmt, name);
1504#endif
1505 return 1;
1506}
1507
1508static int
1509seq_print_sym_offset(struct trace_seq *s, const char *fmt,
1510 unsigned long address)
1511{
1512#ifdef CONFIG_KALLSYMS
1513 char str[KSYM_SYMBOL_LEN];
1514 const char *name;
1515
1516 sprint_symbol(str, address);
1517 name = kretprobed(str);
1518
1519 return trace_seq_printf(s, fmt, name);
1520#endif
1521 return 1;
1522}
1523
1524#ifndef CONFIG_64BIT
1525# define IP_FMT "%08lx"
1526#else
1527# define IP_FMT "%016lx"
1528#endif
1529
1530int
1531seq_print_ip_sym(struct trace_seq *s, unsigned long ip, unsigned long sym_flags)
1532{
1533 int ret;
1534
1535 if (!ip)
1536 return trace_seq_printf(s, "0");
1537
1538 if (sym_flags & TRACE_ITER_SYM_OFFSET)
1539 ret = seq_print_sym_offset(s, "%s", ip);
1540 else
1541 ret = seq_print_sym_short(s, "%s", ip);
1542
1543 if (!ret)
1544 return 0;
1545
1546 if (sym_flags & TRACE_ITER_SYM_ADDR)
1547 ret = trace_seq_printf(s, " <" IP_FMT ">", ip);
1548 return ret;
1549}
1550
1551static inline int seq_print_user_ip(struct trace_seq *s, struct mm_struct *mm,
1552 unsigned long ip, unsigned long sym_flags)
1553{
1554 struct file *file = NULL;
1555 unsigned long vmstart = 0;
1556 int ret = 1;
1557
1558 if (mm) {
1559 const struct vm_area_struct *vma;
1560
1561 down_read(&mm->mmap_sem);
1562 vma = find_vma(mm, ip);
1563 if (vma) {
1564 file = vma->vm_file;
1565 vmstart = vma->vm_start;
1566 }
1567 if (file) {
1568 ret = trace_seq_path(s, &file->f_path);
1569 if (ret)
1570 ret = trace_seq_printf(s, "[+0x%lx]", ip - vmstart);
1571 }
1572 up_read(&mm->mmap_sem);
1573 }
1574 if (ret && ((sym_flags & TRACE_ITER_SYM_ADDR) || !file))
1575 ret = trace_seq_printf(s, " <" IP_FMT ">", ip);
1576 return ret;
1577}
1578
1579static int
1580seq_print_userip_objs(const struct userstack_entry *entry, struct trace_seq *s,
1581 unsigned long sym_flags)
1582{
1583 struct mm_struct *mm = NULL;
1584 int ret = 1;
1585 unsigned int i;
1586
1587 if (trace_flags & TRACE_ITER_SYM_USEROBJ) {
1588 struct task_struct *task;
1589 /*
1590 * we do the lookup on the thread group leader,
1591 * since individual threads might have already quit!
1592 */
1593 rcu_read_lock();
1594 task = find_task_by_vpid(entry->ent.tgid);
1595 if (task)
1596 mm = get_task_mm(task);
1597 rcu_read_unlock();
1598 }
1599
1600 for (i = 0; i < FTRACE_STACK_ENTRIES; i++) {
1601 unsigned long ip = entry->caller[i];
1602
1603 if (ip == ULONG_MAX || !ret)
1604 break;
1605 if (i && ret)
1606 ret = trace_seq_puts(s, " <- ");
1607 if (!ip) {
1608 if (ret)
1609 ret = trace_seq_puts(s, "??");
1610 continue;
1611 }
1612 if (!ret)
1613 break;
1614 if (ret)
1615 ret = seq_print_user_ip(s, mm, ip, sym_flags);
1616 }
1617
1618 if (mm)
1619 mmput(mm);
1620 return ret;
1621} 1535}
1622 1536
1623static void print_lat_help_header(struct seq_file *m) 1537static void print_lat_help_header(struct seq_file *m)
@@ -1658,11 +1572,11 @@ print_trace_header(struct seq_file *m, struct trace_iterator *iter)
1658 total = entries + 1572 total = entries +
1659 ring_buffer_overruns(iter->tr->buffer); 1573 ring_buffer_overruns(iter->tr->buffer);
1660 1574
1661 seq_printf(m, "%s latency trace v1.1.5 on %s\n", 1575 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
1662 name, UTS_RELEASE); 1576 name, UTS_RELEASE);
1663 seq_puts(m, "-----------------------------------" 1577 seq_puts(m, "# -----------------------------------"
1664 "---------------------------------\n"); 1578 "---------------------------------\n");
1665 seq_printf(m, " latency: %lu us, #%lu/%lu, CPU#%d |" 1579 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
1666 " (M:%s VP:%d, KP:%d, SP:%d HP:%d", 1580 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
1667 nsecs_to_usecs(data->saved_latency), 1581 nsecs_to_usecs(data->saved_latency),
1668 entries, 1582 entries,
@@ -1684,121 +1598,24 @@ print_trace_header(struct seq_file *m, struct trace_iterator *iter)
1684#else 1598#else
1685 seq_puts(m, ")\n"); 1599 seq_puts(m, ")\n");
1686#endif 1600#endif
1687 seq_puts(m, " -----------------\n"); 1601 seq_puts(m, "# -----------------\n");
1688 seq_printf(m, " | task: %.16s-%d " 1602 seq_printf(m, "# | task: %.16s-%d "
1689 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n", 1603 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
1690 data->comm, data->pid, data->uid, data->nice, 1604 data->comm, data->pid, data->uid, data->nice,
1691 data->policy, data->rt_priority); 1605 data->policy, data->rt_priority);
1692 seq_puts(m, " -----------------\n"); 1606 seq_puts(m, "# -----------------\n");
1693 1607
1694 if (data->critical_start) { 1608 if (data->critical_start) {
1695 seq_puts(m, " => started at: "); 1609 seq_puts(m, "# => started at: ");
1696 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags); 1610 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
1697 trace_print_seq(m, &iter->seq); 1611 trace_print_seq(m, &iter->seq);
1698 seq_puts(m, "\n => ended at: "); 1612 seq_puts(m, "\n# => ended at: ");
1699 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags); 1613 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
1700 trace_print_seq(m, &iter->seq); 1614 trace_print_seq(m, &iter->seq);
1701 seq_puts(m, "\n"); 1615 seq_puts(m, "#\n");
1702 }
1703
1704 seq_puts(m, "\n");
1705}
1706
1707static void
1708lat_print_generic(struct trace_seq *s, struct trace_entry *entry, int cpu)
1709{
1710 int hardirq, softirq;
1711 char *comm;
1712
1713 comm = trace_find_cmdline(entry->pid);
1714
1715 trace_seq_printf(s, "%8.8s-%-5d ", comm, entry->pid);
1716 trace_seq_printf(s, "%3d", cpu);
1717 trace_seq_printf(s, "%c%c",
1718 (entry->flags & TRACE_FLAG_IRQS_OFF) ? 'd' :
1719 (entry->flags & TRACE_FLAG_IRQS_NOSUPPORT) ? 'X' : '.',
1720 ((entry->flags & TRACE_FLAG_NEED_RESCHED) ? 'N' : '.'));
1721
1722 hardirq = entry->flags & TRACE_FLAG_HARDIRQ;
1723 softirq = entry->flags & TRACE_FLAG_SOFTIRQ;
1724 if (hardirq && softirq) {
1725 trace_seq_putc(s, 'H');
1726 } else {
1727 if (hardirq) {
1728 trace_seq_putc(s, 'h');
1729 } else {
1730 if (softirq)
1731 trace_seq_putc(s, 's');
1732 else
1733 trace_seq_putc(s, '.');
1734 }
1735 }
1736
1737 if (entry->preempt_count)
1738 trace_seq_printf(s, "%x", entry->preempt_count);
1739 else
1740 trace_seq_puts(s, ".");
1741}
1742
1743unsigned long preempt_mark_thresh = 100;
1744
1745static void
1746lat_print_timestamp(struct trace_seq *s, u64 abs_usecs,
1747 unsigned long rel_usecs)
1748{
1749 trace_seq_printf(s, " %4lldus", abs_usecs);
1750 if (rel_usecs > preempt_mark_thresh)
1751 trace_seq_puts(s, "!: ");
1752 else if (rel_usecs > 1)
1753 trace_seq_puts(s, "+: ");
1754 else
1755 trace_seq_puts(s, " : ");
1756}
1757
1758static const char state_to_char[] = TASK_STATE_TO_CHAR_STR;
1759
1760static int task_state_char(unsigned long state)
1761{
1762 int bit = state ? __ffs(state) + 1 : 0;
1763
1764 return bit < sizeof(state_to_char) - 1 ? state_to_char[bit] : '?';
1765}
1766
1767/*
1768 * The message is supposed to contain an ending newline.
1769 * If the printing stops prematurely, try to add a newline of our own.
1770 */
1771void trace_seq_print_cont(struct trace_seq *s, struct trace_iterator *iter)
1772{
1773 struct trace_entry *ent;
1774 struct trace_field_cont *cont;
1775 bool ok = true;
1776
1777 ent = peek_next_entry(iter, iter->cpu, NULL);
1778 if (!ent || ent->type != TRACE_CONT) {
1779 trace_seq_putc(s, '\n');
1780 return;
1781 } 1616 }
1782 1617
1783 do { 1618 seq_puts(m, "#\n");
1784 cont = (struct trace_field_cont *)ent;
1785 if (ok)
1786 ok = (trace_seq_printf(s, "%s", cont->buf) > 0);
1787
1788 ftrace_disable_cpu();
1789
1790 if (iter->buffer_iter[iter->cpu])
1791 ring_buffer_read(iter->buffer_iter[iter->cpu], NULL);
1792 else
1793 ring_buffer_consume(iter->tr->buffer, iter->cpu, NULL);
1794
1795 ftrace_enable_cpu();
1796
1797 ent = peek_next_entry(iter, iter->cpu, NULL);
1798 } while (ent && ent->type == TRACE_CONT);
1799
1800 if (!ok)
1801 trace_seq_putc(s, '\n');
1802} 1619}
1803 1620
1804static void test_cpu_buff_start(struct trace_iterator *iter) 1621static void test_cpu_buff_start(struct trace_iterator *iter)
@@ -1815,141 +1632,11 @@ static void test_cpu_buff_start(struct trace_iterator *iter)
1815 return; 1632 return;
1816 1633
1817 cpumask_set_cpu(iter->cpu, iter->started); 1634 cpumask_set_cpu(iter->cpu, iter->started);
1818 trace_seq_printf(s, "##### CPU %u buffer started ####\n", iter->cpu);
1819}
1820
1821static enum print_line_t
1822print_lat_fmt(struct trace_iterator *iter, unsigned int trace_idx, int cpu)
1823{
1824 struct trace_seq *s = &iter->seq;
1825 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1826 struct trace_entry *next_entry;
1827 unsigned long verbose = (trace_flags & TRACE_ITER_VERBOSE);
1828 struct trace_entry *entry = iter->ent;
1829 unsigned long abs_usecs;
1830 unsigned long rel_usecs;
1831 u64 next_ts;
1832 char *comm;
1833 int S, T;
1834 int i;
1835
1836 if (entry->type == TRACE_CONT)
1837 return TRACE_TYPE_HANDLED;
1838
1839 test_cpu_buff_start(iter);
1840
1841 next_entry = find_next_entry(iter, NULL, &next_ts);
1842 if (!next_entry)
1843 next_ts = iter->ts;
1844 rel_usecs = ns2usecs(next_ts - iter->ts);
1845 abs_usecs = ns2usecs(iter->ts - iter->tr->time_start);
1846
1847 if (verbose) {
1848 comm = trace_find_cmdline(entry->pid);
1849 trace_seq_printf(s, "%16s %5d %3d %d %08x %08x [%08lx]"
1850 " %ld.%03ldms (+%ld.%03ldms): ",
1851 comm,
1852 entry->pid, cpu, entry->flags,
1853 entry->preempt_count, trace_idx,
1854 ns2usecs(iter->ts),
1855 abs_usecs/1000,
1856 abs_usecs % 1000, rel_usecs/1000,
1857 rel_usecs % 1000);
1858 } else {
1859 lat_print_generic(s, entry, cpu);
1860 lat_print_timestamp(s, abs_usecs, rel_usecs);
1861 }
1862 switch (entry->type) {
1863 case TRACE_FN: {
1864 struct ftrace_entry *field;
1865
1866 trace_assign_type(field, entry);
1867
1868 seq_print_ip_sym(s, field->ip, sym_flags);
1869 trace_seq_puts(s, " (");
1870 seq_print_ip_sym(s, field->parent_ip, sym_flags);
1871 trace_seq_puts(s, ")\n");
1872 break;
1873 }
1874 case TRACE_CTX:
1875 case TRACE_WAKE: {
1876 struct ctx_switch_entry *field;
1877
1878 trace_assign_type(field, entry);
1879
1880 T = task_state_char(field->next_state);
1881 S = task_state_char(field->prev_state);
1882 comm = trace_find_cmdline(field->next_pid);
1883 trace_seq_printf(s, " %5d:%3d:%c %s [%03d] %5d:%3d:%c %s\n",
1884 field->prev_pid,
1885 field->prev_prio,
1886 S, entry->type == TRACE_CTX ? "==>" : " +",
1887 field->next_cpu,
1888 field->next_pid,
1889 field->next_prio,
1890 T, comm);
1891 break;
1892 }
1893 case TRACE_SPECIAL: {
1894 struct special_entry *field;
1895
1896 trace_assign_type(field, entry);
1897
1898 trace_seq_printf(s, "# %ld %ld %ld\n",
1899 field->arg1,
1900 field->arg2,
1901 field->arg3);
1902 break;
1903 }
1904 case TRACE_STACK: {
1905 struct stack_entry *field;
1906
1907 trace_assign_type(field, entry);
1908
1909 for (i = 0; i < FTRACE_STACK_ENTRIES; i++) {
1910 if (i)
1911 trace_seq_puts(s, " <= ");
1912 seq_print_ip_sym(s, field->caller[i], sym_flags);
1913 }
1914 trace_seq_puts(s, "\n");
1915 break;
1916 }
1917 case TRACE_PRINT: {
1918 struct print_entry *field;
1919
1920 trace_assign_type(field, entry);
1921
1922 seq_print_ip_sym(s, field->ip, sym_flags);
1923 trace_seq_printf(s, ": %s", field->buf);
1924 if (entry->flags & TRACE_FLAG_CONT)
1925 trace_seq_print_cont(s, iter);
1926 break;
1927 }
1928 case TRACE_BRANCH: {
1929 struct trace_branch *field;
1930
1931 trace_assign_type(field, entry);
1932
1933 trace_seq_printf(s, "[%s] %s:%s:%d\n",
1934 field->correct ? " ok " : " MISS ",
1935 field->func,
1936 field->file,
1937 field->line);
1938 break;
1939 }
1940 case TRACE_USER_STACK: {
1941 struct userstack_entry *field;
1942
1943 trace_assign_type(field, entry);
1944 1635
1945 seq_print_userip_objs(field, s, sym_flags); 1636 /* Don't print started cpu buffer for the first entry of the trace */
1946 trace_seq_putc(s, '\n'); 1637 if (iter->idx > 1)
1947 break; 1638 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
1948 } 1639 iter->cpu);
1949 default:
1950 trace_seq_printf(s, "Unknown type %d\n", entry->type);
1951 }
1952 return TRACE_TYPE_HANDLED;
1953} 1640}
1954 1641
1955static enum print_line_t print_trace_fmt(struct trace_iterator *iter) 1642static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
@@ -1957,333 +1644,84 @@ static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
1957 struct trace_seq *s = &iter->seq; 1644 struct trace_seq *s = &iter->seq;
1958 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK); 1645 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1959 struct trace_entry *entry; 1646 struct trace_entry *entry;
1960 unsigned long usec_rem; 1647 struct trace_event *event;
1961 unsigned long long t;
1962 unsigned long secs;
1963 char *comm;
1964 int ret;
1965 int S, T;
1966 int i;
1967 1648
1968 entry = iter->ent; 1649 entry = iter->ent;
1969 1650
1970 if (entry->type == TRACE_CONT)
1971 return TRACE_TYPE_HANDLED;
1972
1973 test_cpu_buff_start(iter); 1651 test_cpu_buff_start(iter);
1974 1652
1975 comm = trace_find_cmdline(iter->ent->pid); 1653 event = ftrace_find_event(entry->type);
1976
1977 t = ns2usecs(iter->ts);
1978 usec_rem = do_div(t, 1000000ULL);
1979 secs = (unsigned long)t;
1980
1981 ret = trace_seq_printf(s, "%16s-%-5d ", comm, entry->pid);
1982 if (!ret)
1983 return TRACE_TYPE_PARTIAL_LINE;
1984 ret = trace_seq_printf(s, "[%03d] ", iter->cpu);
1985 if (!ret)
1986 return TRACE_TYPE_PARTIAL_LINE;
1987 ret = trace_seq_printf(s, "%5lu.%06lu: ", secs, usec_rem);
1988 if (!ret)
1989 return TRACE_TYPE_PARTIAL_LINE;
1990
1991 switch (entry->type) {
1992 case TRACE_FN: {
1993 struct ftrace_entry *field;
1994
1995 trace_assign_type(field, entry);
1996
1997 ret = seq_print_ip_sym(s, field->ip, sym_flags);
1998 if (!ret)
1999 return TRACE_TYPE_PARTIAL_LINE;
2000 if ((sym_flags & TRACE_ITER_PRINT_PARENT) &&
2001 field->parent_ip) {
2002 ret = trace_seq_printf(s, " <-");
2003 if (!ret)
2004 return TRACE_TYPE_PARTIAL_LINE;
2005 ret = seq_print_ip_sym(s,
2006 field->parent_ip,
2007 sym_flags);
2008 if (!ret)
2009 return TRACE_TYPE_PARTIAL_LINE;
2010 }
2011 ret = trace_seq_printf(s, "\n");
2012 if (!ret)
2013 return TRACE_TYPE_PARTIAL_LINE;
2014 break;
2015 }
2016 case TRACE_CTX:
2017 case TRACE_WAKE: {
2018 struct ctx_switch_entry *field;
2019
2020 trace_assign_type(field, entry);
2021
2022 T = task_state_char(field->next_state);
2023 S = task_state_char(field->prev_state);
2024 ret = trace_seq_printf(s, " %5d:%3d:%c %s [%03d] %5d:%3d:%c\n",
2025 field->prev_pid,
2026 field->prev_prio,
2027 S,
2028 entry->type == TRACE_CTX ? "==>" : " +",
2029 field->next_cpu,
2030 field->next_pid,
2031 field->next_prio,
2032 T);
2033 if (!ret)
2034 return TRACE_TYPE_PARTIAL_LINE;
2035 break;
2036 }
2037 case TRACE_SPECIAL: {
2038 struct special_entry *field;
2039 1654
2040 trace_assign_type(field, entry); 1655 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2041 1656 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2042 ret = trace_seq_printf(s, "# %ld %ld %ld\n", 1657 if (!trace_print_lat_context(iter))
2043 field->arg1, 1658 goto partial;
2044 field->arg2, 1659 } else {
2045 field->arg3); 1660 if (!trace_print_context(iter))
2046 if (!ret) 1661 goto partial;
2047 return TRACE_TYPE_PARTIAL_LINE;
2048 break;
2049 }
2050 case TRACE_STACK: {
2051 struct stack_entry *field;
2052
2053 trace_assign_type(field, entry);
2054
2055 for (i = 0; i < FTRACE_STACK_ENTRIES; i++) {
2056 if (i) {
2057 ret = trace_seq_puts(s, " <= ");
2058 if (!ret)
2059 return TRACE_TYPE_PARTIAL_LINE;
2060 }
2061 ret = seq_print_ip_sym(s, field->caller[i],
2062 sym_flags);
2063 if (!ret)
2064 return TRACE_TYPE_PARTIAL_LINE;
2065 } 1662 }
2066 ret = trace_seq_puts(s, "\n");
2067 if (!ret)
2068 return TRACE_TYPE_PARTIAL_LINE;
2069 break;
2070 }
2071 case TRACE_PRINT: {
2072 struct print_entry *field;
2073
2074 trace_assign_type(field, entry);
2075
2076 seq_print_ip_sym(s, field->ip, sym_flags);
2077 trace_seq_printf(s, ": %s", field->buf);
2078 if (entry->flags & TRACE_FLAG_CONT)
2079 trace_seq_print_cont(s, iter);
2080 break;
2081 }
2082 case TRACE_GRAPH_RET: {
2083 return print_graph_function(iter);
2084 } 1663 }
2085 case TRACE_GRAPH_ENT: {
2086 return print_graph_function(iter);
2087 }
2088 case TRACE_BRANCH: {
2089 struct trace_branch *field;
2090
2091 trace_assign_type(field, entry);
2092 1664
2093 trace_seq_printf(s, "[%s] %s:%s:%d\n", 1665 if (event)
2094 field->correct ? " ok " : " MISS ", 1666 return event->trace(iter, sym_flags);
2095 field->func,
2096 field->file,
2097 field->line);
2098 break;
2099 }
2100 case TRACE_USER_STACK: {
2101 struct userstack_entry *field;
2102 1667
2103 trace_assign_type(field, entry); 1668 if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
1669 goto partial;
2104 1670
2105 ret = seq_print_userip_objs(field, s, sym_flags);
2106 if (!ret)
2107 return TRACE_TYPE_PARTIAL_LINE;
2108 ret = trace_seq_putc(s, '\n');
2109 if (!ret)
2110 return TRACE_TYPE_PARTIAL_LINE;
2111 break;
2112 }
2113 }
2114 return TRACE_TYPE_HANDLED; 1671 return TRACE_TYPE_HANDLED;
1672partial:
1673 return TRACE_TYPE_PARTIAL_LINE;
2115} 1674}
2116 1675
2117static enum print_line_t print_raw_fmt(struct trace_iterator *iter) 1676static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
2118{ 1677{
2119 struct trace_seq *s = &iter->seq; 1678 struct trace_seq *s = &iter->seq;
2120 struct trace_entry *entry; 1679 struct trace_entry *entry;
2121 int ret; 1680 struct trace_event *event;
2122 int S, T;
2123 1681
2124 entry = iter->ent; 1682 entry = iter->ent;
2125 1683
2126 if (entry->type == TRACE_CONT) 1684 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2127 return TRACE_TYPE_HANDLED; 1685 if (!trace_seq_printf(s, "%d %d %llu ",
2128 1686 entry->pid, iter->cpu, iter->ts))
2129 ret = trace_seq_printf(s, "%d %d %llu ", 1687 goto partial;
2130 entry->pid, iter->cpu, iter->ts);
2131 if (!ret)
2132 return TRACE_TYPE_PARTIAL_LINE;
2133
2134 switch (entry->type) {
2135 case TRACE_FN: {
2136 struct ftrace_entry *field;
2137
2138 trace_assign_type(field, entry);
2139
2140 ret = trace_seq_printf(s, "%x %x\n",
2141 field->ip,
2142 field->parent_ip);
2143 if (!ret)
2144 return TRACE_TYPE_PARTIAL_LINE;
2145 break;
2146 } 1688 }
2147 case TRACE_CTX:
2148 case TRACE_WAKE: {
2149 struct ctx_switch_entry *field;
2150
2151 trace_assign_type(field, entry);
2152
2153 T = task_state_char(field->next_state);
2154 S = entry->type == TRACE_WAKE ? '+' :
2155 task_state_char(field->prev_state);
2156 ret = trace_seq_printf(s, "%d %d %c %d %d %d %c\n",
2157 field->prev_pid,
2158 field->prev_prio,
2159 S,
2160 field->next_cpu,
2161 field->next_pid,
2162 field->next_prio,
2163 T);
2164 if (!ret)
2165 return TRACE_TYPE_PARTIAL_LINE;
2166 break;
2167 }
2168 case TRACE_SPECIAL:
2169 case TRACE_USER_STACK:
2170 case TRACE_STACK: {
2171 struct special_entry *field;
2172
2173 trace_assign_type(field, entry);
2174 1689
2175 ret = trace_seq_printf(s, "# %ld %ld %ld\n", 1690 event = ftrace_find_event(entry->type);
2176 field->arg1, 1691 if (event)
2177 field->arg2, 1692 return event->raw(iter, 0);
2178 field->arg3);
2179 if (!ret)
2180 return TRACE_TYPE_PARTIAL_LINE;
2181 break;
2182 }
2183 case TRACE_PRINT: {
2184 struct print_entry *field;
2185 1693
2186 trace_assign_type(field, entry); 1694 if (!trace_seq_printf(s, "%d ?\n", entry->type))
1695 goto partial;
2187 1696
2188 trace_seq_printf(s, "# %lx %s", field->ip, field->buf);
2189 if (entry->flags & TRACE_FLAG_CONT)
2190 trace_seq_print_cont(s, iter);
2191 break;
2192 }
2193 }
2194 return TRACE_TYPE_HANDLED; 1697 return TRACE_TYPE_HANDLED;
1698partial:
1699 return TRACE_TYPE_PARTIAL_LINE;
2195} 1700}
2196 1701
2197#define SEQ_PUT_FIELD_RET(s, x) \
2198do { \
2199 if (!trace_seq_putmem(s, &(x), sizeof(x))) \
2200 return 0; \
2201} while (0)
2202
2203#define SEQ_PUT_HEX_FIELD_RET(s, x) \
2204do { \
2205 BUILD_BUG_ON(sizeof(x) > MAX_MEMHEX_BYTES); \
2206 if (!trace_seq_putmem_hex(s, &(x), sizeof(x))) \
2207 return 0; \
2208} while (0)
2209
2210static enum print_line_t print_hex_fmt(struct trace_iterator *iter) 1702static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
2211{ 1703{
2212 struct trace_seq *s = &iter->seq; 1704 struct trace_seq *s = &iter->seq;
2213 unsigned char newline = '\n'; 1705 unsigned char newline = '\n';
2214 struct trace_entry *entry; 1706 struct trace_entry *entry;
2215 int S, T; 1707 struct trace_event *event;
2216 1708
2217 entry = iter->ent; 1709 entry = iter->ent;
2218 1710
2219 if (entry->type == TRACE_CONT) 1711 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2220 return TRACE_TYPE_HANDLED; 1712 SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
2221 1713 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
2222 SEQ_PUT_HEX_FIELD_RET(s, entry->pid); 1714 SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
2223 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
2224 SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
2225
2226 switch (entry->type) {
2227 case TRACE_FN: {
2228 struct ftrace_entry *field;
2229
2230 trace_assign_type(field, entry);
2231
2232 SEQ_PUT_HEX_FIELD_RET(s, field->ip);
2233 SEQ_PUT_HEX_FIELD_RET(s, field->parent_ip);
2234 break;
2235 } 1715 }
2236 case TRACE_CTX:
2237 case TRACE_WAKE: {
2238 struct ctx_switch_entry *field;
2239
2240 trace_assign_type(field, entry);
2241
2242 T = task_state_char(field->next_state);
2243 S = entry->type == TRACE_WAKE ? '+' :
2244 task_state_char(field->prev_state);
2245 SEQ_PUT_HEX_FIELD_RET(s, field->prev_pid);
2246 SEQ_PUT_HEX_FIELD_RET(s, field->prev_prio);
2247 SEQ_PUT_HEX_FIELD_RET(s, S);
2248 SEQ_PUT_HEX_FIELD_RET(s, field->next_cpu);
2249 SEQ_PUT_HEX_FIELD_RET(s, field->next_pid);
2250 SEQ_PUT_HEX_FIELD_RET(s, field->next_prio);
2251 SEQ_PUT_HEX_FIELD_RET(s, T);
2252 break;
2253 }
2254 case TRACE_SPECIAL:
2255 case TRACE_USER_STACK:
2256 case TRACE_STACK: {
2257 struct special_entry *field;
2258
2259 trace_assign_type(field, entry);
2260 1716
2261 SEQ_PUT_HEX_FIELD_RET(s, field->arg1); 1717 event = ftrace_find_event(entry->type);
2262 SEQ_PUT_HEX_FIELD_RET(s, field->arg2); 1718 if (event) {
2263 SEQ_PUT_HEX_FIELD_RET(s, field->arg3); 1719 enum print_line_t ret = event->hex(iter, 0);
2264 break; 1720 if (ret != TRACE_TYPE_HANDLED)
2265 } 1721 return ret;
2266 } 1722 }
2267 SEQ_PUT_FIELD_RET(s, newline);
2268
2269 return TRACE_TYPE_HANDLED;
2270}
2271
2272static enum print_line_t print_printk_msg_only(struct trace_iterator *iter)
2273{
2274 struct trace_seq *s = &iter->seq;
2275 struct trace_entry *entry = iter->ent;
2276 struct print_entry *field;
2277 int ret;
2278
2279 trace_assign_type(field, entry);
2280 1723
2281 ret = trace_seq_printf(s, field->buf); 1724 SEQ_PUT_FIELD_RET(s, newline);
2282 if (!ret)
2283 return TRACE_TYPE_PARTIAL_LINE;
2284
2285 if (entry->flags & TRACE_FLAG_CONT)
2286 trace_seq_print_cont(s, iter);
2287 1725
2288 return TRACE_TYPE_HANDLED; 1726 return TRACE_TYPE_HANDLED;
2289} 1727}
@@ -2292,59 +1730,37 @@ static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
2292{ 1730{
2293 struct trace_seq *s = &iter->seq; 1731 struct trace_seq *s = &iter->seq;
2294 struct trace_entry *entry; 1732 struct trace_entry *entry;
1733 struct trace_event *event;
2295 1734
2296 entry = iter->ent; 1735 entry = iter->ent;
2297 1736
2298 if (entry->type == TRACE_CONT) 1737 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2299 return TRACE_TYPE_HANDLED; 1738 SEQ_PUT_FIELD_RET(s, entry->pid);
2300 1739 SEQ_PUT_FIELD_RET(s, iter->cpu);
2301 SEQ_PUT_FIELD_RET(s, entry->pid); 1740 SEQ_PUT_FIELD_RET(s, iter->ts);
2302 SEQ_PUT_FIELD_RET(s, entry->cpu);
2303 SEQ_PUT_FIELD_RET(s, iter->ts);
2304
2305 switch (entry->type) {
2306 case TRACE_FN: {
2307 struct ftrace_entry *field;
2308
2309 trace_assign_type(field, entry);
2310
2311 SEQ_PUT_FIELD_RET(s, field->ip);
2312 SEQ_PUT_FIELD_RET(s, field->parent_ip);
2313 break;
2314 }
2315 case TRACE_CTX: {
2316 struct ctx_switch_entry *field;
2317
2318 trace_assign_type(field, entry);
2319
2320 SEQ_PUT_FIELD_RET(s, field->prev_pid);
2321 SEQ_PUT_FIELD_RET(s, field->prev_prio);
2322 SEQ_PUT_FIELD_RET(s, field->prev_state);
2323 SEQ_PUT_FIELD_RET(s, field->next_pid);
2324 SEQ_PUT_FIELD_RET(s, field->next_prio);
2325 SEQ_PUT_FIELD_RET(s, field->next_state);
2326 break;
2327 } 1741 }
2328 case TRACE_SPECIAL:
2329 case TRACE_USER_STACK:
2330 case TRACE_STACK: {
2331 struct special_entry *field;
2332
2333 trace_assign_type(field, entry);
2334 1742
2335 SEQ_PUT_FIELD_RET(s, field->arg1); 1743 event = ftrace_find_event(entry->type);
2336 SEQ_PUT_FIELD_RET(s, field->arg2); 1744 return event ? event->binary(iter, 0) : TRACE_TYPE_HANDLED;
2337 SEQ_PUT_FIELD_RET(s, field->arg3);
2338 break;
2339 }
2340 }
2341 return 1;
2342} 1745}
2343 1746
2344static int trace_empty(struct trace_iterator *iter) 1747static int trace_empty(struct trace_iterator *iter)
2345{ 1748{
2346 int cpu; 1749 int cpu;
2347 1750
1751 /* If we are looking at one CPU buffer, only check that one */
1752 if (iter->cpu_file != TRACE_PIPE_ALL_CPU) {
1753 cpu = iter->cpu_file;
1754 if (iter->buffer_iter[cpu]) {
1755 if (!ring_buffer_iter_empty(iter->buffer_iter[cpu]))
1756 return 0;
1757 } else {
1758 if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
1759 return 0;
1760 }
1761 return 1;
1762 }
1763
2348 for_each_tracing_cpu(cpu) { 1764 for_each_tracing_cpu(cpu) {
2349 if (iter->buffer_iter[cpu]) { 1765 if (iter->buffer_iter[cpu]) {
2350 if (!ring_buffer_iter_empty(iter->buffer_iter[cpu])) 1766 if (!ring_buffer_iter_empty(iter->buffer_iter[cpu]))
@@ -2368,10 +1784,15 @@ static enum print_line_t print_trace_line(struct trace_iterator *iter)
2368 return ret; 1784 return ret;
2369 } 1785 }
2370 1786
1787 if (iter->ent->type == TRACE_BPRINT &&
1788 trace_flags & TRACE_ITER_PRINTK &&
1789 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
1790 return trace_print_bprintk_msg_only(iter);
1791
2371 if (iter->ent->type == TRACE_PRINT && 1792 if (iter->ent->type == TRACE_PRINT &&
2372 trace_flags & TRACE_ITER_PRINTK && 1793 trace_flags & TRACE_ITER_PRINTK &&
2373 trace_flags & TRACE_ITER_PRINTK_MSGONLY) 1794 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2374 return print_printk_msg_only(iter); 1795 return trace_print_printk_msg_only(iter);
2375 1796
2376 if (trace_flags & TRACE_ITER_BIN) 1797 if (trace_flags & TRACE_ITER_BIN)
2377 return print_bin_fmt(iter); 1798 return print_bin_fmt(iter);
@@ -2382,9 +1803,6 @@ static enum print_line_t print_trace_line(struct trace_iterator *iter)
2382 if (trace_flags & TRACE_ITER_RAW) 1803 if (trace_flags & TRACE_ITER_RAW)
2383 return print_raw_fmt(iter); 1804 return print_raw_fmt(iter);
2384 1805
2385 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2386 return print_lat_fmt(iter, iter->idx, iter->cpu);
2387
2388 return print_trace_fmt(iter); 1806 return print_trace_fmt(iter);
2389} 1807}
2390 1808
@@ -2426,30 +1844,45 @@ static struct seq_operations tracer_seq_ops = {
2426}; 1844};
2427 1845
2428static struct trace_iterator * 1846static struct trace_iterator *
2429__tracing_open(struct inode *inode, struct file *file, int *ret) 1847__tracing_open(struct inode *inode, struct file *file)
2430{ 1848{
1849 long cpu_file = (long) inode->i_private;
1850 void *fail_ret = ERR_PTR(-ENOMEM);
2431 struct trace_iterator *iter; 1851 struct trace_iterator *iter;
2432 struct seq_file *m; 1852 struct seq_file *m;
2433 int cpu; 1853 int cpu, ret;
2434 1854
2435 if (tracing_disabled) { 1855 if (tracing_disabled)
2436 *ret = -ENODEV; 1856 return ERR_PTR(-ENODEV);
2437 return NULL;
2438 }
2439 1857
2440 iter = kzalloc(sizeof(*iter), GFP_KERNEL); 1858 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
2441 if (!iter) { 1859 if (!iter)
2442 *ret = -ENOMEM; 1860 return ERR_PTR(-ENOMEM);
2443 goto out;
2444 }
2445 1861
1862 /*
1863 * We make a copy of the current tracer to avoid concurrent
1864 * changes on it while we are reading.
1865 */
2446 mutex_lock(&trace_types_lock); 1866 mutex_lock(&trace_types_lock);
1867 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
1868 if (!iter->trace)
1869 goto fail;
1870
1871 if (current_trace)
1872 *iter->trace = *current_trace;
1873
1874 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL))
1875 goto fail;
1876
1877 cpumask_clear(iter->started);
1878
2447 if (current_trace && current_trace->print_max) 1879 if (current_trace && current_trace->print_max)
2448 iter->tr = &max_tr; 1880 iter->tr = &max_tr;
2449 else 1881 else
2450 iter->tr = inode->i_private; 1882 iter->tr = &global_trace;
2451 iter->trace = current_trace;
2452 iter->pos = -1; 1883 iter->pos = -1;
1884 mutex_init(&iter->mutex);
1885 iter->cpu_file = cpu_file;
2453 1886
2454 /* Notify the tracer early; before we stop tracing. */ 1887 /* Notify the tracer early; before we stop tracing. */
2455 if (iter->trace && iter->trace->open) 1888 if (iter->trace && iter->trace->open)
@@ -2459,20 +1892,24 @@ __tracing_open(struct inode *inode, struct file *file, int *ret)
2459 if (ring_buffer_overruns(iter->tr->buffer)) 1892 if (ring_buffer_overruns(iter->tr->buffer))
2460 iter->iter_flags |= TRACE_FILE_ANNOTATE; 1893 iter->iter_flags |= TRACE_FILE_ANNOTATE;
2461 1894
1895 if (iter->cpu_file == TRACE_PIPE_ALL_CPU) {
1896 for_each_tracing_cpu(cpu) {
2462 1897
2463 for_each_tracing_cpu(cpu) { 1898 iter->buffer_iter[cpu] =
2464 1899 ring_buffer_read_start(iter->tr->buffer, cpu);
1900 }
1901 } else {
1902 cpu = iter->cpu_file;
2465 iter->buffer_iter[cpu] = 1903 iter->buffer_iter[cpu] =
2466 ring_buffer_read_start(iter->tr->buffer, cpu); 1904 ring_buffer_read_start(iter->tr->buffer, cpu);
2467
2468 if (!iter->buffer_iter[cpu])
2469 goto fail_buffer;
2470 } 1905 }
2471 1906
2472 /* TODO stop tracer */ 1907 /* TODO stop tracer */
2473 *ret = seq_open(file, &tracer_seq_ops); 1908 ret = seq_open(file, &tracer_seq_ops);
2474 if (*ret) 1909 if (ret < 0) {
1910 fail_ret = ERR_PTR(ret);
2475 goto fail_buffer; 1911 goto fail_buffer;
1912 }
2476 1913
2477 m = file->private_data; 1914 m = file->private_data;
2478 m->private = iter; 1915 m->private = iter;
@@ -2482,7 +1919,6 @@ __tracing_open(struct inode *inode, struct file *file, int *ret)
2482 1919
2483 mutex_unlock(&trace_types_lock); 1920 mutex_unlock(&trace_types_lock);
2484 1921
2485 out:
2486 return iter; 1922 return iter;
2487 1923
2488 fail_buffer: 1924 fail_buffer:
@@ -2490,10 +1926,13 @@ __tracing_open(struct inode *inode, struct file *file, int *ret)
2490 if (iter->buffer_iter[cpu]) 1926 if (iter->buffer_iter[cpu])
2491 ring_buffer_read_finish(iter->buffer_iter[cpu]); 1927 ring_buffer_read_finish(iter->buffer_iter[cpu]);
2492 } 1928 }
1929 free_cpumask_var(iter->started);
1930 fail:
2493 mutex_unlock(&trace_types_lock); 1931 mutex_unlock(&trace_types_lock);
1932 kfree(iter->trace);
2494 kfree(iter); 1933 kfree(iter);
2495 1934
2496 return ERR_PTR(-ENOMEM); 1935 return fail_ret;
2497} 1936}
2498 1937
2499int tracing_open_generic(struct inode *inode, struct file *filp) 1938int tracing_open_generic(struct inode *inode, struct file *filp)
@@ -2505,12 +1944,17 @@ int tracing_open_generic(struct inode *inode, struct file *filp)
2505 return 0; 1944 return 0;
2506} 1945}
2507 1946
2508int tracing_release(struct inode *inode, struct file *file) 1947static int tracing_release(struct inode *inode, struct file *file)
2509{ 1948{
2510 struct seq_file *m = (struct seq_file *)file->private_data; 1949 struct seq_file *m = (struct seq_file *)file->private_data;
2511 struct trace_iterator *iter = m->private; 1950 struct trace_iterator *iter;
2512 int cpu; 1951 int cpu;
2513 1952
1953 if (!(file->f_mode & FMODE_READ))
1954 return 0;
1955
1956 iter = m->private;
1957
2514 mutex_lock(&trace_types_lock); 1958 mutex_lock(&trace_types_lock);
2515 for_each_tracing_cpu(cpu) { 1959 for_each_tracing_cpu(cpu) {
2516 if (iter->buffer_iter[cpu]) 1960 if (iter->buffer_iter[cpu])
@@ -2525,33 +1969,39 @@ int tracing_release(struct inode *inode, struct file *file)
2525 mutex_unlock(&trace_types_lock); 1969 mutex_unlock(&trace_types_lock);
2526 1970
2527 seq_release(inode, file); 1971 seq_release(inode, file);
1972 mutex_destroy(&iter->mutex);
1973 free_cpumask_var(iter->started);
1974 kfree(iter->trace);
2528 kfree(iter); 1975 kfree(iter);
2529 return 0; 1976 return 0;
2530} 1977}
2531 1978
2532static int tracing_open(struct inode *inode, struct file *file) 1979static int tracing_open(struct inode *inode, struct file *file)
2533{ 1980{
2534 int ret;
2535
2536 __tracing_open(inode, file, &ret);
2537
2538 return ret;
2539}
2540
2541static int tracing_lt_open(struct inode *inode, struct file *file)
2542{
2543 struct trace_iterator *iter; 1981 struct trace_iterator *iter;
2544 int ret; 1982 int ret = 0;
2545 1983
2546 iter = __tracing_open(inode, file, &ret); 1984 /* If this file was open for write, then erase contents */
1985 if ((file->f_mode & FMODE_WRITE) &&
1986 !(file->f_flags & O_APPEND)) {
1987 long cpu = (long) inode->i_private;
2547 1988
2548 if (!ret) 1989 if (cpu == TRACE_PIPE_ALL_CPU)
2549 iter->iter_flags |= TRACE_FILE_LAT_FMT; 1990 tracing_reset_online_cpus(&global_trace);
1991 else
1992 tracing_reset(&global_trace, cpu);
1993 }
2550 1994
1995 if (file->f_mode & FMODE_READ) {
1996 iter = __tracing_open(inode, file);
1997 if (IS_ERR(iter))
1998 ret = PTR_ERR(iter);
1999 else if (trace_flags & TRACE_ITER_LATENCY_FMT)
2000 iter->iter_flags |= TRACE_FILE_LAT_FMT;
2001 }
2551 return ret; 2002 return ret;
2552} 2003}
2553 2004
2554
2555static void * 2005static void *
2556t_next(struct seq_file *m, void *v, loff_t *pos) 2006t_next(struct seq_file *m, void *v, loff_t *pos)
2557{ 2007{
@@ -2623,21 +2073,22 @@ static int show_traces_open(struct inode *inode, struct file *file)
2623 return ret; 2073 return ret;
2624} 2074}
2625 2075
2626static struct file_operations tracing_fops = { 2076static ssize_t
2627 .open = tracing_open, 2077tracing_write_stub(struct file *filp, const char __user *ubuf,
2628 .read = seq_read, 2078 size_t count, loff_t *ppos)
2629 .llseek = seq_lseek, 2079{
2630 .release = tracing_release, 2080 return count;
2631}; 2081}
2632 2082
2633static struct file_operations tracing_lt_fops = { 2083static const struct file_operations tracing_fops = {
2634 .open = tracing_lt_open, 2084 .open = tracing_open,
2635 .read = seq_read, 2085 .read = seq_read,
2086 .write = tracing_write_stub,
2636 .llseek = seq_lseek, 2087 .llseek = seq_lseek,
2637 .release = tracing_release, 2088 .release = tracing_release,
2638}; 2089};
2639 2090
2640static struct file_operations show_traces_fops = { 2091static const struct file_operations show_traces_fops = {
2641 .open = show_traces_open, 2092 .open = show_traces_open,
2642 .read = seq_read, 2093 .read = seq_read,
2643 .release = seq_release, 2094 .release = seq_release,
@@ -2730,7 +2181,7 @@ err_unlock:
2730 return err; 2181 return err;
2731} 2182}
2732 2183
2733static struct file_operations tracing_cpumask_fops = { 2184static const struct file_operations tracing_cpumask_fops = {
2734 .open = tracing_open_generic, 2185 .open = tracing_open_generic,
2735 .read = tracing_cpumask_read, 2186 .read = tracing_cpumask_read,
2736 .write = tracing_cpumask_write, 2187 .write = tracing_cpumask_write,
@@ -2740,57 +2191,62 @@ static ssize_t
2740tracing_trace_options_read(struct file *filp, char __user *ubuf, 2191tracing_trace_options_read(struct file *filp, char __user *ubuf,
2741 size_t cnt, loff_t *ppos) 2192 size_t cnt, loff_t *ppos)
2742{ 2193{
2743 int i; 2194 struct tracer_opt *trace_opts;
2195 u32 tracer_flags;
2196 int len = 0;
2744 char *buf; 2197 char *buf;
2745 int r = 0; 2198 int r = 0;
2746 int len = 0; 2199 int i;
2747 u32 tracer_flags = current_trace->flags->val;
2748 struct tracer_opt *trace_opts = current_trace->flags->opts;
2749 2200
2750 2201
2751 /* calulate max size */ 2202 /* calculate max size */
2752 for (i = 0; trace_options[i]; i++) { 2203 for (i = 0; trace_options[i]; i++) {
2753 len += strlen(trace_options[i]); 2204 len += strlen(trace_options[i]);
2754 len += 3; /* "no" and space */ 2205 len += 3; /* "no" and newline */
2755 } 2206 }
2756 2207
2208 mutex_lock(&trace_types_lock);
2209 tracer_flags = current_trace->flags->val;
2210 trace_opts = current_trace->flags->opts;
2211
2757 /* 2212 /*
2758 * Increase the size with names of options specific 2213 * Increase the size with names of options specific
2759 * of the current tracer. 2214 * of the current tracer.
2760 */ 2215 */
2761 for (i = 0; trace_opts[i].name; i++) { 2216 for (i = 0; trace_opts[i].name; i++) {
2762 len += strlen(trace_opts[i].name); 2217 len += strlen(trace_opts[i].name);
2763 len += 3; /* "no" and space */ 2218 len += 3; /* "no" and newline */
2764 } 2219 }
2765 2220
2766 /* +2 for \n and \0 */ 2221 /* +2 for \n and \0 */
2767 buf = kmalloc(len + 2, GFP_KERNEL); 2222 buf = kmalloc(len + 2, GFP_KERNEL);
2768 if (!buf) 2223 if (!buf) {
2224 mutex_unlock(&trace_types_lock);
2769 return -ENOMEM; 2225 return -ENOMEM;
2226 }
2770 2227
2771 for (i = 0; trace_options[i]; i++) { 2228 for (i = 0; trace_options[i]; i++) {
2772 if (trace_flags & (1 << i)) 2229 if (trace_flags & (1 << i))
2773 r += sprintf(buf + r, "%s ", trace_options[i]); 2230 r += sprintf(buf + r, "%s\n", trace_options[i]);
2774 else 2231 else
2775 r += sprintf(buf + r, "no%s ", trace_options[i]); 2232 r += sprintf(buf + r, "no%s\n", trace_options[i]);
2776 } 2233 }
2777 2234
2778 for (i = 0; trace_opts[i].name; i++) { 2235 for (i = 0; trace_opts[i].name; i++) {
2779 if (tracer_flags & trace_opts[i].bit) 2236 if (tracer_flags & trace_opts[i].bit)
2780 r += sprintf(buf + r, "%s ", 2237 r += sprintf(buf + r, "%s\n",
2781 trace_opts[i].name); 2238 trace_opts[i].name);
2782 else 2239 else
2783 r += sprintf(buf + r, "no%s ", 2240 r += sprintf(buf + r, "no%s\n",
2784 trace_opts[i].name); 2241 trace_opts[i].name);
2785 } 2242 }
2243 mutex_unlock(&trace_types_lock);
2786 2244
2787 r += sprintf(buf + r, "\n");
2788 WARN_ON(r >= len + 2); 2245 WARN_ON(r >= len + 2);
2789 2246
2790 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 2247 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2791 2248
2792 kfree(buf); 2249 kfree(buf);
2793
2794 return r; 2250 return r;
2795} 2251}
2796 2252
@@ -2828,6 +2284,34 @@ static int set_tracer_option(struct tracer *trace, char *cmp, int neg)
2828 return 0; 2284 return 0;
2829} 2285}
2830 2286
2287static void set_tracer_flags(unsigned int mask, int enabled)
2288{
2289 /* do nothing if flag is already set */
2290 if (!!(trace_flags & mask) == !!enabled)
2291 return;
2292
2293 if (enabled)
2294 trace_flags |= mask;
2295 else
2296 trace_flags &= ~mask;
2297
2298 if (mask == TRACE_ITER_GLOBAL_CLK) {
2299 u64 (*func)(void);
2300
2301 if (enabled)
2302 func = trace_clock_global;
2303 else
2304 func = trace_clock_local;
2305
2306 mutex_lock(&trace_types_lock);
2307 ring_buffer_set_clock(global_trace.buffer, func);
2308
2309 if (max_tr.buffer)
2310 ring_buffer_set_clock(max_tr.buffer, func);
2311 mutex_unlock(&trace_types_lock);
2312 }
2313}
2314
2831static ssize_t 2315static ssize_t
2832tracing_trace_options_write(struct file *filp, const char __user *ubuf, 2316tracing_trace_options_write(struct file *filp, const char __user *ubuf,
2833 size_t cnt, loff_t *ppos) 2317 size_t cnt, loff_t *ppos)
@@ -2855,17 +2339,16 @@ tracing_trace_options_write(struct file *filp, const char __user *ubuf,
2855 int len = strlen(trace_options[i]); 2339 int len = strlen(trace_options[i]);
2856 2340
2857 if (strncmp(cmp, trace_options[i], len) == 0) { 2341 if (strncmp(cmp, trace_options[i], len) == 0) {
2858 if (neg) 2342 set_tracer_flags(1 << i, !neg);
2859 trace_flags &= ~(1 << i);
2860 else
2861 trace_flags |= (1 << i);
2862 break; 2343 break;
2863 } 2344 }
2864 } 2345 }
2865 2346
2866 /* If no option could be set, test the specific tracer options */ 2347 /* If no option could be set, test the specific tracer options */
2867 if (!trace_options[i]) { 2348 if (!trace_options[i]) {
2349 mutex_lock(&trace_types_lock);
2868 ret = set_tracer_option(current_trace, cmp, neg); 2350 ret = set_tracer_option(current_trace, cmp, neg);
2351 mutex_unlock(&trace_types_lock);
2869 if (ret) 2352 if (ret)
2870 return ret; 2353 return ret;
2871 } 2354 }
@@ -2875,7 +2358,7 @@ tracing_trace_options_write(struct file *filp, const char __user *ubuf,
2875 return cnt; 2358 return cnt;
2876} 2359}
2877 2360
2878static struct file_operations tracing_iter_fops = { 2361static const struct file_operations tracing_iter_fops = {
2879 .open = tracing_open_generic, 2362 .open = tracing_open_generic,
2880 .read = tracing_trace_options_read, 2363 .read = tracing_trace_options_read,
2881 .write = tracing_trace_options_write, 2364 .write = tracing_trace_options_write,
@@ -2886,9 +2369,9 @@ static const char readme_msg[] =
2886 "# mkdir /debug\n" 2369 "# mkdir /debug\n"
2887 "# mount -t debugfs nodev /debug\n\n" 2370 "# mount -t debugfs nodev /debug\n\n"
2888 "# cat /debug/tracing/available_tracers\n" 2371 "# cat /debug/tracing/available_tracers\n"
2889 "wakeup preemptirqsoff preemptoff irqsoff ftrace sched_switch none\n\n" 2372 "wakeup preemptirqsoff preemptoff irqsoff function sched_switch nop\n\n"
2890 "# cat /debug/tracing/current_tracer\n" 2373 "# cat /debug/tracing/current_tracer\n"
2891 "none\n" 2374 "nop\n"
2892 "# echo sched_switch > /debug/tracing/current_tracer\n" 2375 "# echo sched_switch > /debug/tracing/current_tracer\n"
2893 "# cat /debug/tracing/current_tracer\n" 2376 "# cat /debug/tracing/current_tracer\n"
2894 "sched_switch\n" 2377 "sched_switch\n"
@@ -2908,7 +2391,7 @@ tracing_readme_read(struct file *filp, char __user *ubuf,
2908 readme_msg, strlen(readme_msg)); 2391 readme_msg, strlen(readme_msg));
2909} 2392}
2910 2393
2911static struct file_operations tracing_readme_fops = { 2394static const struct file_operations tracing_readme_fops = {
2912 .open = tracing_open_generic, 2395 .open = tracing_open_generic,
2913 .read = tracing_readme_read, 2396 .read = tracing_readme_read,
2914}; 2397};
@@ -2930,7 +2413,7 @@ tracing_ctrl_write(struct file *filp, const char __user *ubuf,
2930{ 2413{
2931 struct trace_array *tr = filp->private_data; 2414 struct trace_array *tr = filp->private_data;
2932 char buf[64]; 2415 char buf[64];
2933 long val; 2416 unsigned long val;
2934 int ret; 2417 int ret;
2935 2418
2936 if (cnt >= sizeof(buf)) 2419 if (cnt >= sizeof(buf))
@@ -2985,13 +2468,105 @@ tracing_set_trace_read(struct file *filp, char __user *ubuf,
2985 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 2468 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2986} 2469}
2987 2470
2988static int tracing_set_tracer(char *buf) 2471int tracer_init(struct tracer *t, struct trace_array *tr)
2472{
2473 tracing_reset_online_cpus(tr);
2474 return t->init(tr);
2475}
2476
2477static int tracing_resize_ring_buffer(unsigned long size)
2478{
2479 int ret;
2480
2481 /*
2482 * If kernel or user changes the size of the ring buffer
2483 * we use the size that was given, and we can forget about
2484 * expanding it later.
2485 */
2486 ring_buffer_expanded = 1;
2487
2488 ret = ring_buffer_resize(global_trace.buffer, size);
2489 if (ret < 0)
2490 return ret;
2491
2492 ret = ring_buffer_resize(max_tr.buffer, size);
2493 if (ret < 0) {
2494 int r;
2495
2496 r = ring_buffer_resize(global_trace.buffer,
2497 global_trace.entries);
2498 if (r < 0) {
2499 /*
2500 * AARGH! We are left with different
2501 * size max buffer!!!!
2502 * The max buffer is our "snapshot" buffer.
2503 * When a tracer needs a snapshot (one of the
2504 * latency tracers), it swaps the max buffer
2505 * with the saved snap shot. We succeeded to
2506 * update the size of the main buffer, but failed to
2507 * update the size of the max buffer. But when we tried
2508 * to reset the main buffer to the original size, we
2509 * failed there too. This is very unlikely to
2510 * happen, but if it does, warn and kill all
2511 * tracing.
2512 */
2513 WARN_ON(1);
2514 tracing_disabled = 1;
2515 }
2516 return ret;
2517 }
2518
2519 global_trace.entries = size;
2520
2521 return ret;
2522}
2523
2524/**
2525 * tracing_update_buffers - used by tracing facility to expand ring buffers
2526 *
2527 * To save on memory when the tracing is never used on a system with it
2528 * configured in. The ring buffers are set to a minimum size. But once
2529 * a user starts to use the tracing facility, then they need to grow
2530 * to their default size.
2531 *
2532 * This function is to be called when a tracer is about to be used.
2533 */
2534int tracing_update_buffers(void)
2535{
2536 int ret = 0;
2537
2538 mutex_lock(&trace_types_lock);
2539 if (!ring_buffer_expanded)
2540 ret = tracing_resize_ring_buffer(trace_buf_size);
2541 mutex_unlock(&trace_types_lock);
2542
2543 return ret;
2544}
2545
2546struct trace_option_dentry;
2547
2548static struct trace_option_dentry *
2549create_trace_option_files(struct tracer *tracer);
2550
2551static void
2552destroy_trace_option_files(struct trace_option_dentry *topts);
2553
2554static int tracing_set_tracer(const char *buf)
2989{ 2555{
2556 static struct trace_option_dentry *topts;
2990 struct trace_array *tr = &global_trace; 2557 struct trace_array *tr = &global_trace;
2991 struct tracer *t; 2558 struct tracer *t;
2992 int ret = 0; 2559 int ret = 0;
2993 2560
2994 mutex_lock(&trace_types_lock); 2561 mutex_lock(&trace_types_lock);
2562
2563 if (!ring_buffer_expanded) {
2564 ret = tracing_resize_ring_buffer(trace_buf_size);
2565 if (ret < 0)
2566 goto out;
2567 ret = 0;
2568 }
2569
2995 for (t = trace_types; t; t = t->next) { 2570 for (t = trace_types; t; t = t->next) {
2996 if (strcmp(t->name, buf) == 0) 2571 if (strcmp(t->name, buf) == 0)
2997 break; 2572 break;
@@ -3007,9 +2582,14 @@ static int tracing_set_tracer(char *buf)
3007 if (current_trace && current_trace->reset) 2582 if (current_trace && current_trace->reset)
3008 current_trace->reset(tr); 2583 current_trace->reset(tr);
3009 2584
2585 destroy_trace_option_files(topts);
2586
3010 current_trace = t; 2587 current_trace = t;
2588
2589 topts = create_trace_option_files(current_trace);
2590
3011 if (t->init) { 2591 if (t->init) {
3012 ret = t->init(tr); 2592 ret = tracer_init(t, tr);
3013 if (ret) 2593 if (ret)
3014 goto out; 2594 goto out;
3015 } 2595 }
@@ -3072,9 +2652,9 @@ static ssize_t
3072tracing_max_lat_write(struct file *filp, const char __user *ubuf, 2652tracing_max_lat_write(struct file *filp, const char __user *ubuf,
3073 size_t cnt, loff_t *ppos) 2653 size_t cnt, loff_t *ppos)
3074{ 2654{
3075 long *ptr = filp->private_data; 2655 unsigned long *ptr = filp->private_data;
3076 char buf[64]; 2656 char buf[64];
3077 long val; 2657 unsigned long val;
3078 int ret; 2658 int ret;
3079 2659
3080 if (cnt >= sizeof(buf)) 2660 if (cnt >= sizeof(buf))
@@ -3094,54 +2674,96 @@ tracing_max_lat_write(struct file *filp, const char __user *ubuf,
3094 return cnt; 2674 return cnt;
3095} 2675}
3096 2676
3097static atomic_t tracing_reader;
3098
3099static int tracing_open_pipe(struct inode *inode, struct file *filp) 2677static int tracing_open_pipe(struct inode *inode, struct file *filp)
3100{ 2678{
2679 long cpu_file = (long) inode->i_private;
3101 struct trace_iterator *iter; 2680 struct trace_iterator *iter;
2681 int ret = 0;
3102 2682
3103 if (tracing_disabled) 2683 if (tracing_disabled)
3104 return -ENODEV; 2684 return -ENODEV;
3105 2685
3106 /* We only allow for reader of the pipe */ 2686 mutex_lock(&trace_types_lock);
3107 if (atomic_inc_return(&tracing_reader) != 1) { 2687
3108 atomic_dec(&tracing_reader); 2688 /* We only allow one reader per cpu */
3109 return -EBUSY; 2689 if (cpu_file == TRACE_PIPE_ALL_CPU) {
2690 if (!cpumask_empty(tracing_reader_cpumask)) {
2691 ret = -EBUSY;
2692 goto out;
2693 }
2694 cpumask_setall(tracing_reader_cpumask);
2695 } else {
2696 if (!cpumask_test_cpu(cpu_file, tracing_reader_cpumask))
2697 cpumask_set_cpu(cpu_file, tracing_reader_cpumask);
2698 else {
2699 ret = -EBUSY;
2700 goto out;
2701 }
3110 } 2702 }
3111 2703
3112 /* create a buffer to store the information to pass to userspace */ 2704 /* create a buffer to store the information to pass to userspace */
3113 iter = kzalloc(sizeof(*iter), GFP_KERNEL); 2705 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
3114 if (!iter) 2706 if (!iter) {
3115 return -ENOMEM; 2707 ret = -ENOMEM;
2708 goto out;
2709 }
3116 2710
3117 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) { 2711 /*
3118 kfree(iter); 2712 * We make a copy of the current tracer to avoid concurrent
3119 return -ENOMEM; 2713 * changes on it while we are reading.
2714 */
2715 iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
2716 if (!iter->trace) {
2717 ret = -ENOMEM;
2718 goto fail;
3120 } 2719 }
2720 if (current_trace)
2721 *iter->trace = *current_trace;
3121 2722
3122 mutex_lock(&trace_types_lock); 2723 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
2724 ret = -ENOMEM;
2725 goto fail;
2726 }
3123 2727
3124 /* trace pipe does not show start of buffer */ 2728 /* trace pipe does not show start of buffer */
3125 cpumask_setall(iter->started); 2729 cpumask_setall(iter->started);
3126 2730
2731 iter->cpu_file = cpu_file;
3127 iter->tr = &global_trace; 2732 iter->tr = &global_trace;
3128 iter->trace = current_trace; 2733 mutex_init(&iter->mutex);
3129 filp->private_data = iter; 2734 filp->private_data = iter;
3130 2735
3131 if (iter->trace->pipe_open) 2736 if (iter->trace->pipe_open)
3132 iter->trace->pipe_open(iter); 2737 iter->trace->pipe_open(iter);
2738
2739out:
3133 mutex_unlock(&trace_types_lock); 2740 mutex_unlock(&trace_types_lock);
2741 return ret;
3134 2742
3135 return 0; 2743fail:
2744 kfree(iter->trace);
2745 kfree(iter);
2746 mutex_unlock(&trace_types_lock);
2747 return ret;
3136} 2748}
3137 2749
3138static int tracing_release_pipe(struct inode *inode, struct file *file) 2750static int tracing_release_pipe(struct inode *inode, struct file *file)
3139{ 2751{
3140 struct trace_iterator *iter = file->private_data; 2752 struct trace_iterator *iter = file->private_data;
3141 2753
2754 mutex_lock(&trace_types_lock);
2755
2756 if (iter->cpu_file == TRACE_PIPE_ALL_CPU)
2757 cpumask_clear(tracing_reader_cpumask);
2758 else
2759 cpumask_clear_cpu(iter->cpu_file, tracing_reader_cpumask);
2760
2761 mutex_unlock(&trace_types_lock);
2762
3142 free_cpumask_var(iter->started); 2763 free_cpumask_var(iter->started);
2764 mutex_destroy(&iter->mutex);
2765 kfree(iter->trace);
3143 kfree(iter); 2766 kfree(iter);
3144 atomic_dec(&tracing_reader);
3145 2767
3146 return 0; 2768 return 0;
3147} 2769}
@@ -3167,67 +2789,57 @@ tracing_poll_pipe(struct file *filp, poll_table *poll_table)
3167 } 2789 }
3168} 2790}
3169 2791
3170/* 2792
3171 * Consumer reader. 2793void default_wait_pipe(struct trace_iterator *iter)
3172 */
3173static ssize_t
3174tracing_read_pipe(struct file *filp, char __user *ubuf,
3175 size_t cnt, loff_t *ppos)
3176{ 2794{
3177 struct trace_iterator *iter = filp->private_data; 2795 DEFINE_WAIT(wait);
3178 ssize_t sret;
3179 2796
3180 /* return any leftover data */ 2797 prepare_to_wait(&trace_wait, &wait, TASK_INTERRUPTIBLE);
3181 sret = trace_seq_to_user(&iter->seq, ubuf, cnt); 2798
3182 if (sret != -EBUSY) 2799 if (trace_empty(iter))
3183 return sret; 2800 schedule();
3184 2801
3185 trace_seq_reset(&iter->seq); 2802 finish_wait(&trace_wait, &wait);
2803}
3186 2804
3187 mutex_lock(&trace_types_lock); 2805/*
3188 if (iter->trace->read) { 2806 * This is a make-shift waitqueue.
3189 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos); 2807 * A tracer might use this callback on some rare cases:
3190 if (sret) 2808 *
3191 goto out; 2809 * 1) the current tracer might hold the runqueue lock when it wakes up
3192 } 2810 * a reader, hence a deadlock (sched, function, and function graph tracers)
2811 * 2) the function tracers, trace all functions, we don't want
2812 * the overhead of calling wake_up and friends
2813 * (and tracing them too)
2814 *
2815 * Anyway, this is really very primitive wakeup.
2816 */
2817void poll_wait_pipe(struct trace_iterator *iter)
2818{
2819 set_current_state(TASK_INTERRUPTIBLE);
2820 /* sleep for 100 msecs, and try again. */
2821 schedule_timeout(HZ / 10);
2822}
2823
2824/* Must be called with trace_types_lock mutex held. */
2825static int tracing_wait_pipe(struct file *filp)
2826{
2827 struct trace_iterator *iter = filp->private_data;
3193 2828
3194waitagain:
3195 sret = 0;
3196 while (trace_empty(iter)) { 2829 while (trace_empty(iter)) {
3197 2830
3198 if ((filp->f_flags & O_NONBLOCK)) { 2831 if ((filp->f_flags & O_NONBLOCK)) {
3199 sret = -EAGAIN; 2832 return -EAGAIN;
3200 goto out;
3201 } 2833 }
3202 2834
3203 /* 2835 mutex_unlock(&iter->mutex);
3204 * This is a make-shift waitqueue. The reason we don't use
3205 * an actual wait queue is because:
3206 * 1) we only ever have one waiter
3207 * 2) the tracing, traces all functions, we don't want
3208 * the overhead of calling wake_up and friends
3209 * (and tracing them too)
3210 * Anyway, this is really very primitive wakeup.
3211 */
3212 set_current_state(TASK_INTERRUPTIBLE);
3213 iter->tr->waiter = current;
3214
3215 mutex_unlock(&trace_types_lock);
3216
3217 /* sleep for 100 msecs, and try again. */
3218 schedule_timeout(HZ/10);
3219
3220 mutex_lock(&trace_types_lock);
3221 2836
3222 iter->tr->waiter = NULL; 2837 iter->trace->wait_pipe(iter);
3223 2838
3224 if (signal_pending(current)) { 2839 mutex_lock(&iter->mutex);
3225 sret = -EINTR;
3226 goto out;
3227 }
3228 2840
3229 if (iter->trace != current_trace) 2841 if (signal_pending(current))
3230 goto out; 2842 return -EINTR;
3231 2843
3232 /* 2844 /*
3233 * We block until we read something and tracing is disabled. 2845 * We block until we read something and tracing is disabled.
@@ -3240,13 +2852,59 @@ waitagain:
3240 */ 2852 */
3241 if (!tracer_enabled && iter->pos) 2853 if (!tracer_enabled && iter->pos)
3242 break; 2854 break;
2855 }
2856
2857 return 1;
2858}
2859
2860/*
2861 * Consumer reader.
2862 */
2863static ssize_t
2864tracing_read_pipe(struct file *filp, char __user *ubuf,
2865 size_t cnt, loff_t *ppos)
2866{
2867 struct trace_iterator *iter = filp->private_data;
2868 static struct tracer *old_tracer;
2869 ssize_t sret;
2870
2871 /* return any leftover data */
2872 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
2873 if (sret != -EBUSY)
2874 return sret;
2875
2876 trace_seq_init(&iter->seq);
2877
2878 /* copy the tracer to avoid using a global lock all around */
2879 mutex_lock(&trace_types_lock);
2880 if (unlikely(old_tracer != current_trace && current_trace)) {
2881 old_tracer = current_trace;
2882 *iter->trace = *current_trace;
2883 }
2884 mutex_unlock(&trace_types_lock);
3243 2885
3244 continue; 2886 /*
2887 * Avoid more than one consumer on a single file descriptor
2888 * This is just a matter of traces coherency, the ring buffer itself
2889 * is protected.
2890 */
2891 mutex_lock(&iter->mutex);
2892 if (iter->trace->read) {
2893 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
2894 if (sret)
2895 goto out;
3245 } 2896 }
3246 2897
2898waitagain:
2899 sret = tracing_wait_pipe(filp);
2900 if (sret <= 0)
2901 goto out;
2902
3247 /* stop when tracing is finished */ 2903 /* stop when tracing is finished */
3248 if (trace_empty(iter)) 2904 if (trace_empty(iter)) {
2905 sret = 0;
3249 goto out; 2906 goto out;
2907 }
3250 2908
3251 if (cnt >= PAGE_SIZE) 2909 if (cnt >= PAGE_SIZE)
3252 cnt = PAGE_SIZE - 1; 2910 cnt = PAGE_SIZE - 1;
@@ -3267,8 +2925,8 @@ waitagain:
3267 iter->seq.len = len; 2925 iter->seq.len = len;
3268 break; 2926 break;
3269 } 2927 }
3270 2928 if (ret != TRACE_TYPE_NO_CONSUME)
3271 trace_consume(iter); 2929 trace_consume(iter);
3272 2930
3273 if (iter->seq.len >= cnt) 2931 if (iter->seq.len >= cnt)
3274 break; 2932 break;
@@ -3277,7 +2935,7 @@ waitagain:
3277 /* Now copy what we have to the user */ 2935 /* Now copy what we have to the user */
3278 sret = trace_seq_to_user(&iter->seq, ubuf, cnt); 2936 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3279 if (iter->seq.readpos >= iter->seq.len) 2937 if (iter->seq.readpos >= iter->seq.len)
3280 trace_seq_reset(&iter->seq); 2938 trace_seq_init(&iter->seq);
3281 2939
3282 /* 2940 /*
3283 * If there was nothing to send to user, inspite of consuming trace 2941 * If there was nothing to send to user, inspite of consuming trace
@@ -3287,20 +2945,165 @@ waitagain:
3287 goto waitagain; 2945 goto waitagain;
3288 2946
3289out: 2947out:
3290 mutex_unlock(&trace_types_lock); 2948 mutex_unlock(&iter->mutex);
3291 2949
3292 return sret; 2950 return sret;
3293} 2951}
3294 2952
2953static void tracing_pipe_buf_release(struct pipe_inode_info *pipe,
2954 struct pipe_buffer *buf)
2955{
2956 __free_page(buf->page);
2957}
2958
2959static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
2960 unsigned int idx)
2961{
2962 __free_page(spd->pages[idx]);
2963}
2964
2965static struct pipe_buf_operations tracing_pipe_buf_ops = {
2966 .can_merge = 0,
2967 .map = generic_pipe_buf_map,
2968 .unmap = generic_pipe_buf_unmap,
2969 .confirm = generic_pipe_buf_confirm,
2970 .release = tracing_pipe_buf_release,
2971 .steal = generic_pipe_buf_steal,
2972 .get = generic_pipe_buf_get,
2973};
2974
2975static size_t
2976tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
2977{
2978 size_t count;
2979 int ret;
2980
2981 /* Seq buffer is page-sized, exactly what we need. */
2982 for (;;) {
2983 count = iter->seq.len;
2984 ret = print_trace_line(iter);
2985 count = iter->seq.len - count;
2986 if (rem < count) {
2987 rem = 0;
2988 iter->seq.len -= count;
2989 break;
2990 }
2991 if (ret == TRACE_TYPE_PARTIAL_LINE) {
2992 iter->seq.len -= count;
2993 break;
2994 }
2995
2996 trace_consume(iter);
2997 rem -= count;
2998 if (!find_next_entry_inc(iter)) {
2999 rem = 0;
3000 iter->ent = NULL;
3001 break;
3002 }
3003 }
3004
3005 return rem;
3006}
3007
3008static ssize_t tracing_splice_read_pipe(struct file *filp,
3009 loff_t *ppos,
3010 struct pipe_inode_info *pipe,
3011 size_t len,
3012 unsigned int flags)
3013{
3014 struct page *pages[PIPE_BUFFERS];
3015 struct partial_page partial[PIPE_BUFFERS];
3016 struct trace_iterator *iter = filp->private_data;
3017 struct splice_pipe_desc spd = {
3018 .pages = pages,
3019 .partial = partial,
3020 .nr_pages = 0, /* This gets updated below. */
3021 .flags = flags,
3022 .ops = &tracing_pipe_buf_ops,
3023 .spd_release = tracing_spd_release_pipe,
3024 };
3025 static struct tracer *old_tracer;
3026 ssize_t ret;
3027 size_t rem;
3028 unsigned int i;
3029
3030 /* copy the tracer to avoid using a global lock all around */
3031 mutex_lock(&trace_types_lock);
3032 if (unlikely(old_tracer != current_trace && current_trace)) {
3033 old_tracer = current_trace;
3034 *iter->trace = *current_trace;
3035 }
3036 mutex_unlock(&trace_types_lock);
3037
3038 mutex_lock(&iter->mutex);
3039
3040 if (iter->trace->splice_read) {
3041 ret = iter->trace->splice_read(iter, filp,
3042 ppos, pipe, len, flags);
3043 if (ret)
3044 goto out_err;
3045 }
3046
3047 ret = tracing_wait_pipe(filp);
3048 if (ret <= 0)
3049 goto out_err;
3050
3051 if (!iter->ent && !find_next_entry_inc(iter)) {
3052 ret = -EFAULT;
3053 goto out_err;
3054 }
3055
3056 /* Fill as many pages as possible. */
3057 for (i = 0, rem = len; i < PIPE_BUFFERS && rem; i++) {
3058 pages[i] = alloc_page(GFP_KERNEL);
3059 if (!pages[i])
3060 break;
3061
3062 rem = tracing_fill_pipe_page(rem, iter);
3063
3064 /* Copy the data into the page, so we can start over. */
3065 ret = trace_seq_to_buffer(&iter->seq,
3066 page_address(pages[i]),
3067 iter->seq.len);
3068 if (ret < 0) {
3069 __free_page(pages[i]);
3070 break;
3071 }
3072 partial[i].offset = 0;
3073 partial[i].len = iter->seq.len;
3074
3075 trace_seq_init(&iter->seq);
3076 }
3077
3078 mutex_unlock(&iter->mutex);
3079
3080 spd.nr_pages = i;
3081
3082 return splice_to_pipe(pipe, &spd);
3083
3084out_err:
3085 mutex_unlock(&iter->mutex);
3086
3087 return ret;
3088}
3089
3295static ssize_t 3090static ssize_t
3296tracing_entries_read(struct file *filp, char __user *ubuf, 3091tracing_entries_read(struct file *filp, char __user *ubuf,
3297 size_t cnt, loff_t *ppos) 3092 size_t cnt, loff_t *ppos)
3298{ 3093{
3299 struct trace_array *tr = filp->private_data; 3094 struct trace_array *tr = filp->private_data;
3300 char buf[64]; 3095 char buf[96];
3301 int r; 3096 int r;
3302 3097
3303 r = sprintf(buf, "%lu\n", tr->entries >> 10); 3098 mutex_lock(&trace_types_lock);
3099 if (!ring_buffer_expanded)
3100 r = sprintf(buf, "%lu (expanded: %lu)\n",
3101 tr->entries >> 10,
3102 trace_buf_size >> 10);
3103 else
3104 r = sprintf(buf, "%lu\n", tr->entries >> 10);
3105 mutex_unlock(&trace_types_lock);
3106
3304 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 3107 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3305} 3108}
3306 3109
@@ -3344,28 +3147,11 @@ tracing_entries_write(struct file *filp, const char __user *ubuf,
3344 val <<= 10; 3147 val <<= 10;
3345 3148
3346 if (val != global_trace.entries) { 3149 if (val != global_trace.entries) {
3347 ret = ring_buffer_resize(global_trace.buffer, val); 3150 ret = tracing_resize_ring_buffer(val);
3348 if (ret < 0) {
3349 cnt = ret;
3350 goto out;
3351 }
3352
3353 ret = ring_buffer_resize(max_tr.buffer, val);
3354 if (ret < 0) { 3151 if (ret < 0) {
3355 int r;
3356 cnt = ret; 3152 cnt = ret;
3357 r = ring_buffer_resize(global_trace.buffer,
3358 global_trace.entries);
3359 if (r < 0) {
3360 /* AARGH! We are left with different
3361 * size max buffer!!!! */
3362 WARN_ON(1);
3363 tracing_disabled = 1;
3364 }
3365 goto out; 3153 goto out;
3366 } 3154 }
3367
3368 global_trace.entries = val;
3369 } 3155 }
3370 3156
3371 filp->f_pos += cnt; 3157 filp->f_pos += cnt;
@@ -3393,7 +3179,7 @@ static int mark_printk(const char *fmt, ...)
3393 int ret; 3179 int ret;
3394 va_list args; 3180 va_list args;
3395 va_start(args, fmt); 3181 va_start(args, fmt);
3396 ret = trace_vprintk(0, -1, fmt, args); 3182 ret = trace_vprintk(0, fmt, args);
3397 va_end(args); 3183 va_end(args);
3398 return ret; 3184 return ret;
3399} 3185}
@@ -3433,42 +3219,295 @@ tracing_mark_write(struct file *filp, const char __user *ubuf,
3433 return cnt; 3219 return cnt;
3434} 3220}
3435 3221
3436static struct file_operations tracing_max_lat_fops = { 3222static const struct file_operations tracing_max_lat_fops = {
3437 .open = tracing_open_generic, 3223 .open = tracing_open_generic,
3438 .read = tracing_max_lat_read, 3224 .read = tracing_max_lat_read,
3439 .write = tracing_max_lat_write, 3225 .write = tracing_max_lat_write,
3440}; 3226};
3441 3227
3442static struct file_operations tracing_ctrl_fops = { 3228static const struct file_operations tracing_ctrl_fops = {
3443 .open = tracing_open_generic, 3229 .open = tracing_open_generic,
3444 .read = tracing_ctrl_read, 3230 .read = tracing_ctrl_read,
3445 .write = tracing_ctrl_write, 3231 .write = tracing_ctrl_write,
3446}; 3232};
3447 3233
3448static struct file_operations set_tracer_fops = { 3234static const struct file_operations set_tracer_fops = {
3449 .open = tracing_open_generic, 3235 .open = tracing_open_generic,
3450 .read = tracing_set_trace_read, 3236 .read = tracing_set_trace_read,
3451 .write = tracing_set_trace_write, 3237 .write = tracing_set_trace_write,
3452}; 3238};
3453 3239
3454static struct file_operations tracing_pipe_fops = { 3240static const struct file_operations tracing_pipe_fops = {
3455 .open = tracing_open_pipe, 3241 .open = tracing_open_pipe,
3456 .poll = tracing_poll_pipe, 3242 .poll = tracing_poll_pipe,
3457 .read = tracing_read_pipe, 3243 .read = tracing_read_pipe,
3244 .splice_read = tracing_splice_read_pipe,
3458 .release = tracing_release_pipe, 3245 .release = tracing_release_pipe,
3459}; 3246};
3460 3247
3461static struct file_operations tracing_entries_fops = { 3248static const struct file_operations tracing_entries_fops = {
3462 .open = tracing_open_generic, 3249 .open = tracing_open_generic,
3463 .read = tracing_entries_read, 3250 .read = tracing_entries_read,
3464 .write = tracing_entries_write, 3251 .write = tracing_entries_write,
3465}; 3252};
3466 3253
3467static struct file_operations tracing_mark_fops = { 3254static const struct file_operations tracing_mark_fops = {
3468 .open = tracing_open_generic, 3255 .open = tracing_open_generic,
3469 .write = tracing_mark_write, 3256 .write = tracing_mark_write,
3470}; 3257};
3471 3258
3259struct ftrace_buffer_info {
3260 struct trace_array *tr;
3261 void *spare;
3262 int cpu;
3263 unsigned int read;
3264};
3265
3266static int tracing_buffers_open(struct inode *inode, struct file *filp)
3267{
3268 int cpu = (int)(long)inode->i_private;
3269 struct ftrace_buffer_info *info;
3270
3271 if (tracing_disabled)
3272 return -ENODEV;
3273
3274 info = kzalloc(sizeof(*info), GFP_KERNEL);
3275 if (!info)
3276 return -ENOMEM;
3277
3278 info->tr = &global_trace;
3279 info->cpu = cpu;
3280 info->spare = NULL;
3281 /* Force reading ring buffer for first read */
3282 info->read = (unsigned int)-1;
3283
3284 filp->private_data = info;
3285
3286 return nonseekable_open(inode, filp);
3287}
3288
3289static ssize_t
3290tracing_buffers_read(struct file *filp, char __user *ubuf,
3291 size_t count, loff_t *ppos)
3292{
3293 struct ftrace_buffer_info *info = filp->private_data;
3294 unsigned int pos;
3295 ssize_t ret;
3296 size_t size;
3297
3298 if (!count)
3299 return 0;
3300
3301 if (!info->spare)
3302 info->spare = ring_buffer_alloc_read_page(info->tr->buffer);
3303 if (!info->spare)
3304 return -ENOMEM;
3305
3306 /* Do we have previous read data to read? */
3307 if (info->read < PAGE_SIZE)
3308 goto read;
3309
3310 info->read = 0;
3311
3312 ret = ring_buffer_read_page(info->tr->buffer,
3313 &info->spare,
3314 count,
3315 info->cpu, 0);
3316 if (ret < 0)
3317 return 0;
3318
3319 pos = ring_buffer_page_len(info->spare);
3320
3321 if (pos < PAGE_SIZE)
3322 memset(info->spare + pos, 0, PAGE_SIZE - pos);
3323
3324read:
3325 size = PAGE_SIZE - info->read;
3326 if (size > count)
3327 size = count;
3328
3329 ret = copy_to_user(ubuf, info->spare + info->read, size);
3330 if (ret == size)
3331 return -EFAULT;
3332 size -= ret;
3333
3334 *ppos += size;
3335 info->read += size;
3336
3337 return size;
3338}
3339
3340static int tracing_buffers_release(struct inode *inode, struct file *file)
3341{
3342 struct ftrace_buffer_info *info = file->private_data;
3343
3344 if (info->spare)
3345 ring_buffer_free_read_page(info->tr->buffer, info->spare);
3346 kfree(info);
3347
3348 return 0;
3349}
3350
3351struct buffer_ref {
3352 struct ring_buffer *buffer;
3353 void *page;
3354 int ref;
3355};
3356
3357static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
3358 struct pipe_buffer *buf)
3359{
3360 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
3361
3362 if (--ref->ref)
3363 return;
3364
3365 ring_buffer_free_read_page(ref->buffer, ref->page);
3366 kfree(ref);
3367 buf->private = 0;
3368}
3369
3370static int buffer_pipe_buf_steal(struct pipe_inode_info *pipe,
3371 struct pipe_buffer *buf)
3372{
3373 return 1;
3374}
3375
3376static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
3377 struct pipe_buffer *buf)
3378{
3379 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
3380
3381 ref->ref++;
3382}
3383
3384/* Pipe buffer operations for a buffer. */
3385static struct pipe_buf_operations buffer_pipe_buf_ops = {
3386 .can_merge = 0,
3387 .map = generic_pipe_buf_map,
3388 .unmap = generic_pipe_buf_unmap,
3389 .confirm = generic_pipe_buf_confirm,
3390 .release = buffer_pipe_buf_release,
3391 .steal = buffer_pipe_buf_steal,
3392 .get = buffer_pipe_buf_get,
3393};
3394
3395/*
3396 * Callback from splice_to_pipe(), if we need to release some pages
3397 * at the end of the spd in case we error'ed out in filling the pipe.
3398 */
3399static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
3400{
3401 struct buffer_ref *ref =
3402 (struct buffer_ref *)spd->partial[i].private;
3403
3404 if (--ref->ref)
3405 return;
3406
3407 ring_buffer_free_read_page(ref->buffer, ref->page);
3408 kfree(ref);
3409 spd->partial[i].private = 0;
3410}
3411
3412static ssize_t
3413tracing_buffers_splice_read(struct file *file, loff_t *ppos,
3414 struct pipe_inode_info *pipe, size_t len,
3415 unsigned int flags)
3416{
3417 struct ftrace_buffer_info *info = file->private_data;
3418 struct partial_page partial[PIPE_BUFFERS];
3419 struct page *pages[PIPE_BUFFERS];
3420 struct splice_pipe_desc spd = {
3421 .pages = pages,
3422 .partial = partial,
3423 .flags = flags,
3424 .ops = &buffer_pipe_buf_ops,
3425 .spd_release = buffer_spd_release,
3426 };
3427 struct buffer_ref *ref;
3428 int size, i;
3429 size_t ret;
3430
3431 if (*ppos & (PAGE_SIZE - 1)) {
3432 WARN_ONCE(1, "Ftrace: previous read must page-align\n");
3433 return -EINVAL;
3434 }
3435
3436 if (len & (PAGE_SIZE - 1)) {
3437 WARN_ONCE(1, "Ftrace: splice_read should page-align\n");
3438 if (len < PAGE_SIZE)
3439 return -EINVAL;
3440 len &= PAGE_MASK;
3441 }
3442
3443 for (i = 0; i < PIPE_BUFFERS && len; i++, len -= PAGE_SIZE) {
3444 struct page *page;
3445 int r;
3446
3447 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
3448 if (!ref)
3449 break;
3450
3451 ref->ref = 1;
3452 ref->buffer = info->tr->buffer;
3453 ref->page = ring_buffer_alloc_read_page(ref->buffer);
3454 if (!ref->page) {
3455 kfree(ref);
3456 break;
3457 }
3458
3459 r = ring_buffer_read_page(ref->buffer, &ref->page,
3460 len, info->cpu, 0);
3461 if (r < 0) {
3462 ring_buffer_free_read_page(ref->buffer,
3463 ref->page);
3464 kfree(ref);
3465 break;
3466 }
3467
3468 /*
3469 * zero out any left over data, this is going to
3470 * user land.
3471 */
3472 size = ring_buffer_page_len(ref->page);
3473 if (size < PAGE_SIZE)
3474 memset(ref->page + size, 0, PAGE_SIZE - size);
3475
3476 page = virt_to_page(ref->page);
3477
3478 spd.pages[i] = page;
3479 spd.partial[i].len = PAGE_SIZE;
3480 spd.partial[i].offset = 0;
3481 spd.partial[i].private = (unsigned long)ref;
3482 spd.nr_pages++;
3483 *ppos += PAGE_SIZE;
3484 }
3485
3486 spd.nr_pages = i;
3487
3488 /* did we read anything? */
3489 if (!spd.nr_pages) {
3490 if (flags & SPLICE_F_NONBLOCK)
3491 ret = -EAGAIN;
3492 else
3493 ret = 0;
3494 /* TODO: block */
3495 return ret;
3496 }
3497
3498 ret = splice_to_pipe(pipe, &spd);
3499
3500 return ret;
3501}
3502
3503static const struct file_operations tracing_buffers_fops = {
3504 .open = tracing_buffers_open,
3505 .read = tracing_buffers_read,
3506 .release = tracing_buffers_release,
3507 .splice_read = tracing_buffers_splice_read,
3508 .llseek = no_llseek,
3509};
3510
3472#ifdef CONFIG_DYNAMIC_FTRACE 3511#ifdef CONFIG_DYNAMIC_FTRACE
3473 3512
3474int __weak ftrace_arch_read_dyn_info(char *buf, int size) 3513int __weak ftrace_arch_read_dyn_info(char *buf, int size)
@@ -3500,7 +3539,7 @@ tracing_read_dyn_info(struct file *filp, char __user *ubuf,
3500 return r; 3539 return r;
3501} 3540}
3502 3541
3503static struct file_operations tracing_dyn_info_fops = { 3542static const struct file_operations tracing_dyn_info_fops = {
3504 .open = tracing_open_generic, 3543 .open = tracing_open_generic,
3505 .read = tracing_read_dyn_info, 3544 .read = tracing_read_dyn_info,
3506}; 3545};
@@ -3515,6 +3554,9 @@ struct dentry *tracing_init_dentry(void)
3515 if (d_tracer) 3554 if (d_tracer)
3516 return d_tracer; 3555 return d_tracer;
3517 3556
3557 if (!debugfs_initialized())
3558 return NULL;
3559
3518 d_tracer = debugfs_create_dir("tracing", NULL); 3560 d_tracer = debugfs_create_dir("tracing", NULL);
3519 3561
3520 if (!d_tracer && !once) { 3562 if (!d_tracer && !once) {
@@ -3526,15 +3568,350 @@ struct dentry *tracing_init_dentry(void)
3526 return d_tracer; 3568 return d_tracer;
3527} 3569}
3528 3570
3571static struct dentry *d_percpu;
3572
3573struct dentry *tracing_dentry_percpu(void)
3574{
3575 static int once;
3576 struct dentry *d_tracer;
3577
3578 if (d_percpu)
3579 return d_percpu;
3580
3581 d_tracer = tracing_init_dentry();
3582
3583 if (!d_tracer)
3584 return NULL;
3585
3586 d_percpu = debugfs_create_dir("per_cpu", d_tracer);
3587
3588 if (!d_percpu && !once) {
3589 once = 1;
3590 pr_warning("Could not create debugfs directory 'per_cpu'\n");
3591 return NULL;
3592 }
3593
3594 return d_percpu;
3595}
3596
3597static void tracing_init_debugfs_percpu(long cpu)
3598{
3599 struct dentry *d_percpu = tracing_dentry_percpu();
3600 struct dentry *entry, *d_cpu;
3601 /* strlen(cpu) + MAX(log10(cpu)) + '\0' */
3602 char cpu_dir[7];
3603
3604 if (cpu > 999 || cpu < 0)
3605 return;
3606
3607 sprintf(cpu_dir, "cpu%ld", cpu);
3608 d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
3609 if (!d_cpu) {
3610 pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
3611 return;
3612 }
3613
3614 /* per cpu trace_pipe */
3615 entry = debugfs_create_file("trace_pipe", 0444, d_cpu,
3616 (void *) cpu, &tracing_pipe_fops);
3617 if (!entry)
3618 pr_warning("Could not create debugfs 'trace_pipe' entry\n");
3619
3620 /* per cpu trace */
3621 entry = debugfs_create_file("trace", 0644, d_cpu,
3622 (void *) cpu, &tracing_fops);
3623 if (!entry)
3624 pr_warning("Could not create debugfs 'trace' entry\n");
3625
3626 entry = debugfs_create_file("trace_pipe_raw", 0444, d_cpu,
3627 (void *) cpu, &tracing_buffers_fops);
3628 if (!entry)
3629 pr_warning("Could not create debugfs 'trace_pipe_raw' entry\n");
3630}
3631
3529#ifdef CONFIG_FTRACE_SELFTEST 3632#ifdef CONFIG_FTRACE_SELFTEST
3530/* Let selftest have access to static functions in this file */ 3633/* Let selftest have access to static functions in this file */
3531#include "trace_selftest.c" 3634#include "trace_selftest.c"
3532#endif 3635#endif
3533 3636
3637struct trace_option_dentry {
3638 struct tracer_opt *opt;
3639 struct tracer_flags *flags;
3640 struct dentry *entry;
3641};
3642
3643static ssize_t
3644trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
3645 loff_t *ppos)
3646{
3647 struct trace_option_dentry *topt = filp->private_data;
3648 char *buf;
3649
3650 if (topt->flags->val & topt->opt->bit)
3651 buf = "1\n";
3652 else
3653 buf = "0\n";
3654
3655 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
3656}
3657
3658static ssize_t
3659trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
3660 loff_t *ppos)
3661{
3662 struct trace_option_dentry *topt = filp->private_data;
3663 unsigned long val;
3664 char buf[64];
3665 int ret;
3666
3667 if (cnt >= sizeof(buf))
3668 return -EINVAL;
3669
3670 if (copy_from_user(&buf, ubuf, cnt))
3671 return -EFAULT;
3672
3673 buf[cnt] = 0;
3674
3675 ret = strict_strtoul(buf, 10, &val);
3676 if (ret < 0)
3677 return ret;
3678
3679 ret = 0;
3680 switch (val) {
3681 case 0:
3682 /* do nothing if already cleared */
3683 if (!(topt->flags->val & topt->opt->bit))
3684 break;
3685
3686 mutex_lock(&trace_types_lock);
3687 if (current_trace->set_flag)
3688 ret = current_trace->set_flag(topt->flags->val,
3689 topt->opt->bit, 0);
3690 mutex_unlock(&trace_types_lock);
3691 if (ret)
3692 return ret;
3693 topt->flags->val &= ~topt->opt->bit;
3694 break;
3695 case 1:
3696 /* do nothing if already set */
3697 if (topt->flags->val & topt->opt->bit)
3698 break;
3699
3700 mutex_lock(&trace_types_lock);
3701 if (current_trace->set_flag)
3702 ret = current_trace->set_flag(topt->flags->val,
3703 topt->opt->bit, 1);
3704 mutex_unlock(&trace_types_lock);
3705 if (ret)
3706 return ret;
3707 topt->flags->val |= topt->opt->bit;
3708 break;
3709
3710 default:
3711 return -EINVAL;
3712 }
3713
3714 *ppos += cnt;
3715
3716 return cnt;
3717}
3718
3719
3720static const struct file_operations trace_options_fops = {
3721 .open = tracing_open_generic,
3722 .read = trace_options_read,
3723 .write = trace_options_write,
3724};
3725
3726static ssize_t
3727trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
3728 loff_t *ppos)
3729{
3730 long index = (long)filp->private_data;
3731 char *buf;
3732
3733 if (trace_flags & (1 << index))
3734 buf = "1\n";
3735 else
3736 buf = "0\n";
3737
3738 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
3739}
3740
3741static ssize_t
3742trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
3743 loff_t *ppos)
3744{
3745 long index = (long)filp->private_data;
3746 char buf[64];
3747 unsigned long val;
3748 int ret;
3749
3750 if (cnt >= sizeof(buf))
3751 return -EINVAL;
3752
3753 if (copy_from_user(&buf, ubuf, cnt))
3754 return -EFAULT;
3755
3756 buf[cnt] = 0;
3757
3758 ret = strict_strtoul(buf, 10, &val);
3759 if (ret < 0)
3760 return ret;
3761
3762 switch (val) {
3763 case 0:
3764 trace_flags &= ~(1 << index);
3765 break;
3766 case 1:
3767 trace_flags |= 1 << index;
3768 break;
3769
3770 default:
3771 return -EINVAL;
3772 }
3773
3774 *ppos += cnt;
3775
3776 return cnt;
3777}
3778
3779static const struct file_operations trace_options_core_fops = {
3780 .open = tracing_open_generic,
3781 .read = trace_options_core_read,
3782 .write = trace_options_core_write,
3783};
3784
3785static struct dentry *trace_options_init_dentry(void)
3786{
3787 struct dentry *d_tracer;
3788 static struct dentry *t_options;
3789
3790 if (t_options)
3791 return t_options;
3792
3793 d_tracer = tracing_init_dentry();
3794 if (!d_tracer)
3795 return NULL;
3796
3797 t_options = debugfs_create_dir("options", d_tracer);
3798 if (!t_options) {
3799 pr_warning("Could not create debugfs directory 'options'\n");
3800 return NULL;
3801 }
3802
3803 return t_options;
3804}
3805
3806static void
3807create_trace_option_file(struct trace_option_dentry *topt,
3808 struct tracer_flags *flags,
3809 struct tracer_opt *opt)
3810{
3811 struct dentry *t_options;
3812 struct dentry *entry;
3813
3814 t_options = trace_options_init_dentry();
3815 if (!t_options)
3816 return;
3817
3818 topt->flags = flags;
3819 topt->opt = opt;
3820
3821 entry = debugfs_create_file(opt->name, 0644, t_options, topt,
3822 &trace_options_fops);
3823
3824 topt->entry = entry;
3825
3826}
3827
3828static struct trace_option_dentry *
3829create_trace_option_files(struct tracer *tracer)
3830{
3831 struct trace_option_dentry *topts;
3832 struct tracer_flags *flags;
3833 struct tracer_opt *opts;
3834 int cnt;
3835
3836 if (!tracer)
3837 return NULL;
3838
3839 flags = tracer->flags;
3840
3841 if (!flags || !flags->opts)
3842 return NULL;
3843
3844 opts = flags->opts;
3845
3846 for (cnt = 0; opts[cnt].name; cnt++)
3847 ;
3848
3849 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
3850 if (!topts)
3851 return NULL;
3852
3853 for (cnt = 0; opts[cnt].name; cnt++)
3854 create_trace_option_file(&topts[cnt], flags,
3855 &opts[cnt]);
3856
3857 return topts;
3858}
3859
3860static void
3861destroy_trace_option_files(struct trace_option_dentry *topts)
3862{
3863 int cnt;
3864
3865 if (!topts)
3866 return;
3867
3868 for (cnt = 0; topts[cnt].opt; cnt++) {
3869 if (topts[cnt].entry)
3870 debugfs_remove(topts[cnt].entry);
3871 }
3872
3873 kfree(topts);
3874}
3875
3876static struct dentry *
3877create_trace_option_core_file(const char *option, long index)
3878{
3879 struct dentry *t_options;
3880 struct dentry *entry;
3881
3882 t_options = trace_options_init_dentry();
3883 if (!t_options)
3884 return NULL;
3885
3886 entry = debugfs_create_file(option, 0644, t_options, (void *)index,
3887 &trace_options_core_fops);
3888
3889 return entry;
3890}
3891
3892static __init void create_trace_options_dir(void)
3893{
3894 struct dentry *t_options;
3895 struct dentry *entry;
3896 int i;
3897
3898 t_options = trace_options_init_dentry();
3899 if (!t_options)
3900 return;
3901
3902 for (i = 0; trace_options[i]; i++) {
3903 entry = create_trace_option_core_file(trace_options[i], i);
3904 if (!entry)
3905 pr_warning("Could not create debugfs %s entry\n",
3906 trace_options[i]);
3907 }
3908}
3909
3534static __init int tracer_init_debugfs(void) 3910static __init int tracer_init_debugfs(void)
3535{ 3911{
3536 struct dentry *d_tracer; 3912 struct dentry *d_tracer;
3537 struct dentry *entry; 3913 struct dentry *entry;
3914 int cpu;
3538 3915
3539 d_tracer = tracing_init_dentry(); 3916 d_tracer = tracing_init_dentry();
3540 3917
@@ -3548,18 +3925,15 @@ static __init int tracer_init_debugfs(void)
3548 if (!entry) 3925 if (!entry)
3549 pr_warning("Could not create debugfs 'trace_options' entry\n"); 3926 pr_warning("Could not create debugfs 'trace_options' entry\n");
3550 3927
3928 create_trace_options_dir();
3929
3551 entry = debugfs_create_file("tracing_cpumask", 0644, d_tracer, 3930 entry = debugfs_create_file("tracing_cpumask", 0644, d_tracer,
3552 NULL, &tracing_cpumask_fops); 3931 NULL, &tracing_cpumask_fops);
3553 if (!entry) 3932 if (!entry)
3554 pr_warning("Could not create debugfs 'tracing_cpumask' entry\n"); 3933 pr_warning("Could not create debugfs 'tracing_cpumask' entry\n");
3555 3934
3556 entry = debugfs_create_file("latency_trace", 0444, d_tracer, 3935 entry = debugfs_create_file("trace", 0644, d_tracer,
3557 &global_trace, &tracing_lt_fops); 3936 (void *) TRACE_PIPE_ALL_CPU, &tracing_fops);
3558 if (!entry)
3559 pr_warning("Could not create debugfs 'latency_trace' entry\n");
3560
3561 entry = debugfs_create_file("trace", 0444, d_tracer,
3562 &global_trace, &tracing_fops);
3563 if (!entry) 3937 if (!entry)
3564 pr_warning("Could not create debugfs 'trace' entry\n"); 3938 pr_warning("Could not create debugfs 'trace' entry\n");
3565 3939
@@ -3590,8 +3964,8 @@ static __init int tracer_init_debugfs(void)
3590 if (!entry) 3964 if (!entry)
3591 pr_warning("Could not create debugfs 'README' entry\n"); 3965 pr_warning("Could not create debugfs 'README' entry\n");
3592 3966
3593 entry = debugfs_create_file("trace_pipe", 0644, d_tracer, 3967 entry = debugfs_create_file("trace_pipe", 0444, d_tracer,
3594 NULL, &tracing_pipe_fops); 3968 (void *) TRACE_PIPE_ALL_CPU, &tracing_pipe_fops);
3595 if (!entry) 3969 if (!entry)
3596 pr_warning("Could not create debugfs " 3970 pr_warning("Could not create debugfs "
3597 "'trace_pipe' entry\n"); 3971 "'trace_pipe' entry\n");
@@ -3619,77 +3993,12 @@ static __init int tracer_init_debugfs(void)
3619#ifdef CONFIG_SYSPROF_TRACER 3993#ifdef CONFIG_SYSPROF_TRACER
3620 init_tracer_sysprof_debugfs(d_tracer); 3994 init_tracer_sysprof_debugfs(d_tracer);
3621#endif 3995#endif
3622 return 0;
3623}
3624
3625int trace_vprintk(unsigned long ip, int depth, const char *fmt, va_list args)
3626{
3627 static DEFINE_SPINLOCK(trace_buf_lock);
3628 static char trace_buf[TRACE_BUF_SIZE];
3629
3630 struct ring_buffer_event *event;
3631 struct trace_array *tr = &global_trace;
3632 struct trace_array_cpu *data;
3633 int cpu, len = 0, size, pc;
3634 struct print_entry *entry;
3635 unsigned long irq_flags;
3636
3637 if (tracing_disabled || tracing_selftest_running)
3638 return 0;
3639
3640 pc = preempt_count();
3641 preempt_disable_notrace();
3642 cpu = raw_smp_processor_id();
3643 data = tr->data[cpu];
3644
3645 if (unlikely(atomic_read(&data->disabled)))
3646 goto out;
3647
3648 pause_graph_tracing();
3649 spin_lock_irqsave(&trace_buf_lock, irq_flags);
3650 len = vsnprintf(trace_buf, TRACE_BUF_SIZE, fmt, args);
3651
3652 len = min(len, TRACE_BUF_SIZE-1);
3653 trace_buf[len] = 0;
3654
3655 size = sizeof(*entry) + len + 1;
3656 event = ring_buffer_lock_reserve(tr->buffer, size, &irq_flags);
3657 if (!event)
3658 goto out_unlock;
3659 entry = ring_buffer_event_data(event);
3660 tracing_generic_entry_update(&entry->ent, irq_flags, pc);
3661 entry->ent.type = TRACE_PRINT;
3662 entry->ip = ip;
3663 entry->depth = depth;
3664
3665 memcpy(&entry->buf, trace_buf, len);
3666 entry->buf[len] = 0;
3667 ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
3668
3669 out_unlock:
3670 spin_unlock_irqrestore(&trace_buf_lock, irq_flags);
3671 unpause_graph_tracing();
3672 out:
3673 preempt_enable_notrace();
3674
3675 return len;
3676}
3677EXPORT_SYMBOL_GPL(trace_vprintk);
3678 3996
3679int __ftrace_printk(unsigned long ip, const char *fmt, ...) 3997 for_each_tracing_cpu(cpu)
3680{ 3998 tracing_init_debugfs_percpu(cpu);
3681 int ret;
3682 va_list ap;
3683
3684 if (!(trace_flags & TRACE_ITER_PRINTK))
3685 return 0;
3686 3999
3687 va_start(ap, fmt); 4000 return 0;
3688 ret = trace_vprintk(ip, task_curr_ret_stack(current), fmt, ap);
3689 va_end(ap);
3690 return ret;
3691} 4001}
3692EXPORT_SYMBOL_GPL(__ftrace_printk);
3693 4002
3694static int trace_panic_handler(struct notifier_block *this, 4003static int trace_panic_handler(struct notifier_block *this,
3695 unsigned long event, void *unused) 4004 unsigned long event, void *unused)
@@ -3750,14 +4059,15 @@ trace_printk_seq(struct trace_seq *s)
3750 4059
3751 printk(KERN_TRACE "%s", s->buffer); 4060 printk(KERN_TRACE "%s", s->buffer);
3752 4061
3753 trace_seq_reset(s); 4062 trace_seq_init(s);
3754} 4063}
3755 4064
3756void ftrace_dump(void) 4065static void __ftrace_dump(bool disable_tracing)
3757{ 4066{
3758 static DEFINE_SPINLOCK(ftrace_dump_lock); 4067 static DEFINE_SPINLOCK(ftrace_dump_lock);
3759 /* use static because iter can be a bit big for the stack */ 4068 /* use static because iter can be a bit big for the stack */
3760 static struct trace_iterator iter; 4069 static struct trace_iterator iter;
4070 unsigned int old_userobj;
3761 static int dump_ran; 4071 static int dump_ran;
3762 unsigned long flags; 4072 unsigned long flags;
3763 int cnt = 0, cpu; 4073 int cnt = 0, cpu;
@@ -3769,21 +4079,26 @@ void ftrace_dump(void)
3769 4079
3770 dump_ran = 1; 4080 dump_ran = 1;
3771 4081
3772 /* No turning back! */
3773 tracing_off(); 4082 tracing_off();
3774 ftrace_kill(); 4083
4084 if (disable_tracing)
4085 ftrace_kill();
3775 4086
3776 for_each_tracing_cpu(cpu) { 4087 for_each_tracing_cpu(cpu) {
3777 atomic_inc(&global_trace.data[cpu]->disabled); 4088 atomic_inc(&global_trace.data[cpu]->disabled);
3778 } 4089 }
3779 4090
4091 old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
4092
3780 /* don't look at user memory in panic mode */ 4093 /* don't look at user memory in panic mode */
3781 trace_flags &= ~TRACE_ITER_SYM_USEROBJ; 4094 trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
3782 4095
3783 printk(KERN_TRACE "Dumping ftrace buffer:\n"); 4096 printk(KERN_TRACE "Dumping ftrace buffer:\n");
3784 4097
4098 /* Simulate the iterator */
3785 iter.tr = &global_trace; 4099 iter.tr = &global_trace;
3786 iter.trace = current_trace; 4100 iter.trace = current_trace;
4101 iter.cpu_file = TRACE_PIPE_ALL_CPU;
3787 4102
3788 /* 4103 /*
3789 * We need to stop all tracing on all CPUS to read the 4104 * We need to stop all tracing on all CPUS to read the
@@ -3819,13 +4134,30 @@ void ftrace_dump(void)
3819 else 4134 else
3820 printk(KERN_TRACE "---------------------------------\n"); 4135 printk(KERN_TRACE "---------------------------------\n");
3821 4136
4137 /* Re-enable tracing if requested */
4138 if (!disable_tracing) {
4139 trace_flags |= old_userobj;
4140
4141 for_each_tracing_cpu(cpu) {
4142 atomic_dec(&global_trace.data[cpu]->disabled);
4143 }
4144 tracing_on();
4145 }
4146
3822 out: 4147 out:
3823 spin_unlock_irqrestore(&ftrace_dump_lock, flags); 4148 spin_unlock_irqrestore(&ftrace_dump_lock, flags);
3824} 4149}
3825 4150
4151/* By default: disable tracing after the dump */
4152void ftrace_dump(void)
4153{
4154 __ftrace_dump(true);
4155}
4156
3826__init static int tracer_alloc_buffers(void) 4157__init static int tracer_alloc_buffers(void)
3827{ 4158{
3828 struct trace_array_cpu *data; 4159 struct trace_array_cpu *data;
4160 int ring_buf_size;
3829 int i; 4161 int i;
3830 int ret = -ENOMEM; 4162 int ret = -ENOMEM;
3831 4163
@@ -3835,11 +4167,21 @@ __init static int tracer_alloc_buffers(void)
3835 if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL)) 4167 if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL))
3836 goto out_free_buffer_mask; 4168 goto out_free_buffer_mask;
3837 4169
4170 if (!alloc_cpumask_var(&tracing_reader_cpumask, GFP_KERNEL))
4171 goto out_free_tracing_cpumask;
4172
4173 /* To save memory, keep the ring buffer size to its minimum */
4174 if (ring_buffer_expanded)
4175 ring_buf_size = trace_buf_size;
4176 else
4177 ring_buf_size = 1;
4178
3838 cpumask_copy(tracing_buffer_mask, cpu_possible_mask); 4179 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
3839 cpumask_copy(tracing_cpumask, cpu_all_mask); 4180 cpumask_copy(tracing_cpumask, cpu_all_mask);
4181 cpumask_clear(tracing_reader_cpumask);
3840 4182
3841 /* TODO: make the number of buffers hot pluggable with CPUS */ 4183 /* TODO: make the number of buffers hot pluggable with CPUS */
3842 global_trace.buffer = ring_buffer_alloc(trace_buf_size, 4184 global_trace.buffer = ring_buffer_alloc(ring_buf_size,
3843 TRACE_BUFFER_FLAGS); 4185 TRACE_BUFFER_FLAGS);
3844 if (!global_trace.buffer) { 4186 if (!global_trace.buffer) {
3845 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n"); 4187 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
@@ -3850,7 +4192,7 @@ __init static int tracer_alloc_buffers(void)
3850 4192
3851 4193
3852#ifdef CONFIG_TRACER_MAX_TRACE 4194#ifdef CONFIG_TRACER_MAX_TRACE
3853 max_tr.buffer = ring_buffer_alloc(trace_buf_size, 4195 max_tr.buffer = ring_buffer_alloc(ring_buf_size,
3854 TRACE_BUFFER_FLAGS); 4196 TRACE_BUFFER_FLAGS);
3855 if (!max_tr.buffer) { 4197 if (!max_tr.buffer) {
3856 printk(KERN_ERR "tracer: failed to allocate max ring buffer!\n"); 4198 printk(KERN_ERR "tracer: failed to allocate max ring buffer!\n");
@@ -3871,14 +4213,10 @@ __init static int tracer_alloc_buffers(void)
3871 trace_init_cmdlines(); 4213 trace_init_cmdlines();
3872 4214
3873 register_tracer(&nop_trace); 4215 register_tracer(&nop_trace);
4216 current_trace = &nop_trace;
3874#ifdef CONFIG_BOOT_TRACER 4217#ifdef CONFIG_BOOT_TRACER
3875 register_tracer(&boot_tracer); 4218 register_tracer(&boot_tracer);
3876 current_trace = &boot_tracer;
3877 current_trace->init(&global_trace);
3878#else
3879 current_trace = &nop_trace;
3880#endif 4219#endif
3881
3882 /* All seems OK, enable tracing */ 4220 /* All seems OK, enable tracing */
3883 tracing_disabled = 0; 4221 tracing_disabled = 0;
3884 4222
@@ -3886,14 +4224,38 @@ __init static int tracer_alloc_buffers(void)
3886 &trace_panic_notifier); 4224 &trace_panic_notifier);
3887 4225
3888 register_die_notifier(&trace_die_notifier); 4226 register_die_notifier(&trace_die_notifier);
3889 ret = 0; 4227
4228 return 0;
3890 4229
3891out_free_cpumask: 4230out_free_cpumask:
4231 free_cpumask_var(tracing_reader_cpumask);
4232out_free_tracing_cpumask:
3892 free_cpumask_var(tracing_cpumask); 4233 free_cpumask_var(tracing_cpumask);
3893out_free_buffer_mask: 4234out_free_buffer_mask:
3894 free_cpumask_var(tracing_buffer_mask); 4235 free_cpumask_var(tracing_buffer_mask);
3895out: 4236out:
3896 return ret; 4237 return ret;
3897} 4238}
4239
4240__init static int clear_boot_tracer(void)
4241{
4242 /*
4243 * The default tracer at boot buffer is an init section.
4244 * This function is called in lateinit. If we did not
4245 * find the boot tracer, then clear it out, to prevent
4246 * later registration from accessing the buffer that is
4247 * about to be freed.
4248 */
4249 if (!default_bootup_tracer)
4250 return 0;
4251
4252 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
4253 default_bootup_tracer);
4254 default_bootup_tracer = NULL;
4255
4256 return 0;
4257}
4258
3898early_initcall(tracer_alloc_buffers); 4259early_initcall(tracer_alloc_buffers);
3899fs_initcall(tracer_init_debugfs); 4260fs_initcall(tracer_init_debugfs);
4261late_initcall(clear_boot_tracer);
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-13 18:13:16 -0500