aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/trace/trace.c
diff options
context:
space:
mode:
authorIngo Molnar <mingo@elte.hu>2008-07-10 05:43:00 -0400
committerIngo Molnar <mingo@elte.hu>2008-07-10 05:43:00 -0400
commitbac0c9103b31c3dd83ad9d731dd9834e2ba75e4f (patch)
tree702dd6a7ce06d224d594c2293af546b11ac9f51b /kernel/trace/trace.c
parent6329d3021bcfa9038621e6e917d98929421d8ec8 (diff)
parent98a05ed4bd7774f533ab185fe0bf2fdc58292d7c (diff)
Merge branch 'tracing/ftrace' into auto-ftrace-next
Diffstat (limited to 'kernel/trace/trace.c')
-rw-r--r--kernel/trace/trace.c3100
1 files changed, 3100 insertions, 0 deletions
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
new file mode 100644
index 000000000000..9ade79369bfb
--- /dev/null
+++ b/kernel/trace/trace.c
@@ -0,0 +1,3100 @@
1/*
2 * ring buffer based function tracer
3 *
4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
6 *
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
9 *
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
12 * Copyright (C) 2004 William Lee Irwin III
13 */
14#include <linux/utsrelease.h>
15#include <linux/kallsyms.h>
16#include <linux/seq_file.h>
17#include <linux/debugfs.h>
18#include <linux/pagemap.h>
19#include <linux/hardirq.h>
20#include <linux/linkage.h>
21#include <linux/uaccess.h>
22#include <linux/ftrace.h>
23#include <linux/module.h>
24#include <linux/percpu.h>
25#include <linux/ctype.h>
26#include <linux/init.h>
27#include <linux/poll.h>
28#include <linux/gfp.h>
29#include <linux/fs.h>
30#include <linux/kprobes.h>
31#include <linux/writeback.h>
32
33#include <linux/stacktrace.h>
34
35#include "trace.h"
36
37unsigned long __read_mostly tracing_max_latency = (cycle_t)ULONG_MAX;
38unsigned long __read_mostly tracing_thresh;
39
40static unsigned long __read_mostly tracing_nr_buffers;
41static cpumask_t __read_mostly tracing_buffer_mask;
42
43#define for_each_tracing_cpu(cpu) \
44 for_each_cpu_mask(cpu, tracing_buffer_mask)
45
46static int trace_alloc_page(void);
47static int trace_free_page(void);
48
49static int tracing_disabled = 1;
50
51static unsigned long tracing_pages_allocated;
52
53long
54ns2usecs(cycle_t nsec)
55{
56 nsec += 500;
57 do_div(nsec, 1000);
58 return nsec;
59}
60
61cycle_t ftrace_now(int cpu)
62{
63 return cpu_clock(cpu);
64}
65
66/*
67 * The global_trace is the descriptor that holds the tracing
68 * buffers for the live tracing. For each CPU, it contains
69 * a link list of pages that will store trace entries. The
70 * page descriptor of the pages in the memory is used to hold
71 * the link list by linking the lru item in the page descriptor
72 * to each of the pages in the buffer per CPU.
73 *
74 * For each active CPU there is a data field that holds the
75 * pages for the buffer for that CPU. Each CPU has the same number
76 * of pages allocated for its buffer.
77 */
78static struct trace_array global_trace;
79
80static DEFINE_PER_CPU(struct trace_array_cpu, global_trace_cpu);
81
82/*
83 * The max_tr is used to snapshot the global_trace when a maximum
84 * latency is reached. Some tracers will use this to store a maximum
85 * trace while it continues examining live traces.
86 *
87 * The buffers for the max_tr are set up the same as the global_trace.
88 * When a snapshot is taken, the link list of the max_tr is swapped
89 * with the link list of the global_trace and the buffers are reset for
90 * the global_trace so the tracing can continue.
91 */
92static struct trace_array max_tr;
93
94static DEFINE_PER_CPU(struct trace_array_cpu, max_data);
95
96/* tracer_enabled is used to toggle activation of a tracer */
97static int tracer_enabled = 1;
98
99/*
100 * trace_nr_entries is the number of entries that is allocated
101 * for a buffer. Note, the number of entries is always rounded
102 * to ENTRIES_PER_PAGE.
103 */
104static unsigned long trace_nr_entries = 65536UL;
105
106/* trace_types holds a link list of available tracers. */
107static struct tracer *trace_types __read_mostly;
108
109/* current_trace points to the tracer that is currently active */
110static struct tracer *current_trace __read_mostly;
111
112/*
113 * max_tracer_type_len is used to simplify the allocating of
114 * buffers to read userspace tracer names. We keep track of
115 * the longest tracer name registered.
116 */
117static int max_tracer_type_len;
118
119/*
120 * trace_types_lock is used to protect the trace_types list.
121 * This lock is also used to keep user access serialized.
122 * Accesses from userspace will grab this lock while userspace
123 * activities happen inside the kernel.
124 */
125static DEFINE_MUTEX(trace_types_lock);
126
127/* trace_wait is a waitqueue for tasks blocked on trace_poll */
128static DECLARE_WAIT_QUEUE_HEAD(trace_wait);
129
130/* trace_flags holds iter_ctrl options */
131unsigned long trace_flags = TRACE_ITER_PRINT_PARENT;
132
133static notrace void no_trace_init(struct trace_array *tr)
134{
135 int cpu;
136
137 if(tr->ctrl)
138 for_each_online_cpu(cpu)
139 tracing_reset(tr->data[cpu]);
140 tracer_enabled = 0;
141}
142
143/* dummy trace to disable tracing */
144static struct tracer no_tracer __read_mostly = {
145 .name = "none",
146 .init = no_trace_init
147};
148
149
150/**
151 * trace_wake_up - wake up tasks waiting for trace input
152 *
153 * Simply wakes up any task that is blocked on the trace_wait
154 * queue. These is used with trace_poll for tasks polling the trace.
155 */
156void trace_wake_up(void)
157{
158 /*
159 * The runqueue_is_locked() can fail, but this is the best we
160 * have for now:
161 */
162 if (!(trace_flags & TRACE_ITER_BLOCK) && !runqueue_is_locked())
163 wake_up(&trace_wait);
164}
165
166#define ENTRIES_PER_PAGE (PAGE_SIZE / sizeof(struct trace_entry))
167
168static int __init set_nr_entries(char *str)
169{
170 unsigned long nr_entries;
171 int ret;
172
173 if (!str)
174 return 0;
175 ret = strict_strtoul(str, 0, &nr_entries);
176 /* nr_entries can not be zero */
177 if (ret < 0 || nr_entries == 0)
178 return 0;
179 trace_nr_entries = nr_entries;
180 return 1;
181}
182__setup("trace_entries=", set_nr_entries);
183
184unsigned long nsecs_to_usecs(unsigned long nsecs)
185{
186 return nsecs / 1000;
187}
188
189/*
190 * trace_flag_type is an enumeration that holds different
191 * states when a trace occurs. These are:
192 * IRQS_OFF - interrupts were disabled
193 * NEED_RESCED - reschedule is requested
194 * HARDIRQ - inside an interrupt handler
195 * SOFTIRQ - inside a softirq handler
196 */
197enum trace_flag_type {
198 TRACE_FLAG_IRQS_OFF = 0x01,
199 TRACE_FLAG_NEED_RESCHED = 0x02,
200 TRACE_FLAG_HARDIRQ = 0x04,
201 TRACE_FLAG_SOFTIRQ = 0x08,
202};
203
204/*
205 * TRACE_ITER_SYM_MASK masks the options in trace_flags that
206 * control the output of kernel symbols.
207 */
208#define TRACE_ITER_SYM_MASK \
209 (TRACE_ITER_PRINT_PARENT|TRACE_ITER_SYM_OFFSET|TRACE_ITER_SYM_ADDR)
210
211/* These must match the bit postions in trace_iterator_flags */
212static const char *trace_options[] = {
213 "print-parent",
214 "sym-offset",
215 "sym-addr",
216 "verbose",
217 "raw",
218 "hex",
219 "bin",
220 "block",
221 "stacktrace",
222 "sched-tree",
223 NULL
224};
225
226/*
227 * ftrace_max_lock is used to protect the swapping of buffers
228 * when taking a max snapshot. The buffers themselves are
229 * protected by per_cpu spinlocks. But the action of the swap
230 * needs its own lock.
231 *
232 * This is defined as a raw_spinlock_t in order to help
233 * with performance when lockdep debugging is enabled.
234 */
235static raw_spinlock_t ftrace_max_lock =
236 (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
237
238/*
239 * Copy the new maximum trace into the separate maximum-trace
240 * structure. (this way the maximum trace is permanently saved,
241 * for later retrieval via /debugfs/tracing/latency_trace)
242 */
243static void
244__update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
245{
246 struct trace_array_cpu *data = tr->data[cpu];
247
248 max_tr.cpu = cpu;
249 max_tr.time_start = data->preempt_timestamp;
250
251 data = max_tr.data[cpu];
252 data->saved_latency = tracing_max_latency;
253
254 memcpy(data->comm, tsk->comm, TASK_COMM_LEN);
255 data->pid = tsk->pid;
256 data->uid = tsk->uid;
257 data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
258 data->policy = tsk->policy;
259 data->rt_priority = tsk->rt_priority;
260
261 /* record this tasks comm */
262 tracing_record_cmdline(current);
263}
264
265#define CHECK_COND(cond) \
266 if (unlikely(cond)) { \
267 tracing_disabled = 1; \
268 WARN_ON(1); \
269 return -1; \
270 }
271
272/**
273 * check_pages - integrity check of trace buffers
274 *
275 * As a safty measure we check to make sure the data pages have not
276 * been corrupted.
277 */
278int check_pages(struct trace_array_cpu *data)
279{
280 struct page *page, *tmp;
281
282 CHECK_COND(data->trace_pages.next->prev != &data->trace_pages);
283 CHECK_COND(data->trace_pages.prev->next != &data->trace_pages);
284
285 list_for_each_entry_safe(page, tmp, &data->trace_pages, lru) {
286 CHECK_COND(page->lru.next->prev != &page->lru);
287 CHECK_COND(page->lru.prev->next != &page->lru);
288 }
289
290 return 0;
291}
292
293/**
294 * head_page - page address of the first page in per_cpu buffer.
295 *
296 * head_page returns the page address of the first page in
297 * a per_cpu buffer. This also preforms various consistency
298 * checks to make sure the buffer has not been corrupted.
299 */
300void *head_page(struct trace_array_cpu *data)
301{
302 struct page *page;
303
304 if (list_empty(&data->trace_pages))
305 return NULL;
306
307 page = list_entry(data->trace_pages.next, struct page, lru);
308 BUG_ON(&page->lru == &data->trace_pages);
309
310 return page_address(page);
311}
312
313/**
314 * trace_seq_printf - sequence printing of trace information
315 * @s: trace sequence descriptor
316 * @fmt: printf format string
317 *
318 * The tracer may use either sequence operations or its own
319 * copy to user routines. To simplify formating of a trace
320 * trace_seq_printf is used to store strings into a special
321 * buffer (@s). Then the output may be either used by
322 * the sequencer or pulled into another buffer.
323 */
324int
325trace_seq_printf(struct trace_seq *s, const char *fmt, ...)
326{
327 int len = (PAGE_SIZE - 1) - s->len;
328 va_list ap;
329 int ret;
330
331 if (!len)
332 return 0;
333
334 va_start(ap, fmt);
335 ret = vsnprintf(s->buffer + s->len, len, fmt, ap);
336 va_end(ap);
337
338 /* If we can't write it all, don't bother writing anything */
339 if (ret >= len)
340 return 0;
341
342 s->len += ret;
343
344 return len;
345}
346
347/**
348 * trace_seq_puts - trace sequence printing of simple string
349 * @s: trace sequence descriptor
350 * @str: simple string to record
351 *
352 * The tracer may use either the sequence operations or its own
353 * copy to user routines. This function records a simple string
354 * into a special buffer (@s) for later retrieval by a sequencer
355 * or other mechanism.
356 */
357static int
358trace_seq_puts(struct trace_seq *s, const char *str)
359{
360 int len = strlen(str);
361
362 if (len > ((PAGE_SIZE - 1) - s->len))
363 return 0;
364
365 memcpy(s->buffer + s->len, str, len);
366 s->len += len;
367
368 return len;
369}
370
371static int
372trace_seq_putc(struct trace_seq *s, unsigned char c)
373{
374 if (s->len >= (PAGE_SIZE - 1))
375 return 0;
376
377 s->buffer[s->len++] = c;
378
379 return 1;
380}
381
382static int
383trace_seq_putmem(struct trace_seq *s, void *mem, size_t len)
384{
385 if (len > ((PAGE_SIZE - 1) - s->len))
386 return 0;
387
388 memcpy(s->buffer + s->len, mem, len);
389 s->len += len;
390
391 return len;
392}
393
394#define HEX_CHARS 17
395static const char hex2asc[] = "0123456789abcdef";
396
397static int
398trace_seq_putmem_hex(struct trace_seq *s, void *mem, size_t len)
399{
400 unsigned char hex[HEX_CHARS];
401 unsigned char *data = mem;
402 unsigned char byte;
403 int i, j;
404
405 BUG_ON(len >= HEX_CHARS);
406
407#ifdef __BIG_ENDIAN
408 for (i = 0, j = 0; i < len; i++) {
409#else
410 for (i = len-1, j = 0; i >= 0; i--) {
411#endif
412 byte = data[i];
413
414 hex[j++] = hex2asc[byte & 0x0f];
415 hex[j++] = hex2asc[byte >> 4];
416 }
417 hex[j++] = ' ';
418
419 return trace_seq_putmem(s, hex, j);
420}
421
422static void
423trace_seq_reset(struct trace_seq *s)
424{
425 s->len = 0;
426 s->readpos = 0;
427}
428
429ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
430{
431 int len;
432 int ret;
433
434 if (s->len <= s->readpos)
435 return -EBUSY;
436
437 len = s->len - s->readpos;
438 if (cnt > len)
439 cnt = len;
440 ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
441 if (ret)
442 return -EFAULT;
443
444 s->readpos += len;
445 return cnt;
446}
447
448static void
449trace_print_seq(struct seq_file *m, struct trace_seq *s)
450{
451 int len = s->len >= PAGE_SIZE ? PAGE_SIZE - 1 : s->len;
452
453 s->buffer[len] = 0;
454 seq_puts(m, s->buffer);
455
456 trace_seq_reset(s);
457}
458
459/*
460 * flip the trace buffers between two trace descriptors.
461 * This usually is the buffers between the global_trace and
462 * the max_tr to record a snapshot of a current trace.
463 *
464 * The ftrace_max_lock must be held.
465 */
466static void
467flip_trace(struct trace_array_cpu *tr1, struct trace_array_cpu *tr2)
468{
469 struct list_head flip_pages;
470
471 INIT_LIST_HEAD(&flip_pages);
472
473 memcpy(&tr1->trace_head_idx, &tr2->trace_head_idx,
474 sizeof(struct trace_array_cpu) -
475 offsetof(struct trace_array_cpu, trace_head_idx));
476
477 check_pages(tr1);
478 check_pages(tr2);
479 list_splice_init(&tr1->trace_pages, &flip_pages);
480 list_splice_init(&tr2->trace_pages, &tr1->trace_pages);
481 list_splice_init(&flip_pages, &tr2->trace_pages);
482 BUG_ON(!list_empty(&flip_pages));
483 check_pages(tr1);
484 check_pages(tr2);
485}
486
487/**
488 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
489 * @tr: tracer
490 * @tsk: the task with the latency
491 * @cpu: The cpu that initiated the trace.
492 *
493 * Flip the buffers between the @tr and the max_tr and record information
494 * about which task was the cause of this latency.
495 */
496void
497update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
498{
499 struct trace_array_cpu *data;
500 int i;
501
502 WARN_ON_ONCE(!irqs_disabled());
503 __raw_spin_lock(&ftrace_max_lock);
504 /* clear out all the previous traces */
505 for_each_tracing_cpu(i) {
506 data = tr->data[i];
507 flip_trace(max_tr.data[i], data);
508 tracing_reset(data);
509 }
510
511 __update_max_tr(tr, tsk, cpu);
512 __raw_spin_unlock(&ftrace_max_lock);
513}
514
515/**
516 * update_max_tr_single - only copy one trace over, and reset the rest
517 * @tr - tracer
518 * @tsk - task with the latency
519 * @cpu - the cpu of the buffer to copy.
520 *
521 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
522 */
523void
524update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
525{
526 struct trace_array_cpu *data = tr->data[cpu];
527 int i;
528
529 WARN_ON_ONCE(!irqs_disabled());
530 __raw_spin_lock(&ftrace_max_lock);
531 for_each_tracing_cpu(i)
532 tracing_reset(max_tr.data[i]);
533
534 flip_trace(max_tr.data[cpu], data);
535 tracing_reset(data);
536
537 __update_max_tr(tr, tsk, cpu);
538 __raw_spin_unlock(&ftrace_max_lock);
539}
540
541/**
542 * register_tracer - register a tracer with the ftrace system.
543 * @type - the plugin for the tracer
544 *
545 * Register a new plugin tracer.
546 */
547int register_tracer(struct tracer *type)
548{
549 struct tracer *t;
550 int len;
551 int ret = 0;
552
553 if (!type->name) {
554 pr_info("Tracer must have a name\n");
555 return -1;
556 }
557
558 mutex_lock(&trace_types_lock);
559 for (t = trace_types; t; t = t->next) {
560 if (strcmp(type->name, t->name) == 0) {
561 /* already found */
562 pr_info("Trace %s already registered\n",
563 type->name);
564 ret = -1;
565 goto out;
566 }
567 }
568
569#ifdef CONFIG_FTRACE_STARTUP_TEST
570 if (type->selftest) {
571 struct tracer *saved_tracer = current_trace;
572 struct trace_array_cpu *data;
573 struct trace_array *tr = &global_trace;
574 int saved_ctrl = tr->ctrl;
575 int i;
576 /*
577 * Run a selftest on this tracer.
578 * Here we reset the trace buffer, and set the current
579 * tracer to be this tracer. The tracer can then run some
580 * internal tracing to verify that everything is in order.
581 * If we fail, we do not register this tracer.
582 */
583 for_each_tracing_cpu(i) {
584 data = tr->data[i];
585 if (!head_page(data))
586 continue;
587 tracing_reset(data);
588 }
589 current_trace = type;
590 tr->ctrl = 0;
591 /* the test is responsible for initializing and enabling */
592 pr_info("Testing tracer %s: ", type->name);
593 ret = type->selftest(type, tr);
594 /* the test is responsible for resetting too */
595 current_trace = saved_tracer;
596 tr->ctrl = saved_ctrl;
597 if (ret) {
598 printk(KERN_CONT "FAILED!\n");
599 goto out;
600 }
601 /* Only reset on passing, to avoid touching corrupted buffers */
602 for_each_tracing_cpu(i) {
603 data = tr->data[i];
604 if (!head_page(data))
605 continue;
606 tracing_reset(data);
607 }
608 printk(KERN_CONT "PASSED\n");
609 }
610#endif
611
612 type->next = trace_types;
613 trace_types = type;
614 len = strlen(type->name);
615 if (len > max_tracer_type_len)
616 max_tracer_type_len = len;
617
618 out:
619 mutex_unlock(&trace_types_lock);
620
621 return ret;
622}
623
624void unregister_tracer(struct tracer *type)
625{
626 struct tracer **t;
627 int len;
628
629 mutex_lock(&trace_types_lock);
630 for (t = &trace_types; *t; t = &(*t)->next) {
631 if (*t == type)
632 goto found;
633 }
634 pr_info("Trace %s not registered\n", type->name);
635 goto out;
636
637 found:
638 *t = (*t)->next;
639 if (strlen(type->name) != max_tracer_type_len)
640 goto out;
641
642 max_tracer_type_len = 0;
643 for (t = &trace_types; *t; t = &(*t)->next) {
644 len = strlen((*t)->name);
645 if (len > max_tracer_type_len)
646 max_tracer_type_len = len;
647 }
648 out:
649 mutex_unlock(&trace_types_lock);
650}
651
652void tracing_reset(struct trace_array_cpu *data)
653{
654 data->trace_idx = 0;
655 data->overrun = 0;
656 data->trace_head = data->trace_tail = head_page(data);
657 data->trace_head_idx = 0;
658 data->trace_tail_idx = 0;
659}
660
661#define SAVED_CMDLINES 128
662static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
663static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
664static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
665static int cmdline_idx;
666static DEFINE_SPINLOCK(trace_cmdline_lock);
667
668/* temporary disable recording */
669atomic_t trace_record_cmdline_disabled __read_mostly;
670
671static void trace_init_cmdlines(void)
672{
673 memset(&map_pid_to_cmdline, -1, sizeof(map_pid_to_cmdline));
674 memset(&map_cmdline_to_pid, -1, sizeof(map_cmdline_to_pid));
675 cmdline_idx = 0;
676}
677
678void trace_stop_cmdline_recording(void);
679
680static void trace_save_cmdline(struct task_struct *tsk)
681{
682 unsigned map;
683 unsigned idx;
684
685 if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
686 return;
687
688 /*
689 * It's not the end of the world if we don't get
690 * the lock, but we also don't want to spin
691 * nor do we want to disable interrupts,
692 * so if we miss here, then better luck next time.
693 */
694 if (!spin_trylock(&trace_cmdline_lock))
695 return;
696
697 idx = map_pid_to_cmdline[tsk->pid];
698 if (idx >= SAVED_CMDLINES) {
699 idx = (cmdline_idx + 1) % SAVED_CMDLINES;
700
701 map = map_cmdline_to_pid[idx];
702 if (map <= PID_MAX_DEFAULT)
703 map_pid_to_cmdline[map] = (unsigned)-1;
704
705 map_pid_to_cmdline[tsk->pid] = idx;
706
707 cmdline_idx = idx;
708 }
709
710 memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
711
712 spin_unlock(&trace_cmdline_lock);
713}
714
715static char *trace_find_cmdline(int pid)
716{
717 char *cmdline = "<...>";
718 unsigned map;
719
720 if (!pid)
721 return "<idle>";
722
723 if (pid > PID_MAX_DEFAULT)
724 goto out;
725
726 map = map_pid_to_cmdline[pid];
727 if (map >= SAVED_CMDLINES)
728 goto out;
729
730 cmdline = saved_cmdlines[map];
731
732 out:
733 return cmdline;
734}
735
736void tracing_record_cmdline(struct task_struct *tsk)
737{
738 if (atomic_read(&trace_record_cmdline_disabled))
739 return;
740
741 trace_save_cmdline(tsk);
742}
743
744static inline struct list_head *
745trace_next_list(struct trace_array_cpu *data, struct list_head *next)
746{
747 /*
748 * Roundrobin - but skip the head (which is not a real page):
749 */
750 next = next->next;
751 if (unlikely(next == &data->trace_pages))
752 next = next->next;
753 BUG_ON(next == &data->trace_pages);
754
755 return next;
756}
757
758static inline void *
759trace_next_page(struct trace_array_cpu *data, void *addr)
760{
761 struct list_head *next;
762 struct page *page;
763
764 page = virt_to_page(addr);
765
766 next = trace_next_list(data, &page->lru);
767 page = list_entry(next, struct page, lru);
768
769 return page_address(page);
770}
771
772static inline struct trace_entry *
773tracing_get_trace_entry(struct trace_array *tr, struct trace_array_cpu *data)
774{
775 unsigned long idx, idx_next;
776 struct trace_entry *entry;
777
778 data->trace_idx++;
779 idx = data->trace_head_idx;
780 idx_next = idx + 1;
781
782 BUG_ON(idx * TRACE_ENTRY_SIZE >= PAGE_SIZE);
783
784 entry = data->trace_head + idx * TRACE_ENTRY_SIZE;
785
786 if (unlikely(idx_next >= ENTRIES_PER_PAGE)) {
787 data->trace_head = trace_next_page(data, data->trace_head);
788 idx_next = 0;
789 }
790
791 if (data->trace_head == data->trace_tail &&
792 idx_next == data->trace_tail_idx) {
793 /* overrun */
794 data->overrun++;
795 data->trace_tail_idx++;
796 if (data->trace_tail_idx >= ENTRIES_PER_PAGE) {
797 data->trace_tail =
798 trace_next_page(data, data->trace_tail);
799 data->trace_tail_idx = 0;
800 }
801 }
802
803 data->trace_head_idx = idx_next;
804
805 return entry;
806}
807
808static inline void
809tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags)
810{
811 struct task_struct *tsk = current;
812 unsigned long pc;
813
814 pc = preempt_count();
815
816 entry->preempt_count = pc & 0xff;
817 entry->pid = (tsk) ? tsk->pid : 0;
818 entry->t = ftrace_now(raw_smp_processor_id());
819 entry->flags = (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
820 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
821 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
822 (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
823}
824
825void
826trace_function(struct trace_array *tr, struct trace_array_cpu *data,
827 unsigned long ip, unsigned long parent_ip, unsigned long flags)
828{
829 struct trace_entry *entry;
830 unsigned long irq_flags;
831
832 raw_local_irq_save(irq_flags);
833 __raw_spin_lock(&data->lock);
834 entry = tracing_get_trace_entry(tr, data);
835 tracing_generic_entry_update(entry, flags);
836 entry->type = TRACE_FN;
837 entry->fn.ip = ip;
838 entry->fn.parent_ip = parent_ip;
839 __raw_spin_unlock(&data->lock);
840 raw_local_irq_restore(irq_flags);
841}
842
843void
844ftrace(struct trace_array *tr, struct trace_array_cpu *data,
845 unsigned long ip, unsigned long parent_ip, unsigned long flags)
846{
847 if (likely(!atomic_read(&data->disabled)))
848 trace_function(tr, data, ip, parent_ip, flags);
849}
850
851void __trace_stack(struct trace_array *tr,
852 struct trace_array_cpu *data,
853 unsigned long flags,
854 int skip)
855{
856 struct trace_entry *entry;
857 struct stack_trace trace;
858
859 if (!(trace_flags & TRACE_ITER_STACKTRACE))
860 return;
861
862 entry = tracing_get_trace_entry(tr, data);
863 tracing_generic_entry_update(entry, flags);
864 entry->type = TRACE_STACK;
865
866 memset(&entry->stack, 0, sizeof(entry->stack));
867
868 trace.nr_entries = 0;
869 trace.max_entries = FTRACE_STACK_ENTRIES;
870 trace.skip = skip;
871 trace.entries = entry->stack.caller;
872
873 save_stack_trace(&trace);
874}
875
876void
877__trace_special(void *__tr, void *__data,
878 unsigned long arg1, unsigned long arg2, unsigned long arg3)
879{
880 struct trace_array_cpu *data = __data;
881 struct trace_array *tr = __tr;
882 struct trace_entry *entry;
883 unsigned long irq_flags;
884
885 raw_local_irq_save(irq_flags);
886 __raw_spin_lock(&data->lock);
887 entry = tracing_get_trace_entry(tr, data);
888 tracing_generic_entry_update(entry, 0);
889 entry->type = TRACE_SPECIAL;
890 entry->special.arg1 = arg1;
891 entry->special.arg2 = arg2;
892 entry->special.arg3 = arg3;
893 __trace_stack(tr, data, irq_flags, 4);
894 __raw_spin_unlock(&data->lock);
895 raw_local_irq_restore(irq_flags);
896
897 trace_wake_up();
898}
899
900void
901tracing_sched_switch_trace(struct trace_array *tr,
902 struct trace_array_cpu *data,
903 struct task_struct *prev,
904 struct task_struct *next,
905 unsigned long flags)
906{
907 struct trace_entry *entry;
908 unsigned long irq_flags;
909
910 raw_local_irq_save(irq_flags);
911 __raw_spin_lock(&data->lock);
912 entry = tracing_get_trace_entry(tr, data);
913 tracing_generic_entry_update(entry, flags);
914 entry->type = TRACE_CTX;
915 entry->ctx.prev_pid = prev->pid;
916 entry->ctx.prev_prio = prev->prio;
917 entry->ctx.prev_state = prev->state;
918 entry->ctx.next_pid = next->pid;
919 entry->ctx.next_prio = next->prio;
920 entry->ctx.next_state = next->state;
921 __trace_stack(tr, data, flags, 5);
922 __raw_spin_unlock(&data->lock);
923 raw_local_irq_restore(irq_flags);
924}
925
926void
927tracing_sched_wakeup_trace(struct trace_array *tr,
928 struct trace_array_cpu *data,
929 struct task_struct *wakee,
930 struct task_struct *curr,
931 unsigned long flags)
932{
933 struct trace_entry *entry;
934 unsigned long irq_flags;
935
936 raw_local_irq_save(irq_flags);
937 __raw_spin_lock(&data->lock);
938 entry = tracing_get_trace_entry(tr, data);
939 tracing_generic_entry_update(entry, flags);
940 entry->type = TRACE_WAKE;
941 entry->ctx.prev_pid = curr->pid;
942 entry->ctx.prev_prio = curr->prio;
943 entry->ctx.prev_state = curr->state;
944 entry->ctx.next_pid = wakee->pid;
945 entry->ctx.next_prio = wakee->prio;
946 entry->ctx.next_state = wakee->state;
947 __trace_stack(tr, data, flags, 6);
948 __raw_spin_unlock(&data->lock);
949 raw_local_irq_restore(irq_flags);
950
951 trace_wake_up();
952}
953
954void
955ftrace_special(unsigned long arg1, unsigned long arg2, unsigned long arg3)
956{
957 struct trace_array *tr = &global_trace;
958 struct trace_array_cpu *data;
959 unsigned long flags;
960 long disabled;
961 int cpu;
962
963 if (tracing_disabled || current_trace == &no_tracer || !tr->ctrl)
964 return;
965
966 local_irq_save(flags);
967 cpu = raw_smp_processor_id();
968 data = tr->data[cpu];
969 disabled = atomic_inc_return(&data->disabled);
970
971 if (likely(disabled == 1))
972 __trace_special(tr, data, arg1, arg2, arg3);
973
974 atomic_dec(&data->disabled);
975 local_irq_restore(flags);
976}
977
978#ifdef CONFIG_FTRACE
979static void
980function_trace_call(unsigned long ip, unsigned long parent_ip)
981{
982 struct trace_array *tr = &global_trace;
983 struct trace_array_cpu *data;
984 unsigned long flags;
985 long disabled;
986 int cpu;
987
988 if (unlikely(!tracer_enabled))
989 return;
990
991 if (skip_trace(ip))
992 return;
993
994 local_irq_save(flags);
995 cpu = raw_smp_processor_id();
996 data = tr->data[cpu];
997 disabled = atomic_inc_return(&data->disabled);
998
999 if (likely(disabled == 1))
1000 trace_function(tr, data, ip, parent_ip, flags);
1001
1002 atomic_dec(&data->disabled);
1003 local_irq_restore(flags);
1004}
1005
1006static struct ftrace_ops trace_ops __read_mostly =
1007{
1008 .func = function_trace_call,
1009};
1010
1011void tracing_start_function_trace(void)
1012{
1013 register_ftrace_function(&trace_ops);
1014}
1015
1016void tracing_stop_function_trace(void)
1017{
1018 unregister_ftrace_function(&trace_ops);
1019}
1020#endif
1021
1022enum trace_file_type {
1023 TRACE_FILE_LAT_FMT = 1,
1024};
1025
1026static struct trace_entry *
1027trace_entry_idx(struct trace_array *tr, struct trace_array_cpu *data,
1028 struct trace_iterator *iter, int cpu)
1029{
1030 struct page *page;
1031 struct trace_entry *array;
1032
1033 if (iter->next_idx[cpu] >= tr->entries ||
1034 iter->next_idx[cpu] >= data->trace_idx ||
1035 (data->trace_head == data->trace_tail &&
1036 data->trace_head_idx == data->trace_tail_idx))
1037 return NULL;
1038
1039 if (!iter->next_page[cpu]) {
1040 /* Initialize the iterator for this cpu trace buffer */
1041 WARN_ON(!data->trace_tail);
1042 page = virt_to_page(data->trace_tail);
1043 iter->next_page[cpu] = &page->lru;
1044 iter->next_page_idx[cpu] = data->trace_tail_idx;
1045 }
1046
1047 page = list_entry(iter->next_page[cpu], struct page, lru);
1048 BUG_ON(&data->trace_pages == &page->lru);
1049
1050 array = page_address(page);
1051
1052 WARN_ON(iter->next_page_idx[cpu] >= ENTRIES_PER_PAGE);
1053 return &array[iter->next_page_idx[cpu]];
1054}
1055
1056static struct trace_entry *
1057find_next_entry(struct trace_iterator *iter, int *ent_cpu)
1058{
1059 struct trace_array *tr = iter->tr;
1060 struct trace_entry *ent, *next = NULL;
1061 int next_cpu = -1;
1062 int cpu;
1063
1064 for_each_tracing_cpu(cpu) {
1065 if (!head_page(tr->data[cpu]))
1066 continue;
1067 ent = trace_entry_idx(tr, tr->data[cpu], iter, cpu);
1068 /*
1069 * Pick the entry with the smallest timestamp:
1070 */
1071 if (ent && (!next || ent->t < next->t)) {
1072 next = ent;
1073 next_cpu = cpu;
1074 }
1075 }
1076
1077 if (ent_cpu)
1078 *ent_cpu = next_cpu;
1079
1080 return next;
1081}
1082
1083static void trace_iterator_increment(struct trace_iterator *iter)
1084{
1085 iter->idx++;
1086 iter->next_idx[iter->cpu]++;
1087 iter->next_page_idx[iter->cpu]++;
1088
1089 if (iter->next_page_idx[iter->cpu] >= ENTRIES_PER_PAGE) {
1090 struct trace_array_cpu *data = iter->tr->data[iter->cpu];
1091
1092 iter->next_page_idx[iter->cpu] = 0;
1093 iter->next_page[iter->cpu] =
1094 trace_next_list(data, iter->next_page[iter->cpu]);
1095 }
1096}
1097
1098static void trace_consume(struct trace_iterator *iter)
1099{
1100 struct trace_array_cpu *data = iter->tr->data[iter->cpu];
1101
1102 data->trace_tail_idx++;
1103 if (data->trace_tail_idx >= ENTRIES_PER_PAGE) {
1104 data->trace_tail = trace_next_page(data, data->trace_tail);
1105 data->trace_tail_idx = 0;
1106 }
1107
1108 /* Check if we empty it, then reset the index */
1109 if (data->trace_head == data->trace_tail &&
1110 data->trace_head_idx == data->trace_tail_idx)
1111 data->trace_idx = 0;
1112}
1113
1114static void *find_next_entry_inc(struct trace_iterator *iter)
1115{
1116 struct trace_entry *next;
1117 int next_cpu = -1;
1118
1119 next = find_next_entry(iter, &next_cpu);
1120
1121 iter->prev_ent = iter->ent;
1122 iter->prev_cpu = iter->cpu;
1123
1124 iter->ent = next;
1125 iter->cpu = next_cpu;
1126
1127 if (next)
1128 trace_iterator_increment(iter);
1129
1130 return next ? iter : NULL;
1131}
1132
1133static void *s_next(struct seq_file *m, void *v, loff_t *pos)
1134{
1135 struct trace_iterator *iter = m->private;
1136 void *last_ent = iter->ent;
1137 int i = (int)*pos;
1138 void *ent;
1139
1140 (*pos)++;
1141
1142 /* can't go backwards */
1143 if (iter->idx > i)
1144 return NULL;
1145
1146 if (iter->idx < 0)
1147 ent = find_next_entry_inc(iter);
1148 else
1149 ent = iter;
1150
1151 while (ent && iter->idx < i)
1152 ent = find_next_entry_inc(iter);
1153
1154 iter->pos = *pos;
1155
1156 if (last_ent && !ent)
1157 seq_puts(m, "\n\nvim:ft=help\n");
1158
1159 return ent;
1160}
1161
1162static void *s_start(struct seq_file *m, loff_t *pos)
1163{
1164 struct trace_iterator *iter = m->private;
1165 void *p = NULL;
1166 loff_t l = 0;
1167 int i;
1168
1169 mutex_lock(&trace_types_lock);
1170
1171 if (!current_trace || current_trace != iter->trace) {
1172 mutex_unlock(&trace_types_lock);
1173 return NULL;
1174 }
1175
1176 atomic_inc(&trace_record_cmdline_disabled);
1177
1178 /* let the tracer grab locks here if needed */
1179 if (current_trace->start)
1180 current_trace->start(iter);
1181
1182 if (*pos != iter->pos) {
1183 iter->ent = NULL;
1184 iter->cpu = 0;
1185 iter->idx = -1;
1186 iter->prev_ent = NULL;
1187 iter->prev_cpu = -1;
1188
1189 for_each_tracing_cpu(i) {
1190 iter->next_idx[i] = 0;
1191 iter->next_page[i] = NULL;
1192 }
1193
1194 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
1195 ;
1196
1197 } else {
1198 l = *pos - 1;
1199 p = s_next(m, p, &l);
1200 }
1201
1202 return p;
1203}
1204
1205static void s_stop(struct seq_file *m, void *p)
1206{
1207 struct trace_iterator *iter = m->private;
1208
1209 atomic_dec(&trace_record_cmdline_disabled);
1210
1211 /* let the tracer release locks here if needed */
1212 if (current_trace && current_trace == iter->trace && iter->trace->stop)
1213 iter->trace->stop(iter);
1214
1215 mutex_unlock(&trace_types_lock);
1216}
1217
1218#define KRETPROBE_MSG "[unknown/kretprobe'd]"
1219
1220#ifdef CONFIG_KRETPROBES
1221static inline int kretprobed(unsigned long addr)
1222{
1223 return addr == (unsigned long)kretprobe_trampoline;
1224}
1225#else
1226static inline int kretprobed(unsigned long addr)
1227{
1228 return 0;
1229}
1230#endif /* CONFIG_KRETPROBES */
1231
1232static int
1233seq_print_sym_short(struct trace_seq *s, const char *fmt, unsigned long address)
1234{
1235#ifdef CONFIG_KALLSYMS
1236 char str[KSYM_SYMBOL_LEN];
1237
1238 kallsyms_lookup(address, NULL, NULL, NULL, str);
1239
1240 return trace_seq_printf(s, fmt, str);
1241#endif
1242 return 1;
1243}
1244
1245static int
1246seq_print_sym_offset(struct trace_seq *s, const char *fmt,
1247 unsigned long address)
1248{
1249#ifdef CONFIG_KALLSYMS
1250 char str[KSYM_SYMBOL_LEN];
1251
1252 sprint_symbol(str, address);
1253 return trace_seq_printf(s, fmt, str);
1254#endif
1255 return 1;
1256}
1257
1258#ifndef CONFIG_64BIT
1259# define IP_FMT "%08lx"
1260#else
1261# define IP_FMT "%016lx"
1262#endif
1263
1264static int
1265seq_print_ip_sym(struct trace_seq *s, unsigned long ip, unsigned long sym_flags)
1266{
1267 int ret;
1268
1269 if (!ip)
1270 return trace_seq_printf(s, "0");
1271
1272 if (sym_flags & TRACE_ITER_SYM_OFFSET)
1273 ret = seq_print_sym_offset(s, "%s", ip);
1274 else
1275 ret = seq_print_sym_short(s, "%s", ip);
1276
1277 if (!ret)
1278 return 0;
1279
1280 if (sym_flags & TRACE_ITER_SYM_ADDR)
1281 ret = trace_seq_printf(s, " <" IP_FMT ">", ip);
1282 return ret;
1283}
1284
1285static void print_lat_help_header(struct seq_file *m)
1286{
1287 seq_puts(m, "# _------=> CPU# \n");
1288 seq_puts(m, "# / _-----=> irqs-off \n");
1289 seq_puts(m, "# | / _----=> need-resched \n");
1290 seq_puts(m, "# || / _---=> hardirq/softirq \n");
1291 seq_puts(m, "# ||| / _--=> preempt-depth \n");
1292 seq_puts(m, "# |||| / \n");
1293 seq_puts(m, "# ||||| delay \n");
1294 seq_puts(m, "# cmd pid ||||| time | caller \n");
1295 seq_puts(m, "# \\ / ||||| \\ | / \n");
1296}
1297
1298static void print_func_help_header(struct seq_file *m)
1299{
1300 seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n");
1301 seq_puts(m, "# | | | | |\n");
1302}
1303
1304
1305static void
1306print_trace_header(struct seq_file *m, struct trace_iterator *iter)
1307{
1308 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1309 struct trace_array *tr = iter->tr;
1310 struct trace_array_cpu *data = tr->data[tr->cpu];
1311 struct tracer *type = current_trace;
1312 unsigned long total = 0;
1313 unsigned long entries = 0;
1314 int cpu;
1315 const char *name = "preemption";
1316
1317 if (type)
1318 name = type->name;
1319
1320 for_each_tracing_cpu(cpu) {
1321 if (head_page(tr->data[cpu])) {
1322 total += tr->data[cpu]->trace_idx;
1323 if (tr->data[cpu]->trace_idx > tr->entries)
1324 entries += tr->entries;
1325 else
1326 entries += tr->data[cpu]->trace_idx;
1327 }
1328 }
1329
1330 seq_printf(m, "%s latency trace v1.1.5 on %s\n",
1331 name, UTS_RELEASE);
1332 seq_puts(m, "-----------------------------------"
1333 "---------------------------------\n");
1334 seq_printf(m, " latency: %lu us, #%lu/%lu, CPU#%d |"
1335 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
1336 nsecs_to_usecs(data->saved_latency),
1337 entries,
1338 total,
1339 tr->cpu,
1340#if defined(CONFIG_PREEMPT_NONE)
1341 "server",
1342#elif defined(CONFIG_PREEMPT_VOLUNTARY)
1343 "desktop",
1344#elif defined(CONFIG_PREEMPT_DESKTOP)
1345 "preempt",
1346#else
1347 "unknown",
1348#endif
1349 /* These are reserved for later use */
1350 0, 0, 0, 0);
1351#ifdef CONFIG_SMP
1352 seq_printf(m, " #P:%d)\n", num_online_cpus());
1353#else
1354 seq_puts(m, ")\n");
1355#endif
1356 seq_puts(m, " -----------------\n");
1357 seq_printf(m, " | task: %.16s-%d "
1358 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
1359 data->comm, data->pid, data->uid, data->nice,
1360 data->policy, data->rt_priority);
1361 seq_puts(m, " -----------------\n");
1362
1363 if (data->critical_start) {
1364 seq_puts(m, " => started at: ");
1365 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
1366 trace_print_seq(m, &iter->seq);
1367 seq_puts(m, "\n => ended at: ");
1368 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
1369 trace_print_seq(m, &iter->seq);
1370 seq_puts(m, "\n");
1371 }
1372
1373 seq_puts(m, "\n");
1374}
1375
1376static void
1377lat_print_generic(struct trace_seq *s, struct trace_entry *entry, int cpu)
1378{
1379 int hardirq, softirq;
1380 char *comm;
1381
1382 comm = trace_find_cmdline(entry->pid);
1383
1384 trace_seq_printf(s, "%8.8s-%-5d ", comm, entry->pid);
1385 trace_seq_printf(s, "%d", cpu);
1386 trace_seq_printf(s, "%c%c",
1387 (entry->flags & TRACE_FLAG_IRQS_OFF) ? 'd' : '.',
1388 ((entry->flags & TRACE_FLAG_NEED_RESCHED) ? 'N' : '.'));
1389
1390 hardirq = entry->flags & TRACE_FLAG_HARDIRQ;
1391 softirq = entry->flags & TRACE_FLAG_SOFTIRQ;
1392 if (hardirq && softirq) {
1393 trace_seq_putc(s, 'H');
1394 } else {
1395 if (hardirq) {
1396 trace_seq_putc(s, 'h');
1397 } else {
1398 if (softirq)
1399 trace_seq_putc(s, 's');
1400 else
1401 trace_seq_putc(s, '.');
1402 }
1403 }
1404
1405 if (entry->preempt_count)
1406 trace_seq_printf(s, "%x", entry->preempt_count);
1407 else
1408 trace_seq_puts(s, ".");
1409}
1410
1411unsigned long preempt_mark_thresh = 100;
1412
1413static void
1414lat_print_timestamp(struct trace_seq *s, unsigned long long abs_usecs,
1415 unsigned long rel_usecs)
1416{
1417 trace_seq_printf(s, " %4lldus", abs_usecs);
1418 if (rel_usecs > preempt_mark_thresh)
1419 trace_seq_puts(s, "!: ");
1420 else if (rel_usecs > 1)
1421 trace_seq_puts(s, "+: ");
1422 else
1423 trace_seq_puts(s, " : ");
1424}
1425
1426static const char state_to_char[] = TASK_STATE_TO_CHAR_STR;
1427
1428static int
1429print_lat_fmt(struct trace_iterator *iter, unsigned int trace_idx, int cpu)
1430{
1431 struct trace_seq *s = &iter->seq;
1432 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1433 struct trace_entry *next_entry = find_next_entry(iter, NULL);
1434 unsigned long verbose = (trace_flags & TRACE_ITER_VERBOSE);
1435 struct trace_entry *entry = iter->ent;
1436 unsigned long abs_usecs;
1437 unsigned long rel_usecs;
1438 char *comm;
1439 int S, T;
1440 int i;
1441 unsigned state;
1442
1443 if (!next_entry)
1444 next_entry = entry;
1445 rel_usecs = ns2usecs(next_entry->t - entry->t);
1446 abs_usecs = ns2usecs(entry->t - iter->tr->time_start);
1447
1448 if (verbose) {
1449 comm = trace_find_cmdline(entry->pid);
1450 trace_seq_printf(s, "%16s %5d %d %d %08x %08x [%08lx]"
1451 " %ld.%03ldms (+%ld.%03ldms): ",
1452 comm,
1453 entry->pid, cpu, entry->flags,
1454 entry->preempt_count, trace_idx,
1455 ns2usecs(entry->t),
1456 abs_usecs/1000,
1457 abs_usecs % 1000, rel_usecs/1000,
1458 rel_usecs % 1000);
1459 } else {
1460 lat_print_generic(s, entry, cpu);
1461 lat_print_timestamp(s, abs_usecs, rel_usecs);
1462 }
1463 switch (entry->type) {
1464 case TRACE_FN:
1465 seq_print_ip_sym(s, entry->fn.ip, sym_flags);
1466 trace_seq_puts(s, " (");
1467 if (kretprobed(entry->fn.parent_ip))
1468 trace_seq_puts(s, KRETPROBE_MSG);
1469 else
1470 seq_print_ip_sym(s, entry->fn.parent_ip, sym_flags);
1471 trace_seq_puts(s, ")\n");
1472 break;
1473 case TRACE_CTX:
1474 case TRACE_WAKE:
1475 T = entry->ctx.next_state < sizeof(state_to_char) ?
1476 state_to_char[entry->ctx.next_state] : 'X';
1477
1478 state = entry->ctx.prev_state ? __ffs(entry->ctx.prev_state) + 1 : 0;
1479 S = state < sizeof(state_to_char) - 1 ? state_to_char[state] : 'X';
1480 comm = trace_find_cmdline(entry->ctx.next_pid);
1481 trace_seq_printf(s, " %5d:%3d:%c %s %5d:%3d:%c %s\n",
1482 entry->ctx.prev_pid,
1483 entry->ctx.prev_prio,
1484 S, entry->type == TRACE_CTX ? "==>" : " +",
1485 entry->ctx.next_pid,
1486 entry->ctx.next_prio,
1487 T, comm);
1488 break;
1489 case TRACE_SPECIAL:
1490 trace_seq_printf(s, "# %ld %ld %ld\n",
1491 entry->special.arg1,
1492 entry->special.arg2,
1493 entry->special.arg3);
1494 break;
1495 case TRACE_STACK:
1496 for (i = 0; i < FTRACE_STACK_ENTRIES; i++) {
1497 if (i)
1498 trace_seq_puts(s, " <= ");
1499 seq_print_ip_sym(s, entry->stack.caller[i], sym_flags);
1500 }
1501 trace_seq_puts(s, "\n");
1502 break;
1503 default:
1504 trace_seq_printf(s, "Unknown type %d\n", entry->type);
1505 }
1506 return 1;
1507}
1508
1509static int print_trace_fmt(struct trace_iterator *iter)
1510{
1511 struct trace_seq *s = &iter->seq;
1512 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1513 struct trace_entry *entry;
1514 unsigned long usec_rem;
1515 unsigned long long t;
1516 unsigned long secs;
1517 char *comm;
1518 int ret;
1519 int S, T;
1520 int i;
1521
1522 entry = iter->ent;
1523
1524 comm = trace_find_cmdline(iter->ent->pid);
1525
1526 t = ns2usecs(entry->t);
1527 usec_rem = do_div(t, 1000000ULL);
1528 secs = (unsigned long)t;
1529
1530 ret = trace_seq_printf(s, "%16s-%-5d ", comm, entry->pid);
1531 if (!ret)
1532 return 0;
1533 ret = trace_seq_printf(s, "[%02d] ", iter->cpu);
1534 if (!ret)
1535 return 0;
1536 ret = trace_seq_printf(s, "%5lu.%06lu: ", secs, usec_rem);
1537 if (!ret)
1538 return 0;
1539
1540 switch (entry->type) {
1541 case TRACE_FN:
1542 ret = seq_print_ip_sym(s, entry->fn.ip, sym_flags);
1543 if (!ret)
1544 return 0;
1545 if ((sym_flags & TRACE_ITER_PRINT_PARENT) &&
1546 entry->fn.parent_ip) {
1547 ret = trace_seq_printf(s, " <-");
1548 if (!ret)
1549 return 0;
1550 if (kretprobed(entry->fn.parent_ip))
1551 ret = trace_seq_puts(s, KRETPROBE_MSG);
1552 else
1553 ret = seq_print_ip_sym(s, entry->fn.parent_ip,
1554 sym_flags);
1555 if (!ret)
1556 return 0;
1557 }
1558 ret = trace_seq_printf(s, "\n");
1559 if (!ret)
1560 return 0;
1561 break;
1562 case TRACE_CTX:
1563 case TRACE_WAKE:
1564 S = entry->ctx.prev_state < sizeof(state_to_char) ?
1565 state_to_char[entry->ctx.prev_state] : 'X';
1566 T = entry->ctx.next_state < sizeof(state_to_char) ?
1567 state_to_char[entry->ctx.next_state] : 'X';
1568 ret = trace_seq_printf(s, " %5d:%3d:%c %s %5d:%3d:%c\n",
1569 entry->ctx.prev_pid,
1570 entry->ctx.prev_prio,
1571 S,
1572 entry->type == TRACE_CTX ? "==>" : " +",
1573 entry->ctx.next_pid,
1574 entry->ctx.next_prio,
1575 T);
1576 if (!ret)
1577 return 0;
1578 break;
1579 case TRACE_SPECIAL:
1580 ret = trace_seq_printf(s, "# %ld %ld %ld\n",
1581 entry->special.arg1,
1582 entry->special.arg2,
1583 entry->special.arg3);
1584 if (!ret)
1585 return 0;
1586 break;
1587 case TRACE_STACK:
1588 for (i = 0; i < FTRACE_STACK_ENTRIES; i++) {
1589 if (i) {
1590 ret = trace_seq_puts(s, " <= ");
1591 if (!ret)
1592 return 0;
1593 }
1594 ret = seq_print_ip_sym(s, entry->stack.caller[i],
1595 sym_flags);
1596 if (!ret)
1597 return 0;
1598 }
1599 ret = trace_seq_puts(s, "\n");
1600 if (!ret)
1601 return 0;
1602 break;
1603 }
1604 return 1;
1605}
1606
1607static int print_raw_fmt(struct trace_iterator *iter)
1608{
1609 struct trace_seq *s = &iter->seq;
1610 struct trace_entry *entry;
1611 int ret;
1612 int S, T;
1613
1614 entry = iter->ent;
1615
1616 ret = trace_seq_printf(s, "%d %d %llu ",
1617 entry->pid, iter->cpu, entry->t);
1618 if (!ret)
1619 return 0;
1620
1621 switch (entry->type) {
1622 case TRACE_FN:
1623 ret = trace_seq_printf(s, "%x %x\n",
1624 entry->fn.ip, entry->fn.parent_ip);
1625 if (!ret)
1626 return 0;
1627 break;
1628 case TRACE_CTX:
1629 case TRACE_WAKE:
1630 S = entry->ctx.prev_state < sizeof(state_to_char) ?
1631 state_to_char[entry->ctx.prev_state] : 'X';
1632 T = entry->ctx.next_state < sizeof(state_to_char) ?
1633 state_to_char[entry->ctx.next_state] : 'X';
1634 if (entry->type == TRACE_WAKE)
1635 S = '+';
1636 ret = trace_seq_printf(s, "%d %d %c %d %d %c\n",
1637 entry->ctx.prev_pid,
1638 entry->ctx.prev_prio,
1639 S,
1640 entry->ctx.next_pid,
1641 entry->ctx.next_prio,
1642 T);
1643 if (!ret)
1644 return 0;
1645 break;
1646 case TRACE_SPECIAL:
1647 case TRACE_STACK:
1648 ret = trace_seq_printf(s, "# %ld %ld %ld\n",
1649 entry->special.arg1,
1650 entry->special.arg2,
1651 entry->special.arg3);
1652 if (!ret)
1653 return 0;
1654 break;
1655 }
1656 return 1;
1657}
1658
1659#define SEQ_PUT_FIELD_RET(s, x) \
1660do { \
1661 if (!trace_seq_putmem(s, &(x), sizeof(x))) \
1662 return 0; \
1663} while (0)
1664
1665#define SEQ_PUT_HEX_FIELD_RET(s, x) \
1666do { \
1667 if (!trace_seq_putmem_hex(s, &(x), sizeof(x))) \
1668 return 0; \
1669} while (0)
1670
1671static int print_hex_fmt(struct trace_iterator *iter)
1672{
1673 struct trace_seq *s = &iter->seq;
1674 unsigned char newline = '\n';
1675 struct trace_entry *entry;
1676 int S, T;
1677
1678 entry = iter->ent;
1679
1680 SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
1681 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
1682 SEQ_PUT_HEX_FIELD_RET(s, entry->t);
1683
1684 switch (entry->type) {
1685 case TRACE_FN:
1686 SEQ_PUT_HEX_FIELD_RET(s, entry->fn.ip);
1687 SEQ_PUT_HEX_FIELD_RET(s, entry->fn.parent_ip);
1688 break;
1689 case TRACE_CTX:
1690 case TRACE_WAKE:
1691 S = entry->ctx.prev_state < sizeof(state_to_char) ?
1692 state_to_char[entry->ctx.prev_state] : 'X';
1693 T = entry->ctx.next_state < sizeof(state_to_char) ?
1694 state_to_char[entry->ctx.next_state] : 'X';
1695 if (entry->type == TRACE_WAKE)
1696 S = '+';
1697 SEQ_PUT_HEX_FIELD_RET(s, entry->ctx.prev_pid);
1698 SEQ_PUT_HEX_FIELD_RET(s, entry->ctx.prev_prio);
1699 SEQ_PUT_HEX_FIELD_RET(s, S);
1700 SEQ_PUT_HEX_FIELD_RET(s, entry->ctx.next_pid);
1701 SEQ_PUT_HEX_FIELD_RET(s, entry->ctx.next_prio);
1702 SEQ_PUT_HEX_FIELD_RET(s, entry->fn.parent_ip);
1703 SEQ_PUT_HEX_FIELD_RET(s, T);
1704 break;
1705 case TRACE_SPECIAL:
1706 case TRACE_STACK:
1707 SEQ_PUT_HEX_FIELD_RET(s, entry->special.arg1);
1708 SEQ_PUT_HEX_FIELD_RET(s, entry->special.arg2);
1709 SEQ_PUT_HEX_FIELD_RET(s, entry->special.arg3);
1710 break;
1711 }
1712 SEQ_PUT_FIELD_RET(s, newline);
1713
1714 return 1;
1715}
1716
1717static int print_bin_fmt(struct trace_iterator *iter)
1718{
1719 struct trace_seq *s = &iter->seq;
1720 struct trace_entry *entry;
1721
1722 entry = iter->ent;
1723
1724 SEQ_PUT_FIELD_RET(s, entry->pid);
1725 SEQ_PUT_FIELD_RET(s, entry->cpu);
1726 SEQ_PUT_FIELD_RET(s, entry->t);
1727
1728 switch (entry->type) {
1729 case TRACE_FN:
1730 SEQ_PUT_FIELD_RET(s, entry->fn.ip);
1731 SEQ_PUT_FIELD_RET(s, entry->fn.parent_ip);
1732 break;
1733 case TRACE_CTX:
1734 SEQ_PUT_FIELD_RET(s, entry->ctx.prev_pid);
1735 SEQ_PUT_FIELD_RET(s, entry->ctx.prev_prio);
1736 SEQ_PUT_FIELD_RET(s, entry->ctx.prev_state);
1737 SEQ_PUT_FIELD_RET(s, entry->ctx.next_pid);
1738 SEQ_PUT_FIELD_RET(s, entry->ctx.next_prio);
1739 SEQ_PUT_FIELD_RET(s, entry->ctx.next_state);
1740 break;
1741 case TRACE_SPECIAL:
1742 case TRACE_STACK:
1743 SEQ_PUT_FIELD_RET(s, entry->special.arg1);
1744 SEQ_PUT_FIELD_RET(s, entry->special.arg2);
1745 SEQ_PUT_FIELD_RET(s, entry->special.arg3);
1746 break;
1747 }
1748 return 1;
1749}
1750
1751static int trace_empty(struct trace_iterator *iter)
1752{
1753 struct trace_array_cpu *data;
1754 int cpu;
1755
1756 for_each_tracing_cpu(cpu) {
1757 data = iter->tr->data[cpu];
1758
1759 if (head_page(data) && data->trace_idx &&
1760 (data->trace_tail != data->trace_head ||
1761 data->trace_tail_idx != data->trace_head_idx))
1762 return 0;
1763 }
1764 return 1;
1765}
1766
1767static int print_trace_line(struct trace_iterator *iter)
1768{
1769 if (iter->trace && iter->trace->print_line)
1770 return iter->trace->print_line(iter);
1771
1772 if (trace_flags & TRACE_ITER_BIN)
1773 return print_bin_fmt(iter);
1774
1775 if (trace_flags & TRACE_ITER_HEX)
1776 return print_hex_fmt(iter);
1777
1778 if (trace_flags & TRACE_ITER_RAW)
1779 return print_raw_fmt(iter);
1780
1781 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
1782 return print_lat_fmt(iter, iter->idx, iter->cpu);
1783
1784 return print_trace_fmt(iter);
1785}
1786
1787static int s_show(struct seq_file *m, void *v)
1788{
1789 struct trace_iterator *iter = v;
1790
1791 if (iter->ent == NULL) {
1792 if (iter->tr) {
1793 seq_printf(m, "# tracer: %s\n", iter->trace->name);
1794 seq_puts(m, "#\n");
1795 }
1796 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
1797 /* print nothing if the buffers are empty */
1798 if (trace_empty(iter))
1799 return 0;
1800 print_trace_header(m, iter);
1801 if (!(trace_flags & TRACE_ITER_VERBOSE))
1802 print_lat_help_header(m);
1803 } else {
1804 if (!(trace_flags & TRACE_ITER_VERBOSE))
1805 print_func_help_header(m);
1806 }
1807 } else {
1808 print_trace_line(iter);
1809 trace_print_seq(m, &iter->seq);
1810 }
1811
1812 return 0;
1813}
1814
1815static struct seq_operations tracer_seq_ops = {
1816 .start = s_start,
1817 .next = s_next,
1818 .stop = s_stop,
1819 .show = s_show,
1820};
1821
1822static struct trace_iterator *
1823__tracing_open(struct inode *inode, struct file *file, int *ret)
1824{
1825 struct trace_iterator *iter;
1826
1827 if (tracing_disabled) {
1828 *ret = -ENODEV;
1829 return NULL;
1830 }
1831
1832 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1833 if (!iter) {
1834 *ret = -ENOMEM;
1835 goto out;
1836 }
1837
1838 mutex_lock(&trace_types_lock);
1839 if (current_trace && current_trace->print_max)
1840 iter->tr = &max_tr;
1841 else
1842 iter->tr = inode->i_private;
1843 iter->trace = current_trace;
1844 iter->pos = -1;
1845
1846 /* TODO stop tracer */
1847 *ret = seq_open(file, &tracer_seq_ops);
1848 if (!*ret) {
1849 struct seq_file *m = file->private_data;
1850 m->private = iter;
1851
1852 /* stop the trace while dumping */
1853 if (iter->tr->ctrl)
1854 tracer_enabled = 0;
1855
1856 if (iter->trace && iter->trace->open)
1857 iter->trace->open(iter);
1858 } else {
1859 kfree(iter);
1860 iter = NULL;
1861 }
1862 mutex_unlock(&trace_types_lock);
1863
1864 out:
1865 return iter;
1866}
1867
1868int tracing_open_generic(struct inode *inode, struct file *filp)
1869{
1870 if (tracing_disabled)
1871 return -ENODEV;
1872
1873 filp->private_data = inode->i_private;
1874 return 0;
1875}
1876
1877int tracing_release(struct inode *inode, struct file *file)
1878{
1879 struct seq_file *m = (struct seq_file *)file->private_data;
1880 struct trace_iterator *iter = m->private;
1881
1882 mutex_lock(&trace_types_lock);
1883 if (iter->trace && iter->trace->close)
1884 iter->trace->close(iter);
1885
1886 /* reenable tracing if it was previously enabled */
1887 if (iter->tr->ctrl)
1888 tracer_enabled = 1;
1889 mutex_unlock(&trace_types_lock);
1890
1891 seq_release(inode, file);
1892 kfree(iter);
1893 return 0;
1894}
1895
1896static int tracing_open(struct inode *inode, struct file *file)
1897{
1898 int ret;
1899
1900 __tracing_open(inode, file, &ret);
1901
1902 return ret;
1903}
1904
1905static int tracing_lt_open(struct inode *inode, struct file *file)
1906{
1907 struct trace_iterator *iter;
1908 int ret;
1909
1910 iter = __tracing_open(inode, file, &ret);
1911
1912 if (!ret)
1913 iter->iter_flags |= TRACE_FILE_LAT_FMT;
1914
1915 return ret;
1916}
1917
1918
1919static void *
1920t_next(struct seq_file *m, void *v, loff_t *pos)
1921{
1922 struct tracer *t = m->private;
1923
1924 (*pos)++;
1925
1926 if (t)
1927 t = t->next;
1928
1929 m->private = t;
1930
1931 return t;
1932}
1933
1934static void *t_start(struct seq_file *m, loff_t *pos)
1935{
1936 struct tracer *t = m->private;
1937 loff_t l = 0;
1938
1939 mutex_lock(&trace_types_lock);
1940 for (; t && l < *pos; t = t_next(m, t, &l))
1941 ;
1942
1943 return t;
1944}
1945
1946static void t_stop(struct seq_file *m, void *p)
1947{
1948 mutex_unlock(&trace_types_lock);
1949}
1950
1951static int t_show(struct seq_file *m, void *v)
1952{
1953 struct tracer *t = v;
1954
1955 if (!t)
1956 return 0;
1957
1958 seq_printf(m, "%s", t->name);
1959 if (t->next)
1960 seq_putc(m, ' ');
1961 else
1962 seq_putc(m, '\n');
1963
1964 return 0;
1965}
1966
1967static struct seq_operations show_traces_seq_ops = {
1968 .start = t_start,
1969 .next = t_next,
1970 .stop = t_stop,
1971 .show = t_show,
1972};
1973
1974static int show_traces_open(struct inode *inode, struct file *file)
1975{
1976 int ret;
1977
1978 if (tracing_disabled)
1979 return -ENODEV;
1980
1981 ret = seq_open(file, &show_traces_seq_ops);
1982 if (!ret) {
1983 struct seq_file *m = file->private_data;
1984 m->private = trace_types;
1985 }
1986
1987 return ret;
1988}
1989
1990static struct file_operations tracing_fops = {
1991 .open = tracing_open,
1992 .read = seq_read,
1993 .llseek = seq_lseek,
1994 .release = tracing_release,
1995};
1996
1997static struct file_operations tracing_lt_fops = {
1998 .open = tracing_lt_open,
1999 .read = seq_read,
2000 .llseek = seq_lseek,
2001 .release = tracing_release,
2002};
2003
2004static struct file_operations show_traces_fops = {
2005 .open = show_traces_open,
2006 .read = seq_read,
2007 .release = seq_release,
2008};
2009
2010/*
2011 * Only trace on a CPU if the bitmask is set:
2012 */
2013static cpumask_t tracing_cpumask = CPU_MASK_ALL;
2014
2015/*
2016 * When tracing/tracing_cpu_mask is modified then this holds
2017 * the new bitmask we are about to install:
2018 */
2019static cpumask_t tracing_cpumask_new;
2020
2021/*
2022 * The tracer itself will not take this lock, but still we want
2023 * to provide a consistent cpumask to user-space:
2024 */
2025static DEFINE_MUTEX(tracing_cpumask_update_lock);
2026
2027/*
2028 * Temporary storage for the character representation of the
2029 * CPU bitmask (and one more byte for the newline):
2030 */
2031static char mask_str[NR_CPUS + 1];
2032
2033static ssize_t
2034tracing_cpumask_read(struct file *filp, char __user *ubuf,
2035 size_t count, loff_t *ppos)
2036{
2037 int len;
2038
2039 mutex_lock(&tracing_cpumask_update_lock);
2040
2041 len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
2042 if (count - len < 2) {
2043 count = -EINVAL;
2044 goto out_err;
2045 }
2046 len += sprintf(mask_str + len, "\n");
2047 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
2048
2049out_err:
2050 mutex_unlock(&tracing_cpumask_update_lock);
2051
2052 return count;
2053}
2054
2055static ssize_t
2056tracing_cpumask_write(struct file *filp, const char __user *ubuf,
2057 size_t count, loff_t *ppos)
2058{
2059 int err, cpu;
2060
2061 mutex_lock(&tracing_cpumask_update_lock);
2062 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
2063 if (err)
2064 goto err_unlock;
2065
2066 raw_local_irq_disable();
2067 __raw_spin_lock(&ftrace_max_lock);
2068 for_each_tracing_cpu(cpu) {
2069 /*
2070 * Increase/decrease the disabled counter if we are
2071 * about to flip a bit in the cpumask:
2072 */
2073 if (cpu_isset(cpu, tracing_cpumask) &&
2074 !cpu_isset(cpu, tracing_cpumask_new)) {
2075 atomic_inc(&global_trace.data[cpu]->disabled);
2076 }
2077 if (!cpu_isset(cpu, tracing_cpumask) &&
2078 cpu_isset(cpu, tracing_cpumask_new)) {
2079 atomic_dec(&global_trace.data[cpu]->disabled);
2080 }
2081 }
2082 __raw_spin_unlock(&ftrace_max_lock);
2083 raw_local_irq_enable();
2084
2085 tracing_cpumask = tracing_cpumask_new;
2086
2087 mutex_unlock(&tracing_cpumask_update_lock);
2088
2089 return count;
2090
2091err_unlock:
2092 mutex_unlock(&tracing_cpumask_update_lock);
2093
2094 return err;
2095}
2096
2097static struct file_operations tracing_cpumask_fops = {
2098 .open = tracing_open_generic,
2099 .read = tracing_cpumask_read,
2100 .write = tracing_cpumask_write,
2101};
2102
2103static ssize_t
2104tracing_iter_ctrl_read(struct file *filp, char __user *ubuf,
2105 size_t cnt, loff_t *ppos)
2106{
2107 char *buf;
2108 int r = 0;
2109 int len = 0;
2110 int i;
2111
2112 /* calulate max size */
2113 for (i = 0; trace_options[i]; i++) {
2114 len += strlen(trace_options[i]);
2115 len += 3; /* "no" and space */
2116 }
2117
2118 /* +2 for \n and \0 */
2119 buf = kmalloc(len + 2, GFP_KERNEL);
2120 if (!buf)
2121 return -ENOMEM;
2122
2123 for (i = 0; trace_options[i]; i++) {
2124 if (trace_flags & (1 << i))
2125 r += sprintf(buf + r, "%s ", trace_options[i]);
2126 else
2127 r += sprintf(buf + r, "no%s ", trace_options[i]);
2128 }
2129
2130 r += sprintf(buf + r, "\n");
2131 WARN_ON(r >= len + 2);
2132
2133 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2134
2135 kfree(buf);
2136
2137 return r;
2138}
2139
2140static ssize_t
2141tracing_iter_ctrl_write(struct file *filp, const char __user *ubuf,
2142 size_t cnt, loff_t *ppos)
2143{
2144 char buf[64];
2145 char *cmp = buf;
2146 int neg = 0;
2147 int i;
2148
2149 if (cnt >= sizeof(buf))
2150 return -EINVAL;
2151
2152 if (copy_from_user(&buf, ubuf, cnt))
2153 return -EFAULT;
2154
2155 buf[cnt] = 0;
2156
2157 if (strncmp(buf, "no", 2) == 0) {
2158 neg = 1;
2159 cmp += 2;
2160 }
2161
2162 for (i = 0; trace_options[i]; i++) {
2163 int len = strlen(trace_options[i]);
2164
2165 if (strncmp(cmp, trace_options[i], len) == 0) {
2166 if (neg)
2167 trace_flags &= ~(1 << i);
2168 else
2169 trace_flags |= (1 << i);
2170 break;
2171 }
2172 }
2173 /*
2174 * If no option could be set, return an error:
2175 */
2176 if (!trace_options[i])
2177 return -EINVAL;
2178
2179 filp->f_pos += cnt;
2180
2181 return cnt;
2182}
2183
2184static struct file_operations tracing_iter_fops = {
2185 .open = tracing_open_generic,
2186 .read = tracing_iter_ctrl_read,
2187 .write = tracing_iter_ctrl_write,
2188};
2189
2190static const char readme_msg[] =
2191 "tracing mini-HOWTO:\n\n"
2192 "# mkdir /debug\n"
2193 "# mount -t debugfs nodev /debug\n\n"
2194 "# cat /debug/tracing/available_tracers\n"
2195 "wakeup preemptirqsoff preemptoff irqsoff ftrace sched_switch none\n\n"
2196 "# cat /debug/tracing/current_tracer\n"
2197 "none\n"
2198 "# echo sched_switch > /debug/tracing/current_tracer\n"
2199 "# cat /debug/tracing/current_tracer\n"
2200 "sched_switch\n"
2201 "# cat /debug/tracing/iter_ctrl\n"
2202 "noprint-parent nosym-offset nosym-addr noverbose\n"
2203 "# echo print-parent > /debug/tracing/iter_ctrl\n"
2204 "# echo 1 > /debug/tracing/tracing_enabled\n"
2205 "# cat /debug/tracing/trace > /tmp/trace.txt\n"
2206 "echo 0 > /debug/tracing/tracing_enabled\n"
2207;
2208
2209static ssize_t
2210tracing_readme_read(struct file *filp, char __user *ubuf,
2211 size_t cnt, loff_t *ppos)
2212{
2213 return simple_read_from_buffer(ubuf, cnt, ppos,
2214 readme_msg, strlen(readme_msg));
2215}
2216
2217static struct file_operations tracing_readme_fops = {
2218 .open = tracing_open_generic,
2219 .read = tracing_readme_read,
2220};
2221
2222static ssize_t
2223tracing_ctrl_read(struct file *filp, char __user *ubuf,
2224 size_t cnt, loff_t *ppos)
2225{
2226 struct trace_array *tr = filp->private_data;
2227 char buf[64];
2228 int r;
2229
2230 r = sprintf(buf, "%ld\n", tr->ctrl);
2231 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2232}
2233
2234static ssize_t
2235tracing_ctrl_write(struct file *filp, const char __user *ubuf,
2236 size_t cnt, loff_t *ppos)
2237{
2238 struct trace_array *tr = filp->private_data;
2239 char buf[64];
2240 long val;
2241 int ret;
2242
2243 if (cnt >= sizeof(buf))
2244 return -EINVAL;
2245
2246 if (copy_from_user(&buf, ubuf, cnt))
2247 return -EFAULT;
2248
2249 buf[cnt] = 0;
2250
2251 ret = strict_strtoul(buf, 10, &val);
2252 if (ret < 0)
2253 return ret;
2254
2255 val = !!val;
2256
2257 mutex_lock(&trace_types_lock);
2258 if (tr->ctrl ^ val) {
2259 if (val)
2260 tracer_enabled = 1;
2261 else
2262 tracer_enabled = 0;
2263
2264 tr->ctrl = val;
2265
2266 if (current_trace && current_trace->ctrl_update)
2267 current_trace->ctrl_update(tr);
2268 }
2269 mutex_unlock(&trace_types_lock);
2270
2271 filp->f_pos += cnt;
2272
2273 return cnt;
2274}
2275
2276static ssize_t
2277tracing_set_trace_read(struct file *filp, char __user *ubuf,
2278 size_t cnt, loff_t *ppos)
2279{
2280 char buf[max_tracer_type_len+2];
2281 int r;
2282
2283 mutex_lock(&trace_types_lock);
2284 if (current_trace)
2285 r = sprintf(buf, "%s\n", current_trace->name);
2286 else
2287 r = sprintf(buf, "\n");
2288 mutex_unlock(&trace_types_lock);
2289
2290 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2291}
2292
2293static ssize_t
2294tracing_set_trace_write(struct file *filp, const char __user *ubuf,
2295 size_t cnt, loff_t *ppos)
2296{
2297 struct trace_array *tr = &global_trace;
2298 struct tracer *t;
2299 char buf[max_tracer_type_len+1];
2300 int i;
2301
2302 if (cnt > max_tracer_type_len)
2303 cnt = max_tracer_type_len;
2304
2305 if (copy_from_user(&buf, ubuf, cnt))
2306 return -EFAULT;
2307
2308 buf[cnt] = 0;
2309
2310 /* strip ending whitespace. */
2311 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
2312 buf[i] = 0;
2313
2314 mutex_lock(&trace_types_lock);
2315 for (t = trace_types; t; t = t->next) {
2316 if (strcmp(t->name, buf) == 0)
2317 break;
2318 }
2319 if (!t || t == current_trace)
2320 goto out;
2321
2322 if (current_trace && current_trace->reset)
2323 current_trace->reset(tr);
2324
2325 current_trace = t;
2326 if (t->init)
2327 t->init(tr);
2328
2329 out:
2330 mutex_unlock(&trace_types_lock);
2331
2332 filp->f_pos += cnt;
2333
2334 return cnt;
2335}
2336
2337static ssize_t
2338tracing_max_lat_read(struct file *filp, char __user *ubuf,
2339 size_t cnt, loff_t *ppos)
2340{
2341 unsigned long *ptr = filp->private_data;
2342 char buf[64];
2343 int r;
2344
2345 r = snprintf(buf, sizeof(buf), "%ld\n",
2346 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
2347 if (r > sizeof(buf))
2348 r = sizeof(buf);
2349 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2350}
2351
2352static ssize_t
2353tracing_max_lat_write(struct file *filp, const char __user *ubuf,
2354 size_t cnt, loff_t *ppos)
2355{
2356 long *ptr = filp->private_data;
2357 char buf[64];
2358 long val;
2359 int ret;
2360
2361 if (cnt >= sizeof(buf))
2362 return -EINVAL;
2363
2364 if (copy_from_user(&buf, ubuf, cnt))
2365 return -EFAULT;
2366
2367 buf[cnt] = 0;
2368
2369 ret = strict_strtoul(buf, 10, &val);
2370 if (ret < 0)
2371 return ret;
2372
2373 *ptr = val * 1000;
2374
2375 return cnt;
2376}
2377
2378static atomic_t tracing_reader;
2379
2380static int tracing_open_pipe(struct inode *inode, struct file *filp)
2381{
2382 struct trace_iterator *iter;
2383
2384 if (tracing_disabled)
2385 return -ENODEV;
2386
2387 /* We only allow for reader of the pipe */
2388 if (atomic_inc_return(&tracing_reader) != 1) {
2389 atomic_dec(&tracing_reader);
2390 return -EBUSY;
2391 }
2392
2393 /* create a buffer to store the information to pass to userspace */
2394 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
2395 if (!iter)
2396 return -ENOMEM;
2397
2398 mutex_lock(&trace_types_lock);
2399 iter->tr = &global_trace;
2400 iter->trace = current_trace;
2401 filp->private_data = iter;
2402
2403 if (iter->trace->pipe_open)
2404 iter->trace->pipe_open(iter);
2405 mutex_unlock(&trace_types_lock);
2406
2407 return 0;
2408}
2409
2410static int tracing_release_pipe(struct inode *inode, struct file *file)
2411{
2412 struct trace_iterator *iter = file->private_data;
2413
2414 kfree(iter);
2415 atomic_dec(&tracing_reader);
2416
2417 return 0;
2418}
2419
2420static unsigned int
2421tracing_poll_pipe(struct file *filp, poll_table *poll_table)
2422{
2423 struct trace_iterator *iter = filp->private_data;
2424
2425 if (trace_flags & TRACE_ITER_BLOCK) {
2426 /*
2427 * Always select as readable when in blocking mode
2428 */
2429 return POLLIN | POLLRDNORM;
2430 } else {
2431 if (!trace_empty(iter))
2432 return POLLIN | POLLRDNORM;
2433 poll_wait(filp, &trace_wait, poll_table);
2434 if (!trace_empty(iter))
2435 return POLLIN | POLLRDNORM;
2436
2437 return 0;
2438 }
2439}
2440
2441/*
2442 * Consumer reader.
2443 */
2444static ssize_t
2445tracing_read_pipe(struct file *filp, char __user *ubuf,
2446 size_t cnt, loff_t *ppos)
2447{
2448 struct trace_iterator *iter = filp->private_data;
2449 struct trace_array_cpu *data;
2450 static cpumask_t mask;
2451 unsigned long flags;
2452#ifdef CONFIG_FTRACE
2453 int ftrace_save;
2454#endif
2455 int cpu;
2456 ssize_t sret;
2457
2458 /* return any leftover data */
2459 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
2460 if (sret != -EBUSY)
2461 return sret;
2462 sret = 0;
2463
2464 trace_seq_reset(&iter->seq);
2465
2466 mutex_lock(&trace_types_lock);
2467 if (iter->trace->read) {
2468 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
2469 if (sret)
2470 goto out;
2471 }
2472
2473 while (trace_empty(iter)) {
2474
2475 if ((filp->f_flags & O_NONBLOCK)) {
2476 sret = -EAGAIN;
2477 goto out;
2478 }
2479
2480 /*
2481 * This is a make-shift waitqueue. The reason we don't use
2482 * an actual wait queue is because:
2483 * 1) we only ever have one waiter
2484 * 2) the tracing, traces all functions, we don't want
2485 * the overhead of calling wake_up and friends
2486 * (and tracing them too)
2487 * Anyway, this is really very primitive wakeup.
2488 */
2489 set_current_state(TASK_INTERRUPTIBLE);
2490 iter->tr->waiter = current;
2491
2492 mutex_unlock(&trace_types_lock);
2493
2494 /* sleep for 100 msecs, and try again. */
2495 schedule_timeout(HZ/10);
2496
2497 mutex_lock(&trace_types_lock);
2498
2499 iter->tr->waiter = NULL;
2500
2501 if (signal_pending(current)) {
2502 sret = -EINTR;
2503 goto out;
2504 }
2505
2506 if (iter->trace != current_trace)
2507 goto out;
2508
2509 /*
2510 * We block until we read something and tracing is disabled.
2511 * We still block if tracing is disabled, but we have never
2512 * read anything. This allows a user to cat this file, and
2513 * then enable tracing. But after we have read something,
2514 * we give an EOF when tracing is again disabled.
2515 *
2516 * iter->pos will be 0 if we haven't read anything.
2517 */
2518 if (!tracer_enabled && iter->pos)
2519 break;
2520
2521 continue;
2522 }
2523
2524 /* stop when tracing is finished */
2525 if (trace_empty(iter))
2526 goto out;
2527
2528 if (cnt >= PAGE_SIZE)
2529 cnt = PAGE_SIZE - 1;
2530
2531 /* reset all but tr, trace, and overruns */
2532 memset(&iter->seq, 0,
2533 sizeof(struct trace_iterator) -
2534 offsetof(struct trace_iterator, seq));
2535 iter->pos = -1;
2536
2537 /*
2538 * We need to stop all tracing on all CPUS to read the
2539 * the next buffer. This is a bit expensive, but is
2540 * not done often. We fill all what we can read,
2541 * and then release the locks again.
2542 */
2543
2544 cpus_clear(mask);
2545 local_irq_save(flags);
2546#ifdef CONFIG_FTRACE
2547 ftrace_save = ftrace_enabled;
2548 ftrace_enabled = 0;
2549#endif
2550 smp_wmb();
2551 for_each_tracing_cpu(cpu) {
2552 data = iter->tr->data[cpu];
2553
2554 if (!head_page(data) || !data->trace_idx)
2555 continue;
2556
2557 atomic_inc(&data->disabled);
2558 cpu_set(cpu, mask);
2559 }
2560
2561 for_each_cpu_mask(cpu, mask) {
2562 data = iter->tr->data[cpu];
2563 __raw_spin_lock(&data->lock);
2564
2565 if (data->overrun > iter->last_overrun[cpu])
2566 iter->overrun[cpu] +=
2567 data->overrun - iter->last_overrun[cpu];
2568 iter->last_overrun[cpu] = data->overrun;
2569 }
2570
2571 while (find_next_entry_inc(iter) != NULL) {
2572 int ret;
2573 int len = iter->seq.len;
2574
2575 ret = print_trace_line(iter);
2576 if (!ret) {
2577 /* don't print partial lines */
2578 iter->seq.len = len;
2579 break;
2580 }
2581
2582 trace_consume(iter);
2583
2584 if (iter->seq.len >= cnt)
2585 break;
2586 }
2587
2588 for_each_cpu_mask(cpu, mask) {
2589 data = iter->tr->data[cpu];
2590 __raw_spin_unlock(&data->lock);
2591 }
2592
2593 for_each_cpu_mask(cpu, mask) {
2594 data = iter->tr->data[cpu];
2595 atomic_dec(&data->disabled);
2596 }
2597#ifdef CONFIG_FTRACE
2598 ftrace_enabled = ftrace_save;
2599#endif
2600 local_irq_restore(flags);
2601
2602 /* Now copy what we have to the user */
2603 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
2604 if (iter->seq.readpos >= iter->seq.len)
2605 trace_seq_reset(&iter->seq);
2606 if (sret == -EBUSY)
2607 sret = 0;
2608
2609out:
2610 mutex_unlock(&trace_types_lock);
2611
2612 return sret;
2613}
2614
2615static ssize_t
2616tracing_entries_read(struct file *filp, char __user *ubuf,
2617 size_t cnt, loff_t *ppos)
2618{
2619 struct trace_array *tr = filp->private_data;
2620 char buf[64];
2621 int r;
2622
2623 r = sprintf(buf, "%lu\n", tr->entries);
2624 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2625}
2626
2627static ssize_t
2628tracing_entries_write(struct file *filp, const char __user *ubuf,
2629 size_t cnt, loff_t *ppos)
2630{
2631 unsigned long val;
2632 char buf[64];
2633 int i, ret;
2634
2635 if (cnt >= sizeof(buf))
2636 return -EINVAL;
2637
2638 if (copy_from_user(&buf, ubuf, cnt))
2639 return -EFAULT;
2640
2641 buf[cnt] = 0;
2642
2643 ret = strict_strtoul(buf, 10, &val);
2644 if (ret < 0)
2645 return ret;
2646
2647 /* must have at least 1 entry */
2648 if (!val)
2649 return -EINVAL;
2650
2651 mutex_lock(&trace_types_lock);
2652
2653 if (current_trace != &no_tracer) {
2654 cnt = -EBUSY;
2655 pr_info("ftrace: set current_tracer to none"
2656 " before modifying buffer size\n");
2657 goto out;
2658 }
2659
2660 if (val > global_trace.entries) {
2661 long pages_requested;
2662 unsigned long freeable_pages;
2663
2664 /* make sure we have enough memory before mapping */
2665 pages_requested =
2666 (val + (ENTRIES_PER_PAGE-1)) / ENTRIES_PER_PAGE;
2667
2668 /* account for each buffer (and max_tr) */
2669 pages_requested *= tracing_nr_buffers * 2;
2670
2671 /* Check for overflow */
2672 if (pages_requested < 0) {
2673 cnt = -ENOMEM;
2674 goto out;
2675 }
2676
2677 freeable_pages = determine_dirtyable_memory();
2678
2679 /* we only allow to request 1/4 of useable memory */
2680 if (pages_requested >
2681 ((freeable_pages + tracing_pages_allocated) / 4)) {
2682 cnt = -ENOMEM;
2683 goto out;
2684 }
2685
2686 while (global_trace.entries < val) {
2687 if (trace_alloc_page()) {
2688 cnt = -ENOMEM;
2689 goto out;
2690 }
2691 /* double check that we don't go over the known pages */
2692 if (tracing_pages_allocated > pages_requested)
2693 break;
2694 }
2695
2696 } else {
2697 /* include the number of entries in val (inc of page entries) */
2698 while (global_trace.entries > val + (ENTRIES_PER_PAGE - 1))
2699 trace_free_page();
2700 }
2701
2702 /* check integrity */
2703 for_each_tracing_cpu(i)
2704 check_pages(global_trace.data[i]);
2705
2706 filp->f_pos += cnt;
2707
2708 /* If check pages failed, return ENOMEM */
2709 if (tracing_disabled)
2710 cnt = -ENOMEM;
2711 out:
2712 max_tr.entries = global_trace.entries;
2713 mutex_unlock(&trace_types_lock);
2714
2715 return cnt;
2716}
2717
2718static struct file_operations tracing_max_lat_fops = {
2719 .open = tracing_open_generic,
2720 .read = tracing_max_lat_read,
2721 .write = tracing_max_lat_write,
2722};
2723
2724static struct file_operations tracing_ctrl_fops = {
2725 .open = tracing_open_generic,
2726 .read = tracing_ctrl_read,
2727 .write = tracing_ctrl_write,
2728};
2729
2730static struct file_operations set_tracer_fops = {
2731 .open = tracing_open_generic,
2732 .read = tracing_set_trace_read,
2733 .write = tracing_set_trace_write,
2734};
2735
2736static struct file_operations tracing_pipe_fops = {
2737 .open = tracing_open_pipe,
2738 .poll = tracing_poll_pipe,
2739 .read = tracing_read_pipe,
2740 .release = tracing_release_pipe,
2741};
2742
2743static struct file_operations tracing_entries_fops = {
2744 .open = tracing_open_generic,
2745 .read = tracing_entries_read,
2746 .write = tracing_entries_write,
2747};
2748
2749#ifdef CONFIG_DYNAMIC_FTRACE
2750
2751static ssize_t
2752tracing_read_long(struct file *filp, char __user *ubuf,
2753 size_t cnt, loff_t *ppos)
2754{
2755 unsigned long *p = filp->private_data;
2756 char buf[64];
2757 int r;
2758
2759 r = sprintf(buf, "%ld\n", *p);
2760
2761 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2762}
2763
2764static struct file_operations tracing_read_long_fops = {
2765 .open = tracing_open_generic,
2766 .read = tracing_read_long,
2767};
2768#endif
2769
2770static struct dentry *d_tracer;
2771
2772struct dentry *tracing_init_dentry(void)
2773{
2774 static int once;
2775
2776 if (d_tracer)
2777 return d_tracer;
2778
2779 d_tracer = debugfs_create_dir("tracing", NULL);
2780
2781 if (!d_tracer && !once) {
2782 once = 1;
2783 pr_warning("Could not create debugfs directory 'tracing'\n");
2784 return NULL;
2785 }
2786
2787 return d_tracer;
2788}
2789
2790#ifdef CONFIG_FTRACE_SELFTEST
2791/* Let selftest have access to static functions in this file */
2792#include "trace_selftest.c"
2793#endif
2794
2795static __init void tracer_init_debugfs(void)
2796{
2797 struct dentry *d_tracer;
2798 struct dentry *entry;
2799
2800 d_tracer = tracing_init_dentry();
2801
2802 entry = debugfs_create_file("tracing_enabled", 0644, d_tracer,
2803 &global_trace, &tracing_ctrl_fops);
2804 if (!entry)
2805 pr_warning("Could not create debugfs 'tracing_enabled' entry\n");
2806
2807 entry = debugfs_create_file("iter_ctrl", 0644, d_tracer,
2808 NULL, &tracing_iter_fops);
2809 if (!entry)
2810 pr_warning("Could not create debugfs 'iter_ctrl' entry\n");
2811
2812 entry = debugfs_create_file("tracing_cpumask", 0644, d_tracer,
2813 NULL, &tracing_cpumask_fops);
2814 if (!entry)
2815 pr_warning("Could not create debugfs 'tracing_cpumask' entry\n");
2816
2817 entry = debugfs_create_file("latency_trace", 0444, d_tracer,
2818 &global_trace, &tracing_lt_fops);
2819 if (!entry)
2820 pr_warning("Could not create debugfs 'latency_trace' entry\n");
2821
2822 entry = debugfs_create_file("trace", 0444, d_tracer,
2823 &global_trace, &tracing_fops);
2824 if (!entry)
2825 pr_warning("Could not create debugfs 'trace' entry\n");
2826
2827 entry = debugfs_create_file("available_tracers", 0444, d_tracer,
2828 &global_trace, &show_traces_fops);
2829 if (!entry)
2830 pr_warning("Could not create debugfs 'trace' entry\n");
2831
2832 entry = debugfs_create_file("current_tracer", 0444, d_tracer,
2833 &global_trace, &set_tracer_fops);
2834 if (!entry)
2835 pr_warning("Could not create debugfs 'trace' entry\n");
2836
2837 entry = debugfs_create_file("tracing_max_latency", 0644, d_tracer,
2838 &tracing_max_latency,
2839 &tracing_max_lat_fops);
2840 if (!entry)
2841 pr_warning("Could not create debugfs "
2842 "'tracing_max_latency' entry\n");
2843
2844 entry = debugfs_create_file("tracing_thresh", 0644, d_tracer,
2845 &tracing_thresh, &tracing_max_lat_fops);
2846 if (!entry)
2847 pr_warning("Could not create debugfs "
2848 "'tracing_threash' entry\n");
2849 entry = debugfs_create_file("README", 0644, d_tracer,
2850 NULL, &tracing_readme_fops);
2851 if (!entry)
2852 pr_warning("Could not create debugfs 'README' entry\n");
2853
2854 entry = debugfs_create_file("trace_pipe", 0644, d_tracer,
2855 NULL, &tracing_pipe_fops);
2856 if (!entry)
2857 pr_warning("Could not create debugfs "
2858 "'tracing_threash' entry\n");
2859
2860 entry = debugfs_create_file("trace_entries", 0644, d_tracer,
2861 &global_trace, &tracing_entries_fops);
2862 if (!entry)
2863 pr_warning("Could not create debugfs "
2864 "'tracing_threash' entry\n");
2865
2866#ifdef CONFIG_DYNAMIC_FTRACE
2867 entry = debugfs_create_file("dyn_ftrace_total_info", 0444, d_tracer,
2868 &ftrace_update_tot_cnt,
2869 &tracing_read_long_fops);
2870 if (!entry)
2871 pr_warning("Could not create debugfs "
2872 "'dyn_ftrace_total_info' entry\n");
2873#endif
2874}
2875
2876static int trace_alloc_page(void)
2877{
2878 struct trace_array_cpu *data;
2879 struct page *page, *tmp;
2880 LIST_HEAD(pages);
2881 void *array;
2882 unsigned pages_allocated = 0;
2883 int i;
2884
2885 /* first allocate a page for each CPU */
2886 for_each_tracing_cpu(i) {
2887 array = (void *)__get_free_page(GFP_KERNEL);
2888 if (array == NULL) {
2889 printk(KERN_ERR "tracer: failed to allocate page"
2890 "for trace buffer!\n");
2891 goto free_pages;
2892 }
2893
2894 pages_allocated++;
2895 page = virt_to_page(array);
2896 list_add(&page->lru, &pages);
2897
2898/* Only allocate if we are actually using the max trace */
2899#ifdef CONFIG_TRACER_MAX_TRACE
2900 array = (void *)__get_free_page(GFP_KERNEL);
2901 if (array == NULL) {
2902 printk(KERN_ERR "tracer: failed to allocate page"
2903 "for trace buffer!\n");
2904 goto free_pages;
2905 }
2906 pages_allocated++;
2907 page = virt_to_page(array);
2908 list_add(&page->lru, &pages);
2909#endif
2910 }
2911
2912 /* Now that we successfully allocate a page per CPU, add them */
2913 for_each_tracing_cpu(i) {
2914 data = global_trace.data[i];
2915 page = list_entry(pages.next, struct page, lru);
2916 list_del_init(&page->lru);
2917 list_add_tail(&page->lru, &data->trace_pages);
2918 ClearPageLRU(page);
2919
2920#ifdef CONFIG_TRACER_MAX_TRACE
2921 data = max_tr.data[i];
2922 page = list_entry(pages.next, struct page, lru);
2923 list_del_init(&page->lru);
2924 list_add_tail(&page->lru, &data->trace_pages);
2925 SetPageLRU(page);
2926#endif
2927 }
2928 tracing_pages_allocated += pages_allocated;
2929 global_trace.entries += ENTRIES_PER_PAGE;
2930
2931 return 0;
2932
2933 free_pages:
2934 list_for_each_entry_safe(page, tmp, &pages, lru) {
2935 list_del_init(&page->lru);
2936 __free_page(page);
2937 }
2938 return -ENOMEM;
2939}
2940
2941static int trace_free_page(void)
2942{
2943 struct trace_array_cpu *data;
2944 struct page *page;
2945 struct list_head *p;
2946 int i;
2947 int ret = 0;
2948
2949 /* free one page from each buffer */
2950 for_each_tracing_cpu(i) {
2951 data = global_trace.data[i];
2952 p = data->trace_pages.next;
2953 if (p == &data->trace_pages) {
2954 /* should never happen */
2955 WARN_ON(1);
2956 tracing_disabled = 1;
2957 ret = -1;
2958 break;
2959 }
2960 page = list_entry(p, struct page, lru);
2961 ClearPageLRU(page);
2962 list_del(&page->lru);
2963 tracing_pages_allocated--;
2964 tracing_pages_allocated--;
2965 __free_page(page);
2966
2967 tracing_reset(data);
2968
2969#ifdef CONFIG_TRACER_MAX_TRACE
2970 data = max_tr.data[i];
2971 p = data->trace_pages.next;
2972 if (p == &data->trace_pages) {
2973 /* should never happen */
2974 WARN_ON(1);
2975 tracing_disabled = 1;
2976 ret = -1;
2977 break;
2978 }
2979 page = list_entry(p, struct page, lru);
2980 ClearPageLRU(page);
2981 list_del(&page->lru);
2982 __free_page(page);
2983
2984 tracing_reset(data);
2985#endif
2986 }
2987 global_trace.entries -= ENTRIES_PER_PAGE;
2988
2989 return ret;
2990}
2991
2992__init static int tracer_alloc_buffers(void)
2993{
2994 struct trace_array_cpu *data;
2995 void *array;
2996 struct page *page;
2997 int pages = 0;
2998 int ret = -ENOMEM;
2999 int i;
3000
3001 /* TODO: make the number of buffers hot pluggable with CPUS */
3002 tracing_nr_buffers = num_possible_cpus();
3003 tracing_buffer_mask = cpu_possible_map;
3004
3005 /* Allocate the first page for all buffers */
3006 for_each_tracing_cpu(i) {
3007 data = global_trace.data[i] = &per_cpu(global_trace_cpu, i);
3008 max_tr.data[i] = &per_cpu(max_data, i);
3009
3010 array = (void *)__get_free_page(GFP_KERNEL);
3011 if (array == NULL) {
3012 printk(KERN_ERR "tracer: failed to allocate page"
3013 "for trace buffer!\n");
3014 goto free_buffers;
3015 }
3016
3017 /* set the array to the list */
3018 INIT_LIST_HEAD(&data->trace_pages);
3019 page = virt_to_page(array);
3020 list_add(&page->lru, &data->trace_pages);
3021 /* use the LRU flag to differentiate the two buffers */
3022 ClearPageLRU(page);
3023
3024 data->lock = (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
3025 max_tr.data[i]->lock = (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
3026
3027/* Only allocate if we are actually using the max trace */
3028#ifdef CONFIG_TRACER_MAX_TRACE
3029 array = (void *)__get_free_page(GFP_KERNEL);
3030 if (array == NULL) {
3031 printk(KERN_ERR "tracer: failed to allocate page"
3032 "for trace buffer!\n");
3033 goto free_buffers;
3034 }
3035
3036 INIT_LIST_HEAD(&max_tr.data[i]->trace_pages);
3037 page = virt_to_page(array);
3038 list_add(&page->lru, &max_tr.data[i]->trace_pages);
3039 SetPageLRU(page);
3040#endif
3041 }
3042
3043 /*
3044 * Since we allocate by orders of pages, we may be able to
3045 * round up a bit.
3046 */
3047 global_trace.entries = ENTRIES_PER_PAGE;
3048 pages++;
3049
3050 while (global_trace.entries < trace_nr_entries) {
3051 if (trace_alloc_page())
3052 break;
3053 pages++;
3054 }
3055 max_tr.entries = global_trace.entries;
3056
3057 pr_info("tracer: %d pages allocated for %ld entries of %ld bytes\n",
3058 pages, trace_nr_entries, (long)TRACE_ENTRY_SIZE);
3059 pr_info(" actual entries %ld\n", global_trace.entries);
3060
3061 tracer_init_debugfs();
3062
3063 trace_init_cmdlines();
3064
3065 register_tracer(&no_tracer);
3066 current_trace = &no_tracer;
3067
3068 /* All seems OK, enable tracing */
3069 global_trace.ctrl = tracer_enabled;
3070 tracing_disabled = 0;
3071
3072 return 0;
3073
3074 free_buffers:
3075 for (i-- ; i >= 0; i--) {
3076 struct page *page, *tmp;
3077 struct trace_array_cpu *data = global_trace.data[i];
3078
3079 if (data) {
3080 list_for_each_entry_safe(page, tmp,
3081 &data->trace_pages, lru) {
3082 list_del_init(&page->lru);
3083 __free_page(page);
3084 }
3085 }
3086
3087#ifdef CONFIG_TRACER_MAX_TRACE
3088 data = max_tr.data[i];
3089 if (data) {
3090 list_for_each_entry_safe(page, tmp,
3091 &data->trace_pages, lru) {
3092 list_del_init(&page->lru);
3093 __free_page(page);
3094 }
3095 }
3096#endif
3097 }
3098 return ret;
3099}
3100fs_initcall(tracer_alloc_buffers);
generated by cgit v1.2.2 (git 2.25.0) at 2025-12-11 11:35:26 -0500