aboutsummaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/relay.c4
-rw-r--r--kernel/sched.c8
-rw-r--r--kernel/softirq.c13
-rw-r--r--kernel/trace/Kconfig55
-rw-r--r--kernel/trace/Makefile4
-rw-r--r--kernel/trace/ftrace.c18
-rw-r--r--kernel/trace/kmemtrace.c350
-rw-r--r--kernel/trace/ring_buffer.c55
-rw-r--r--kernel/trace/trace.c876
-rw-r--r--kernel/trace/trace.h61
-rw-r--r--kernel/trace/trace_boot.c1
-rw-r--r--kernel/trace/trace_branch.c284
-rw-r--r--kernel/trace/trace_functions.c193
-rw-r--r--kernel/trace/trace_functions_graph.c314
-rw-r--r--kernel/trace/trace_hw_branches.c174
-rw-r--r--kernel/trace/trace_irqsoff.c34
-rw-r--r--kernel/trace/trace_mmiotrace.c31
-rw-r--r--kernel/trace/trace_nop.c6
-rw-r--r--kernel/trace/trace_output.c829
-rw-r--r--kernel/trace/trace_output.h60
-rw-r--r--kernel/trace/trace_power.c1
-rw-r--r--kernel/trace/trace_sched_wakeup.c83
-rw-r--r--kernel/trace/trace_selftest.c1
-rw-r--r--kernel/trace/trace_stat.c319
-rw-r--r--kernel/trace/trace_stat.h31
-rw-r--r--kernel/trace/trace_workqueue.c281
-rw-r--r--kernel/workqueue.c16
27 files changed, 2878 insertions, 1224 deletions
diff --git a/kernel/relay.c b/kernel/relay.c
index 9d79b7854fa6..edc0ba6d8160 100644
--- a/kernel/relay.c
+++ b/kernel/relay.c
@@ -677,9 +677,7 @@ int relay_late_setup_files(struct rchan *chan,
677 */ 677 */
678 for_each_online_cpu(i) { 678 for_each_online_cpu(i) {
679 if (unlikely(!chan->buf[i])) { 679 if (unlikely(!chan->buf[i])) {
680 printk(KERN_ERR "relay_late_setup_files: CPU %u " 680 WARN_ONCE(1, KERN_ERR "CPU has no buffer!\n");
681 "has no buffer, it must have!\n", i);
682 BUG();
683 err = -EINVAL; 681 err = -EINVAL;
684 break; 682 break;
685 } 683 }
diff --git a/kernel/sched.c b/kernel/sched.c
index 242d0d47a70d..566c8c9e3a6d 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -4409,10 +4409,7 @@ void scheduler_tick(void)
4409#endif 4409#endif
4410} 4410}
4411 4411
4412#if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \ 4412unsigned long get_parent_ip(unsigned long addr)
4413 defined(CONFIG_PREEMPT_TRACER))
4414
4415static inline unsigned long get_parent_ip(unsigned long addr)
4416{ 4413{
4417 if (in_lock_functions(addr)) { 4414 if (in_lock_functions(addr)) {
4418 addr = CALLER_ADDR2; 4415 addr = CALLER_ADDR2;
@@ -4422,6 +4419,9 @@ static inline unsigned long get_parent_ip(unsigned long addr)
4422 return addr; 4419 return addr;
4423} 4420}
4424 4421
4422#if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \
4423 defined(CONFIG_PREEMPT_TRACER))
4424
4425void __kprobes add_preempt_count(int val) 4425void __kprobes add_preempt_count(int val)
4426{ 4426{
4427#ifdef CONFIG_DEBUG_PREEMPT 4427#ifdef CONFIG_DEBUG_PREEMPT
diff --git a/kernel/softirq.c b/kernel/softirq.c
index bdbe9de9cd8d..6edfc2c11d99 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -21,6 +21,7 @@
21#include <linux/freezer.h> 21#include <linux/freezer.h>
22#include <linux/kthread.h> 22#include <linux/kthread.h>
23#include <linux/rcupdate.h> 23#include <linux/rcupdate.h>
24#include <linux/ftrace.h>
24#include <linux/smp.h> 25#include <linux/smp.h>
25#include <linux/tick.h> 26#include <linux/tick.h>
26 27
@@ -79,13 +80,23 @@ static void __local_bh_disable(unsigned long ip)
79 WARN_ON_ONCE(in_irq()); 80 WARN_ON_ONCE(in_irq());
80 81
81 raw_local_irq_save(flags); 82 raw_local_irq_save(flags);
82 add_preempt_count(SOFTIRQ_OFFSET); 83 /*
84 * The preempt tracer hooks into add_preempt_count and will break
85 * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
86 * is set and before current->softirq_enabled is cleared.
87 * We must manually increment preempt_count here and manually
88 * call the trace_preempt_off later.
89 */
90 preempt_count() += SOFTIRQ_OFFSET;
83 /* 91 /*
84 * Were softirqs turned off above: 92 * Were softirqs turned off above:
85 */ 93 */
86 if (softirq_count() == SOFTIRQ_OFFSET) 94 if (softirq_count() == SOFTIRQ_OFFSET)
87 trace_softirqs_off(ip); 95 trace_softirqs_off(ip);
88 raw_local_irq_restore(flags); 96 raw_local_irq_restore(flags);
97
98 if (preempt_count() == SOFTIRQ_OFFSET)
99 trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
89} 100}
90#else /* !CONFIG_TRACE_IRQFLAGS */ 101#else /* !CONFIG_TRACE_IRQFLAGS */
91static inline void __local_bh_disable(unsigned long ip) 102static inline void __local_bh_disable(unsigned long ip)
diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig
index e2a4ff6fc3a6..dde1d46f77e5 100644
--- a/kernel/trace/Kconfig
+++ b/kernel/trace/Kconfig
@@ -264,6 +264,38 @@ config HW_BRANCH_TRACER
264 This tracer records all branches on the system in a circular 264 This tracer records all branches on the system in a circular
265 buffer giving access to the last N branches for each cpu. 265 buffer giving access to the last N branches for each cpu.
266 266
267config KMEMTRACE
268 bool "Trace SLAB allocations"
269 select TRACING
270 help
271 kmemtrace provides tracing for slab allocator functions, such as
272 kmalloc, kfree, kmem_cache_alloc, kmem_cache_free etc.. Collected
273 data is then fed to the userspace application in order to analyse
274 allocation hotspots, internal fragmentation and so on, making it
275 possible to see how well an allocator performs, as well as debug
276 and profile kernel code.
277
278 This requires an userspace application to use. See
279 Documentation/vm/kmemtrace.txt for more information.
280
281 Saying Y will make the kernel somewhat larger and slower. However,
282 if you disable kmemtrace at run-time or boot-time, the performance
283 impact is minimal (depending on the arch the kernel is built for).
284
285 If unsure, say N.
286
287config WORKQUEUE_TRACER
288 bool "Trace workqueues"
289 select TRACING
290 help
291 The workqueue tracer provides some statistical informations
292 about each cpu workqueue thread such as the number of the
293 works inserted and executed since their creation. It can help
294 to evaluate the amount of work each of them have to perform.
295 For example it can help a developer to decide whether he should
296 choose a per cpu workqueue instead of a singlethreaded one.
297
298
267config DYNAMIC_FTRACE 299config DYNAMIC_FTRACE
268 bool "enable/disable ftrace tracepoints dynamically" 300 bool "enable/disable ftrace tracepoints dynamically"
269 depends on FUNCTION_TRACER 301 depends on FUNCTION_TRACER
@@ -302,4 +334,27 @@ config FTRACE_STARTUP_TEST
302 functioning properly. It will do tests on all the configured 334 functioning properly. It will do tests on all the configured
303 tracers of ftrace. 335 tracers of ftrace.
304 336
337config MMIOTRACE
338 bool "Memory mapped IO tracing"
339 depends on HAVE_MMIOTRACE_SUPPORT && DEBUG_KERNEL && PCI
340 select TRACING
341 help
342 Mmiotrace traces Memory Mapped I/O access and is meant for
343 debugging and reverse engineering. It is called from the ioremap
344 implementation and works via page faults. Tracing is disabled by
345 default and can be enabled at run-time.
346
347 See Documentation/tracers/mmiotrace.txt.
348 If you are not helping to develop drivers, say N.
349
350config MMIOTRACE_TEST
351 tristate "Test module for mmiotrace"
352 depends on MMIOTRACE && m
353 help
354 This is a dumb module for testing mmiotrace. It is very dangerous
355 as it will write garbage to IO memory starting at a given address.
356 However, it should be safe to use on e.g. unused portion of VRAM.
357
358 Say N, unless you absolutely know what you are doing.
359
305endmenu 360endmenu
diff --git a/kernel/trace/Makefile b/kernel/trace/Makefile
index 349d5a93653f..f76d48f3527d 100644
--- a/kernel/trace/Makefile
+++ b/kernel/trace/Makefile
@@ -19,6 +19,8 @@ obj-$(CONFIG_FUNCTION_TRACER) += libftrace.o
19obj-$(CONFIG_RING_BUFFER) += ring_buffer.o 19obj-$(CONFIG_RING_BUFFER) += ring_buffer.o
20 20
21obj-$(CONFIG_TRACING) += trace.o 21obj-$(CONFIG_TRACING) += trace.o
22obj-$(CONFIG_TRACING) += trace_output.o
23obj-$(CONFIG_TRACING) += trace_stat.o
22obj-$(CONFIG_CONTEXT_SWITCH_TRACER) += trace_sched_switch.o 24obj-$(CONFIG_CONTEXT_SWITCH_TRACER) += trace_sched_switch.o
23obj-$(CONFIG_SYSPROF_TRACER) += trace_sysprof.o 25obj-$(CONFIG_SYSPROF_TRACER) += trace_sysprof.o
24obj-$(CONFIG_FUNCTION_TRACER) += trace_functions.o 26obj-$(CONFIG_FUNCTION_TRACER) += trace_functions.o
@@ -33,5 +35,7 @@ obj-$(CONFIG_FUNCTION_GRAPH_TRACER) += trace_functions_graph.o
33obj-$(CONFIG_TRACE_BRANCH_PROFILING) += trace_branch.o 35obj-$(CONFIG_TRACE_BRANCH_PROFILING) += trace_branch.o
34obj-$(CONFIG_HW_BRANCH_TRACER) += trace_hw_branches.o 36obj-$(CONFIG_HW_BRANCH_TRACER) += trace_hw_branches.o
35obj-$(CONFIG_POWER_TRACER) += trace_power.o 37obj-$(CONFIG_POWER_TRACER) += trace_power.o
38obj-$(CONFIG_KMEMTRACE) += kmemtrace.o
39obj-$(CONFIG_WORKQUEUE_TRACER) += trace_workqueue.o
36 40
37libftrace-y := ftrace.o 41libftrace-y := ftrace.o
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
index 7dcf6e9f2b04..7e9a20b69939 100644
--- a/kernel/trace/ftrace.c
+++ b/kernel/trace/ftrace.c
@@ -264,14 +264,6 @@ static void ftrace_update_pid_func(void)
264# error Dynamic ftrace depends on MCOUNT_RECORD 264# error Dynamic ftrace depends on MCOUNT_RECORD
265#endif 265#endif
266 266
267/*
268 * Since MCOUNT_ADDR may point to mcount itself, we do not want
269 * to get it confused by reading a reference in the code as we
270 * are parsing on objcopy output of text. Use a variable for
271 * it instead.
272 */
273static unsigned long mcount_addr = MCOUNT_ADDR;
274
275enum { 267enum {
276 FTRACE_ENABLE_CALLS = (1 << 0), 268 FTRACE_ENABLE_CALLS = (1 << 0),
277 FTRACE_DISABLE_CALLS = (1 << 1), 269 FTRACE_DISABLE_CALLS = (1 << 1),
@@ -290,7 +282,7 @@ static DEFINE_MUTEX(ftrace_regex_lock);
290 282
291struct ftrace_page { 283struct ftrace_page {
292 struct ftrace_page *next; 284 struct ftrace_page *next;
293 unsigned long index; 285 int index;
294 struct dyn_ftrace records[]; 286 struct dyn_ftrace records[];
295}; 287};
296 288
@@ -464,7 +456,7 @@ __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
464 unsigned long ip, fl; 456 unsigned long ip, fl;
465 unsigned long ftrace_addr; 457 unsigned long ftrace_addr;
466 458
467 ftrace_addr = (unsigned long)ftrace_caller; 459 ftrace_addr = (unsigned long)FTRACE_ADDR;
468 460
469 ip = rec->ip; 461 ip = rec->ip;
470 462
@@ -576,7 +568,7 @@ ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
576 568
577 ip = rec->ip; 569 ip = rec->ip;
578 570
579 ret = ftrace_make_nop(mod, rec, mcount_addr); 571 ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
580 if (ret) { 572 if (ret) {
581 ftrace_bug(ret, ip); 573 ftrace_bug(ret, ip);
582 rec->flags |= FTRACE_FL_FAILED; 574 rec->flags |= FTRACE_FL_FAILED;
@@ -787,7 +779,7 @@ enum {
787 779
788struct ftrace_iterator { 780struct ftrace_iterator {
789 struct ftrace_page *pg; 781 struct ftrace_page *pg;
790 unsigned idx; 782 int idx;
791 unsigned flags; 783 unsigned flags;
792 unsigned char buffer[FTRACE_BUFF_MAX+1]; 784 unsigned char buffer[FTRACE_BUFF_MAX+1];
793 unsigned buffer_idx; 785 unsigned buffer_idx;
@@ -1903,7 +1895,7 @@ int register_ftrace_function(struct ftrace_ops *ops)
1903} 1895}
1904 1896
1905/** 1897/**
1906 * unregister_ftrace_function - unresgister a function for profiling. 1898 * unregister_ftrace_function - unregister a function for profiling.
1907 * @ops - ops structure that holds the function to unregister 1899 * @ops - ops structure that holds the function to unregister
1908 * 1900 *
1909 * Unregister a function that was added to be called by ftrace profiling. 1901 * Unregister a function that was added to be called by ftrace profiling.
diff --git a/kernel/trace/kmemtrace.c b/kernel/trace/kmemtrace.c
new file mode 100644
index 000000000000..f04c0625f1cd
--- /dev/null
+++ b/kernel/trace/kmemtrace.c
@@ -0,0 +1,350 @@
1/*
2 * Memory allocator tracing
3 *
4 * Copyright (C) 2008 Eduard - Gabriel Munteanu
5 * Copyright (C) 2008 Pekka Enberg <penberg@cs.helsinki.fi>
6 * Copyright (C) 2008 Frederic Weisbecker <fweisbec@gmail.com>
7 */
8
9#include <linux/dcache.h>
10#include <linux/debugfs.h>
11#include <linux/fs.h>
12#include <linux/seq_file.h>
13#include <trace/kmemtrace.h>
14
15#include "trace.h"
16#include "trace_output.h"
17
18/* Select an alternative, minimalistic output than the original one */
19#define TRACE_KMEM_OPT_MINIMAL 0x1
20
21static struct tracer_opt kmem_opts[] = {
22 /* Default disable the minimalistic output */
23 { TRACER_OPT(kmem_minimalistic, TRACE_KMEM_OPT_MINIMAL) },
24 { }
25};
26
27static struct tracer_flags kmem_tracer_flags = {
28 .val = 0,
29 .opts = kmem_opts
30};
31
32
33static bool kmem_tracing_enabled __read_mostly;
34static struct trace_array *kmemtrace_array;
35
36static int kmem_trace_init(struct trace_array *tr)
37{
38 int cpu;
39 kmemtrace_array = tr;
40
41 for_each_cpu_mask(cpu, cpu_possible_map)
42 tracing_reset(tr, cpu);
43
44 kmem_tracing_enabled = true;
45
46 return 0;
47}
48
49static void kmem_trace_reset(struct trace_array *tr)
50{
51 kmem_tracing_enabled = false;
52}
53
54static void kmemtrace_headers(struct seq_file *s)
55{
56 /* Don't need headers for the original kmemtrace output */
57 if (!(kmem_tracer_flags.val & TRACE_KMEM_OPT_MINIMAL))
58 return;
59
60 seq_printf(s, "#\n");
61 seq_printf(s, "# ALLOC TYPE REQ GIVEN FLAGS "
62 " POINTER NODE CALLER\n");
63 seq_printf(s, "# FREE | | | | "
64 " | | | |\n");
65 seq_printf(s, "# |\n\n");
66}
67
68/*
69 * The two following functions give the original output from kmemtrace,
70 * or something close to....perhaps they need some missing things
71 */
72static enum print_line_t
73kmemtrace_print_alloc_original(struct trace_iterator *iter,
74 struct kmemtrace_alloc_entry *entry)
75{
76 struct trace_seq *s = &iter->seq;
77 int ret;
78
79 /* Taken from the old linux/kmemtrace.h */
80 ret = trace_seq_printf(s, "type_id %d call_site %lu ptr %lu "
81 "bytes_req %lu bytes_alloc %lu gfp_flags %lu node %d\n",
82 entry->type_id, entry->call_site, (unsigned long) entry->ptr,
83 (unsigned long) entry->bytes_req, (unsigned long) entry->bytes_alloc,
84 (unsigned long) entry->gfp_flags, entry->node);
85
86 if (!ret)
87 return TRACE_TYPE_PARTIAL_LINE;
88
89 return TRACE_TYPE_HANDLED;
90}
91
92static enum print_line_t
93kmemtrace_print_free_original(struct trace_iterator *iter,
94 struct kmemtrace_free_entry *entry)
95{
96 struct trace_seq *s = &iter->seq;
97 int ret;
98
99 /* Taken from the old linux/kmemtrace.h */
100 ret = trace_seq_printf(s, "type_id %d call_site %lu ptr %lu\n",
101 entry->type_id, entry->call_site, (unsigned long) entry->ptr);
102
103 if (!ret)
104 return TRACE_TYPE_PARTIAL_LINE;
105
106 return TRACE_TYPE_HANDLED;
107}
108
109
110/* The two other following provide a more minimalistic output */
111static enum print_line_t
112kmemtrace_print_alloc_compress(struct trace_iterator *iter,
113 struct kmemtrace_alloc_entry *entry)
114{
115 struct trace_seq *s = &iter->seq;
116 int ret;
117
118 /* Alloc entry */
119 ret = trace_seq_printf(s, " + ");
120 if (!ret)
121 return TRACE_TYPE_PARTIAL_LINE;
122
123 /* Type */
124 switch (entry->type_id) {
125 case KMEMTRACE_TYPE_KMALLOC:
126 ret = trace_seq_printf(s, "K ");
127 break;
128 case KMEMTRACE_TYPE_CACHE:
129 ret = trace_seq_printf(s, "C ");
130 break;
131 case KMEMTRACE_TYPE_PAGES:
132 ret = trace_seq_printf(s, "P ");
133 break;
134 default:
135 ret = trace_seq_printf(s, "? ");
136 }
137
138 if (!ret)
139 return TRACE_TYPE_PARTIAL_LINE;
140
141 /* Requested */
142 ret = trace_seq_printf(s, "%4zu ", entry->bytes_req);
143 if (!ret)
144 return TRACE_TYPE_PARTIAL_LINE;
145
146 /* Allocated */
147 ret = trace_seq_printf(s, "%4zu ", entry->bytes_alloc);
148 if (!ret)
149 return TRACE_TYPE_PARTIAL_LINE;
150
151 /* Flags
152 * TODO: would be better to see the name of the GFP flag names
153 */
154 ret = trace_seq_printf(s, "%08x ", entry->gfp_flags);
155 if (!ret)
156 return TRACE_TYPE_PARTIAL_LINE;
157
158 /* Pointer to allocated */
159 ret = trace_seq_printf(s, "0x%tx ", (ptrdiff_t)entry->ptr);
160 if (!ret)
161 return TRACE_TYPE_PARTIAL_LINE;
162
163 /* Node */
164 ret = trace_seq_printf(s, "%4d ", entry->node);
165 if (!ret)
166 return TRACE_TYPE_PARTIAL_LINE;
167
168 /* Call site */
169 ret = seq_print_ip_sym(s, entry->call_site, 0);
170 if (!ret)
171 return TRACE_TYPE_PARTIAL_LINE;
172
173 if (!trace_seq_printf(s, "\n"))
174 return TRACE_TYPE_PARTIAL_LINE;
175
176 return TRACE_TYPE_HANDLED;
177}
178
179static enum print_line_t
180kmemtrace_print_free_compress(struct trace_iterator *iter,
181 struct kmemtrace_free_entry *entry)
182{
183 struct trace_seq *s = &iter->seq;
184 int ret;
185
186 /* Free entry */
187 ret = trace_seq_printf(s, " - ");
188 if (!ret)
189 return TRACE_TYPE_PARTIAL_LINE;
190
191 /* Type */
192 switch (entry->type_id) {
193 case KMEMTRACE_TYPE_KMALLOC:
194 ret = trace_seq_printf(s, "K ");
195 break;
196 case KMEMTRACE_TYPE_CACHE:
197 ret = trace_seq_printf(s, "C ");
198 break;
199 case KMEMTRACE_TYPE_PAGES:
200 ret = trace_seq_printf(s, "P ");
201 break;
202 default:
203 ret = trace_seq_printf(s, "? ");
204 }
205
206 if (!ret)
207 return TRACE_TYPE_PARTIAL_LINE;
208
209 /* Skip requested/allocated/flags */
210 ret = trace_seq_printf(s, " ");
211 if (!ret)
212 return TRACE_TYPE_PARTIAL_LINE;
213
214 /* Pointer to allocated */
215 ret = trace_seq_printf(s, "0x%tx ", (ptrdiff_t)entry->ptr);
216 if (!ret)
217 return TRACE_TYPE_PARTIAL_LINE;
218
219 /* Skip node */
220 ret = trace_seq_printf(s, " ");
221 if (!ret)
222 return TRACE_TYPE_PARTIAL_LINE;
223
224 /* Call site */
225 ret = seq_print_ip_sym(s, entry->call_site, 0);
226 if (!ret)
227 return TRACE_TYPE_PARTIAL_LINE;
228
229 if (!trace_seq_printf(s, "\n"))
230 return TRACE_TYPE_PARTIAL_LINE;
231
232 return TRACE_TYPE_HANDLED;
233}
234
235static enum print_line_t kmemtrace_print_line(struct trace_iterator *iter)
236{
237 struct trace_entry *entry = iter->ent;
238
239 switch (entry->type) {
240 case TRACE_KMEM_ALLOC: {
241 struct kmemtrace_alloc_entry *field;
242 trace_assign_type(field, entry);
243 if (kmem_tracer_flags.val & TRACE_KMEM_OPT_MINIMAL)
244 return kmemtrace_print_alloc_compress(iter, field);
245 else
246 return kmemtrace_print_alloc_original(iter, field);
247 }
248
249 case TRACE_KMEM_FREE: {
250 struct kmemtrace_free_entry *field;
251 trace_assign_type(field, entry);
252 if (kmem_tracer_flags.val & TRACE_KMEM_OPT_MINIMAL)
253 return kmemtrace_print_free_compress(iter, field);
254 else
255 return kmemtrace_print_free_original(iter, field);
256 }
257
258 default:
259 return TRACE_TYPE_UNHANDLED;
260 }
261}
262
263/* Trace allocations */
264void kmemtrace_mark_alloc_node(enum kmemtrace_type_id type_id,
265 unsigned long call_site,
266 const void *ptr,
267 size_t bytes_req,
268 size_t bytes_alloc,
269 gfp_t gfp_flags,
270 int node)
271{
272 struct ring_buffer_event *event;
273 struct kmemtrace_alloc_entry *entry;
274 struct trace_array *tr = kmemtrace_array;
275 unsigned long irq_flags;
276
277 if (!kmem_tracing_enabled)
278 return;
279
280 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
281 &irq_flags);
282 if (!event)
283 return;
284 entry = ring_buffer_event_data(event);
285 tracing_generic_entry_update(&entry->ent, 0, 0);
286
287 entry->ent.type = TRACE_KMEM_ALLOC;
288 entry->call_site = call_site;
289 entry->ptr = ptr;
290 entry->bytes_req = bytes_req;
291 entry->bytes_alloc = bytes_alloc;
292 entry->gfp_flags = gfp_flags;
293 entry->node = node;
294
295 ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
296
297 trace_wake_up();
298}
299EXPORT_SYMBOL(kmemtrace_mark_alloc_node);
300
301void kmemtrace_mark_free(enum kmemtrace_type_id type_id,
302 unsigned long call_site,
303 const void *ptr)
304{
305 struct ring_buffer_event *event;
306 struct kmemtrace_free_entry *entry;
307 struct trace_array *tr = kmemtrace_array;
308 unsigned long irq_flags;
309
310 if (!kmem_tracing_enabled)
311 return;
312
313 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
314 &irq_flags);
315 if (!event)
316 return;
317 entry = ring_buffer_event_data(event);
318 tracing_generic_entry_update(&entry->ent, 0, 0);
319
320 entry->ent.type = TRACE_KMEM_FREE;
321 entry->type_id = type_id;
322 entry->call_site = call_site;
323 entry->ptr = ptr;
324
325 ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
326
327 trace_wake_up();
328}
329EXPORT_SYMBOL(kmemtrace_mark_free);
330
331static struct tracer kmem_tracer __read_mostly = {
332 .name = "kmemtrace",
333 .init = kmem_trace_init,
334 .reset = kmem_trace_reset,
335 .print_line = kmemtrace_print_line,
336 .print_header = kmemtrace_headers,
337 .flags = &kmem_tracer_flags
338};
339
340void kmemtrace_init(void)
341{
342 /* earliest opportunity to start kmem tracing */
343}
344
345static int __init init_kmem_tracer(void)
346{
347 return register_tracer(&kmem_tracer);
348}
349
350device_initcall(init_kmem_tracer);
diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c
index bd38c5cfd8ad..b36d7374ceef 100644
--- a/kernel/trace/ring_buffer.c
+++ b/kernel/trace/ring_buffer.c
@@ -123,8 +123,7 @@ void ring_buffer_normalize_time_stamp(int cpu, u64 *ts)
123EXPORT_SYMBOL_GPL(ring_buffer_normalize_time_stamp); 123EXPORT_SYMBOL_GPL(ring_buffer_normalize_time_stamp);
124 124
125#define RB_EVNT_HDR_SIZE (sizeof(struct ring_buffer_event)) 125#define RB_EVNT_HDR_SIZE (sizeof(struct ring_buffer_event))
126#define RB_ALIGNMENT_SHIFT 2 126#define RB_ALIGNMENT 4U
127#define RB_ALIGNMENT (1 << RB_ALIGNMENT_SHIFT)
128#define RB_MAX_SMALL_DATA 28 127#define RB_MAX_SMALL_DATA 28
129 128
130enum { 129enum {
@@ -133,7 +132,7 @@ enum {
133}; 132};
134 133
135/* inline for ring buffer fast paths */ 134/* inline for ring buffer fast paths */
136static inline unsigned 135static unsigned
137rb_event_length(struct ring_buffer_event *event) 136rb_event_length(struct ring_buffer_event *event)
138{ 137{
139 unsigned length; 138 unsigned length;
@@ -151,7 +150,7 @@ rb_event_length(struct ring_buffer_event *event)
151 150
152 case RINGBUF_TYPE_DATA: 151 case RINGBUF_TYPE_DATA:
153 if (event->len) 152 if (event->len)
154 length = event->len << RB_ALIGNMENT_SHIFT; 153 length = event->len * RB_ALIGNMENT;
155 else 154 else
156 length = event->array[0]; 155 length = event->array[0];
157 return length + RB_EVNT_HDR_SIZE; 156 return length + RB_EVNT_HDR_SIZE;
@@ -179,7 +178,7 @@ unsigned ring_buffer_event_length(struct ring_buffer_event *event)
179EXPORT_SYMBOL_GPL(ring_buffer_event_length); 178EXPORT_SYMBOL_GPL(ring_buffer_event_length);
180 179
181/* inline for ring buffer fast paths */ 180/* inline for ring buffer fast paths */
182static inline void * 181static void *
183rb_event_data(struct ring_buffer_event *event) 182rb_event_data(struct ring_buffer_event *event)
184{ 183{
185 BUG_ON(event->type != RINGBUF_TYPE_DATA); 184 BUG_ON(event->type != RINGBUF_TYPE_DATA);
@@ -229,10 +228,9 @@ static void rb_init_page(struct buffer_data_page *bpage)
229 * Also stolen from mm/slob.c. Thanks to Mathieu Desnoyers for pointing 228 * Also stolen from mm/slob.c. Thanks to Mathieu Desnoyers for pointing
230 * this issue out. 229 * this issue out.
231 */ 230 */
232static inline void free_buffer_page(struct buffer_page *bpage) 231static void free_buffer_page(struct buffer_page *bpage)
233{ 232{
234 if (bpage->page) 233 free_page((unsigned long)bpage->page);
235 free_page((unsigned long)bpage->page);
236 kfree(bpage); 234 kfree(bpage);
237} 235}
238 236
@@ -811,7 +809,7 @@ rb_event_index(struct ring_buffer_event *event)
811 return (addr & ~PAGE_MASK) - (PAGE_SIZE - BUF_PAGE_SIZE); 809 return (addr & ~PAGE_MASK) - (PAGE_SIZE - BUF_PAGE_SIZE);
812} 810}
813 811
814static inline int 812static int
815rb_is_commit(struct ring_buffer_per_cpu *cpu_buffer, 813rb_is_commit(struct ring_buffer_per_cpu *cpu_buffer,
816 struct ring_buffer_event *event) 814 struct ring_buffer_event *event)
817{ 815{
@@ -825,7 +823,7 @@ rb_is_commit(struct ring_buffer_per_cpu *cpu_buffer,
825 rb_commit_index(cpu_buffer) == index; 823 rb_commit_index(cpu_buffer) == index;
826} 824}
827 825
828static inline void 826static void
829rb_set_commit_event(struct ring_buffer_per_cpu *cpu_buffer, 827rb_set_commit_event(struct ring_buffer_per_cpu *cpu_buffer,
830 struct ring_buffer_event *event) 828 struct ring_buffer_event *event)
831{ 829{
@@ -850,7 +848,7 @@ rb_set_commit_event(struct ring_buffer_per_cpu *cpu_buffer,
850 local_set(&cpu_buffer->commit_page->page->commit, index); 848 local_set(&cpu_buffer->commit_page->page->commit, index);
851} 849}
852 850
853static inline void 851static void
854rb_set_commit_to_write(struct ring_buffer_per_cpu *cpu_buffer) 852rb_set_commit_to_write(struct ring_buffer_per_cpu *cpu_buffer)
855{ 853{
856 /* 854 /*
@@ -896,7 +894,7 @@ static void rb_reset_reader_page(struct ring_buffer_per_cpu *cpu_buffer)
896 cpu_buffer->reader_page->read = 0; 894 cpu_buffer->reader_page->read = 0;
897} 895}
898 896
899static inline void rb_inc_iter(struct ring_buffer_iter *iter) 897static void rb_inc_iter(struct ring_buffer_iter *iter)
900{ 898{
901 struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer; 899 struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
902 900
@@ -926,7 +924,7 @@ static inline void rb_inc_iter(struct ring_buffer_iter *iter)
926 * and with this, we can determine what to place into the 924 * and with this, we can determine what to place into the
927 * data field. 925 * data field.
928 */ 926 */
929static inline void 927static void
930rb_update_event(struct ring_buffer_event *event, 928rb_update_event(struct ring_buffer_event *event,
931 unsigned type, unsigned length) 929 unsigned type, unsigned length)
932{ 930{
@@ -938,15 +936,11 @@ rb_update_event(struct ring_buffer_event *event,
938 break; 936 break;
939 937
940 case RINGBUF_TYPE_TIME_EXTEND: 938 case RINGBUF_TYPE_TIME_EXTEND:
941 event->len = 939 event->len = DIV_ROUND_UP(RB_LEN_TIME_EXTEND, RB_ALIGNMENT);
942 (RB_LEN_TIME_EXTEND + (RB_ALIGNMENT-1))
943 >> RB_ALIGNMENT_SHIFT;
944 break; 940 break;
945 941
946 case RINGBUF_TYPE_TIME_STAMP: 942 case RINGBUF_TYPE_TIME_STAMP:
947 event->len = 943 event->len = DIV_ROUND_UP(RB_LEN_TIME_STAMP, RB_ALIGNMENT);
948 (RB_LEN_TIME_STAMP + (RB_ALIGNMENT-1))
949 >> RB_ALIGNMENT_SHIFT;
950 break; 944 break;
951 945
952 case RINGBUF_TYPE_DATA: 946 case RINGBUF_TYPE_DATA:
@@ -955,16 +949,14 @@ rb_update_event(struct ring_buffer_event *event,
955 event->len = 0; 949 event->len = 0;
956 event->array[0] = length; 950 event->array[0] = length;
957 } else 951 } else
958 event->len = 952 event->len = DIV_ROUND_UP(length, RB_ALIGNMENT);
959 (length + (RB_ALIGNMENT-1))
960 >> RB_ALIGNMENT_SHIFT;
961 break; 953 break;
962 default: 954 default:
963 BUG(); 955 BUG();
964 } 956 }
965} 957}
966 958
967static inline unsigned rb_calculate_event_length(unsigned length) 959static unsigned rb_calculate_event_length(unsigned length)
968{ 960{
969 struct ring_buffer_event event; /* Used only for sizeof array */ 961 struct ring_buffer_event event; /* Used only for sizeof array */
970 962
@@ -1438,7 +1430,7 @@ int ring_buffer_write(struct ring_buffer *buffer,
1438} 1430}
1439EXPORT_SYMBOL_GPL(ring_buffer_write); 1431EXPORT_SYMBOL_GPL(ring_buffer_write);
1440 1432
1441static inline int rb_per_cpu_empty(struct ring_buffer_per_cpu *cpu_buffer) 1433static int rb_per_cpu_empty(struct ring_buffer_per_cpu *cpu_buffer)
1442{ 1434{
1443 struct buffer_page *reader = cpu_buffer->reader_page; 1435 struct buffer_page *reader = cpu_buffer->reader_page;
1444 struct buffer_page *head = cpu_buffer->head_page; 1436 struct buffer_page *head = cpu_buffer->head_page;
@@ -2277,9 +2269,24 @@ int ring_buffer_swap_cpu(struct ring_buffer *buffer_a,
2277 if (buffer_a->pages != buffer_b->pages) 2269 if (buffer_a->pages != buffer_b->pages)
2278 return -EINVAL; 2270 return -EINVAL;
2279 2271
2272 if (ring_buffer_flags != RB_BUFFERS_ON)
2273 return -EAGAIN;
2274
2275 if (atomic_read(&buffer_a->record_disabled))
2276 return -EAGAIN;
2277
2278 if (atomic_read(&buffer_b->record_disabled))
2279 return -EAGAIN;
2280
2280 cpu_buffer_a = buffer_a->buffers[cpu]; 2281 cpu_buffer_a = buffer_a->buffers[cpu];
2281 cpu_buffer_b = buffer_b->buffers[cpu]; 2282 cpu_buffer_b = buffer_b->buffers[cpu];
2282 2283
2284 if (atomic_read(&cpu_buffer_a->record_disabled))
2285 return -EAGAIN;
2286
2287 if (atomic_read(&cpu_buffer_b->record_disabled))
2288 return -EAGAIN;
2289
2283 /* 2290 /*
2284 * We can't do a synchronize_sched here because this 2291 * We can't do a synchronize_sched here because this
2285 * function can be called in atomic context. 2292 * function can be called in atomic context.
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index 17bb88d86ac2..2f8ac1f008f5 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -37,6 +37,7 @@
37#include <linux/irqflags.h> 37#include <linux/irqflags.h>
38 38
39#include "trace.h" 39#include "trace.h"
40#include "trace_output.h"
40 41
41#define TRACE_BUFFER_FLAGS (RB_FL_OVERWRITE) 42#define TRACE_BUFFER_FLAGS (RB_FL_OVERWRITE)
42 43
@@ -186,9 +187,6 @@ int tracing_is_enabled(void)
186 return tracer_enabled; 187 return tracer_enabled;
187} 188}
188 189
189/* function tracing enabled */
190int ftrace_function_enabled;
191
192/* 190/*
193 * trace_buf_size is the size in bytes that is allocated 191 * trace_buf_size is the size in bytes that is allocated
194 * for a buffer. Note, the number of bytes is always rounded 192 * for a buffer. Note, the number of bytes is always rounded
@@ -329,132 +327,6 @@ __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
329 tracing_record_cmdline(current); 327 tracing_record_cmdline(current);
330} 328}
331 329
332/**
333 * trace_seq_printf - sequence printing of trace information
334 * @s: trace sequence descriptor
335 * @fmt: printf format string
336 *
337 * The tracer may use either sequence operations or its own
338 * copy to user routines. To simplify formating of a trace
339 * trace_seq_printf is used to store strings into a special
340 * buffer (@s). Then the output may be either used by
341 * the sequencer or pulled into another buffer.
342 */
343int
344trace_seq_printf(struct trace_seq *s, const char *fmt, ...)
345{
346 int len = (PAGE_SIZE - 1) - s->len;
347 va_list ap;
348 int ret;
349
350 if (!len)
351 return 0;
352
353 va_start(ap, fmt);
354 ret = vsnprintf(s->buffer + s->len, len, fmt, ap);
355 va_end(ap);
356
357 /* If we can't write it all, don't bother writing anything */
358 if (ret >= len)
359 return 0;
360
361 s->len += ret;
362
363 return len;
364}
365
366/**
367 * trace_seq_puts - trace sequence printing of simple string
368 * @s: trace sequence descriptor
369 * @str: simple string to record
370 *
371 * The tracer may use either the sequence operations or its own
372 * copy to user routines. This function records a simple string
373 * into a special buffer (@s) for later retrieval by a sequencer
374 * or other mechanism.
375 */
376static int
377trace_seq_puts(struct trace_seq *s, const char *str)
378{
379 int len = strlen(str);
380
381 if (len > ((PAGE_SIZE - 1) - s->len))
382 return 0;
383
384 memcpy(s->buffer + s->len, str, len);
385 s->len += len;
386
387 return len;
388}
389
390static int
391trace_seq_putc(struct trace_seq *s, unsigned char c)
392{
393 if (s->len >= (PAGE_SIZE - 1))
394 return 0;
395
396 s->buffer[s->len++] = c;
397
398 return 1;
399}
400
401static int
402trace_seq_putmem(struct trace_seq *s, void *mem, size_t len)
403{
404 if (len > ((PAGE_SIZE - 1) - s->len))
405 return 0;
406
407 memcpy(s->buffer + s->len, mem, len);
408 s->len += len;
409
410 return len;
411}
412
413#define MAX_MEMHEX_BYTES 8
414#define HEX_CHARS (MAX_MEMHEX_BYTES*2 + 1)
415
416static int
417trace_seq_putmem_hex(struct trace_seq *s, void *mem, size_t len)
418{
419 unsigned char hex[HEX_CHARS];
420 unsigned char *data = mem;
421 int i, j;
422
423#ifdef __BIG_ENDIAN
424 for (i = 0, j = 0; i < len; i++) {
425#else
426 for (i = len-1, j = 0; i >= 0; i--) {
427#endif
428 hex[j++] = hex_asc_hi(data[i]);
429 hex[j++] = hex_asc_lo(data[i]);
430 }
431 hex[j++] = ' ';
432
433 return trace_seq_putmem(s, hex, j);
434}
435
436static int
437trace_seq_path(struct trace_seq *s, struct path *path)
438{
439 unsigned char *p;
440
441 if (s->len >= (PAGE_SIZE - 1))
442 return 0;
443 p = d_path(path, s->buffer + s->len, PAGE_SIZE - s->len);
444 if (!IS_ERR(p)) {
445 p = mangle_path(s->buffer + s->len, p, "\n");
446 if (p) {
447 s->len = p - s->buffer;
448 return 1;
449 }
450 } else {
451 s->buffer[s->len++] = '?';
452 return 1;
453 }
454
455 return 0;
456}
457
458static void 330static void
459trace_seq_reset(struct trace_seq *s) 331trace_seq_reset(struct trace_seq *s)
460{ 332{
@@ -543,7 +415,7 @@ update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
543 415
544 ftrace_enable_cpu(); 416 ftrace_enable_cpu();
545 417
546 WARN_ON_ONCE(ret); 418 WARN_ON_ONCE(ret && ret != -EAGAIN);
547 419
548 __update_max_tr(tr, tsk, cpu); 420 __update_max_tr(tr, tsk, cpu);
549 __raw_spin_unlock(&ftrace_max_lock); 421 __raw_spin_unlock(&ftrace_max_lock);
@@ -738,13 +610,12 @@ void tracing_start(void)
738 return; 610 return;
739 611
740 spin_lock_irqsave(&tracing_start_lock, flags); 612 spin_lock_irqsave(&tracing_start_lock, flags);
741 if (--trace_stop_count) 613 if (--trace_stop_count) {
742 goto out; 614 if (trace_stop_count < 0) {
743 615 /* Someone screwed up their debugging */
744 if (trace_stop_count < 0) { 616 WARN_ON_ONCE(1);
745 /* Someone screwed up their debugging */ 617 trace_stop_count = 0;
746 WARN_ON_ONCE(1); 618 }
747 trace_stop_count = 0;
748 goto out; 619 goto out;
749 } 620 }
750 621
@@ -960,10 +831,10 @@ ftrace(struct trace_array *tr, struct trace_array_cpu *data,
960 trace_function(tr, data, ip, parent_ip, flags, pc); 831 trace_function(tr, data, ip, parent_ip, flags, pc);
961} 832}
962 833
963static void ftrace_trace_stack(struct trace_array *tr, 834static void __ftrace_trace_stack(struct trace_array *tr,
964 struct trace_array_cpu *data, 835 struct trace_array_cpu *data,
965 unsigned long flags, 836 unsigned long flags,
966 int skip, int pc) 837 int skip, int pc)
967{ 838{
968#ifdef CONFIG_STACKTRACE 839#ifdef CONFIG_STACKTRACE
969 struct ring_buffer_event *event; 840 struct ring_buffer_event *event;
@@ -971,9 +842,6 @@ static void ftrace_trace_stack(struct trace_array *tr,
971 struct stack_trace trace; 842 struct stack_trace trace;
972 unsigned long irq_flags; 843 unsigned long irq_flags;
973 844
974 if (!(trace_flags & TRACE_ITER_STACKTRACE))
975 return;
976
977 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry), 845 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
978 &irq_flags); 846 &irq_flags);
979 if (!event) 847 if (!event)
@@ -994,12 +862,23 @@ static void ftrace_trace_stack(struct trace_array *tr,
994#endif 862#endif
995} 863}
996 864
865static void ftrace_trace_stack(struct trace_array *tr,
866 struct trace_array_cpu *data,
867 unsigned long flags,
868 int skip, int pc)
869{
870 if (!(trace_flags & TRACE_ITER_STACKTRACE))
871 return;
872
873 __ftrace_trace_stack(tr, data, flags, skip, pc);
874}
875
997void __trace_stack(struct trace_array *tr, 876void __trace_stack(struct trace_array *tr,
998 struct trace_array_cpu *data, 877 struct trace_array_cpu *data,
999 unsigned long flags, 878 unsigned long flags,
1000 int skip) 879 int skip, int pc)
1001{ 880{
1002 ftrace_trace_stack(tr, data, flags, skip, preempt_count()); 881 __ftrace_trace_stack(tr, data, flags, skip, pc);
1003} 882}
1004 883
1005static void ftrace_trace_userstack(struct trace_array *tr, 884static void ftrace_trace_userstack(struct trace_array *tr,
@@ -1163,65 +1042,6 @@ ftrace_special(unsigned long arg1, unsigned long arg2, unsigned long arg3)
1163 local_irq_restore(flags); 1042 local_irq_restore(flags);
1164} 1043}
1165 1044
1166#ifdef CONFIG_FUNCTION_TRACER
1167static void
1168function_trace_call_preempt_only(unsigned long ip, unsigned long parent_ip)
1169{
1170 struct trace_array *tr = &global_trace;
1171 struct trace_array_cpu *data;
1172 unsigned long flags;
1173 long disabled;
1174 int cpu, resched;
1175 int pc;
1176
1177 if (unlikely(!ftrace_function_enabled))
1178 return;
1179
1180 pc = preempt_count();
1181 resched = ftrace_preempt_disable();
1182 local_save_flags(flags);
1183 cpu = raw_smp_processor_id();
1184 data = tr->data[cpu];
1185 disabled = atomic_inc_return(&data->disabled);
1186
1187 if (likely(disabled == 1))
1188 trace_function(tr, data, ip, parent_ip, flags, pc);
1189
1190 atomic_dec(&data->disabled);
1191 ftrace_preempt_enable(resched);
1192}
1193
1194static void
1195function_trace_call(unsigned long ip, unsigned long parent_ip)
1196{
1197 struct trace_array *tr = &global_trace;
1198 struct trace_array_cpu *data;
1199 unsigned long flags;
1200 long disabled;
1201 int cpu;
1202 int pc;
1203
1204 if (unlikely(!ftrace_function_enabled))
1205 return;
1206
1207 /*
1208 * Need to use raw, since this must be called before the
1209 * recursive protection is performed.
1210 */
1211 local_irq_save(flags);
1212 cpu = raw_smp_processor_id();
1213 data = tr->data[cpu];
1214 disabled = atomic_inc_return(&data->disabled);
1215
1216 if (likely(disabled == 1)) {
1217 pc = preempt_count();
1218 trace_function(tr, data, ip, parent_ip, flags, pc);
1219 }
1220
1221 atomic_dec(&data->disabled);
1222 local_irq_restore(flags);
1223}
1224
1225#ifdef CONFIG_FUNCTION_GRAPH_TRACER 1045#ifdef CONFIG_FUNCTION_GRAPH_TRACER
1226int trace_graph_entry(struct ftrace_graph_ent *trace) 1046int trace_graph_entry(struct ftrace_graph_ent *trace)
1227{ 1047{
@@ -1279,31 +1099,6 @@ void trace_graph_return(struct ftrace_graph_ret *trace)
1279} 1099}
1280#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 1100#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
1281 1101
1282static struct ftrace_ops trace_ops __read_mostly =
1283{
1284 .func = function_trace_call,
1285};
1286
1287void tracing_start_function_trace(void)
1288{
1289 ftrace_function_enabled = 0;
1290
1291 if (trace_flags & TRACE_ITER_PREEMPTONLY)
1292 trace_ops.func = function_trace_call_preempt_only;
1293 else
1294 trace_ops.func = function_trace_call;
1295
1296 register_ftrace_function(&trace_ops);
1297 ftrace_function_enabled = 1;
1298}
1299
1300void tracing_stop_function_trace(void)
1301{
1302 ftrace_function_enabled = 0;
1303 unregister_ftrace_function(&trace_ops);
1304}
1305#endif
1306
1307enum trace_file_type { 1102enum trace_file_type {
1308 TRACE_FILE_LAT_FMT = 1, 1103 TRACE_FILE_LAT_FMT = 1,
1309 TRACE_FILE_ANNOTATE = 2, 1104 TRACE_FILE_ANNOTATE = 2,
@@ -1472,154 +1267,6 @@ static void s_stop(struct seq_file *m, void *p)
1472 mutex_unlock(&trace_types_lock); 1267 mutex_unlock(&trace_types_lock);
1473} 1268}
1474 1269
1475#ifdef CONFIG_KRETPROBES
1476static inline const char *kretprobed(const char *name)
1477{
1478 static const char tramp_name[] = "kretprobe_trampoline";
1479 int size = sizeof(tramp_name);
1480
1481 if (strncmp(tramp_name, name, size) == 0)
1482 return "[unknown/kretprobe'd]";
1483 return name;
1484}
1485#else
1486static inline const char *kretprobed(const char *name)
1487{
1488 return name;
1489}
1490#endif /* CONFIG_KRETPROBES */
1491
1492static int
1493seq_print_sym_short(struct trace_seq *s, const char *fmt, unsigned long address)
1494{
1495#ifdef CONFIG_KALLSYMS
1496 char str[KSYM_SYMBOL_LEN];
1497 const char *name;
1498
1499 kallsyms_lookup(address, NULL, NULL, NULL, str);
1500
1501 name = kretprobed(str);
1502
1503 return trace_seq_printf(s, fmt, name);
1504#endif
1505 return 1;
1506}
1507
1508static int
1509seq_print_sym_offset(struct trace_seq *s, const char *fmt,
1510 unsigned long address)
1511{
1512#ifdef CONFIG_KALLSYMS
1513 char str[KSYM_SYMBOL_LEN];
1514 const char *name;
1515
1516 sprint_symbol(str, address);
1517 name = kretprobed(str);
1518
1519 return trace_seq_printf(s, fmt, name);
1520#endif
1521 return 1;
1522}
1523
1524#ifndef CONFIG_64BIT
1525# define IP_FMT "%08lx"
1526#else
1527# define IP_FMT "%016lx"
1528#endif
1529
1530int
1531seq_print_ip_sym(struct trace_seq *s, unsigned long ip, unsigned long sym_flags)
1532{
1533 int ret;
1534
1535 if (!ip)
1536 return trace_seq_printf(s, "0");
1537
1538 if (sym_flags & TRACE_ITER_SYM_OFFSET)
1539 ret = seq_print_sym_offset(s, "%s", ip);
1540 else
1541 ret = seq_print_sym_short(s, "%s", ip);
1542
1543 if (!ret)
1544 return 0;
1545
1546 if (sym_flags & TRACE_ITER_SYM_ADDR)
1547 ret = trace_seq_printf(s, " <" IP_FMT ">", ip);
1548 return ret;
1549}
1550
1551static inline int seq_print_user_ip(struct trace_seq *s, struct mm_struct *mm,
1552 unsigned long ip, unsigned long sym_flags)
1553{
1554 struct file *file = NULL;
1555 unsigned long vmstart = 0;
1556 int ret = 1;
1557
1558 if (mm) {
1559 const struct vm_area_struct *vma;
1560
1561 down_read(&mm->mmap_sem);
1562 vma = find_vma(mm, ip);
1563 if (vma) {
1564 file = vma->vm_file;
1565 vmstart = vma->vm_start;
1566 }
1567 if (file) {
1568 ret = trace_seq_path(s, &file->f_path);
1569 if (ret)
1570 ret = trace_seq_printf(s, "[+0x%lx]", ip - vmstart);
1571 }
1572 up_read(&mm->mmap_sem);
1573 }
1574 if (ret && ((sym_flags & TRACE_ITER_SYM_ADDR) || !file))
1575 ret = trace_seq_printf(s, " <" IP_FMT ">", ip);
1576 return ret;
1577}
1578
1579static int
1580seq_print_userip_objs(const struct userstack_entry *entry, struct trace_seq *s,
1581 unsigned long sym_flags)
1582{
1583 struct mm_struct *mm = NULL;
1584 int ret = 1;
1585 unsigned int i;
1586
1587 if (trace_flags & TRACE_ITER_SYM_USEROBJ) {
1588 struct task_struct *task;
1589 /*
1590 * we do the lookup on the thread group leader,
1591 * since individual threads might have already quit!
1592 */
1593 rcu_read_lock();
1594 task = find_task_by_vpid(entry->ent.tgid);
1595 if (task)
1596 mm = get_task_mm(task);
1597 rcu_read_unlock();
1598 }
1599
1600 for (i = 0; i < FTRACE_STACK_ENTRIES; i++) {
1601 unsigned long ip = entry->caller[i];
1602
1603 if (ip == ULONG_MAX || !ret)
1604 break;
1605 if (i && ret)
1606 ret = trace_seq_puts(s, " <- ");
1607 if (!ip) {
1608 if (ret)
1609 ret = trace_seq_puts(s, "??");
1610 continue;
1611 }
1612 if (!ret)
1613 break;
1614 if (ret)
1615 ret = seq_print_user_ip(s, mm, ip, sym_flags);
1616 }
1617
1618 if (mm)
1619 mmput(mm);
1620 return ret;
1621}
1622
1623static void print_lat_help_header(struct seq_file *m) 1270static void print_lat_help_header(struct seq_file *m)
1624{ 1271{
1625 seq_puts(m, "# _------=> CPU# \n"); 1272 seq_puts(m, "# _------=> CPU# \n");
@@ -1755,52 +1402,6 @@ lat_print_timestamp(struct trace_seq *s, u64 abs_usecs,
1755 trace_seq_puts(s, " : "); 1402 trace_seq_puts(s, " : ");
1756} 1403}
1757 1404
1758static const char state_to_char[] = TASK_STATE_TO_CHAR_STR;
1759
1760static int task_state_char(unsigned long state)
1761{
1762 int bit = state ? __ffs(state) + 1 : 0;
1763
1764 return bit < sizeof(state_to_char) - 1 ? state_to_char[bit] : '?';
1765}
1766
1767/*
1768 * The message is supposed to contain an ending newline.
1769 * If the printing stops prematurely, try to add a newline of our own.
1770 */
1771void trace_seq_print_cont(struct trace_seq *s, struct trace_iterator *iter)
1772{
1773 struct trace_entry *ent;
1774 struct trace_field_cont *cont;
1775 bool ok = true;
1776
1777 ent = peek_next_entry(iter, iter->cpu, NULL);
1778 if (!ent || ent->type != TRACE_CONT) {
1779 trace_seq_putc(s, '\n');
1780 return;
1781 }
1782
1783 do {
1784 cont = (struct trace_field_cont *)ent;
1785 if (ok)
1786 ok = (trace_seq_printf(s, "%s", cont->buf) > 0);
1787
1788 ftrace_disable_cpu();
1789
1790 if (iter->buffer_iter[iter->cpu])
1791 ring_buffer_read(iter->buffer_iter[iter->cpu], NULL);
1792 else
1793 ring_buffer_consume(iter->tr->buffer, iter->cpu, NULL);
1794
1795 ftrace_enable_cpu();
1796
1797 ent = peek_next_entry(iter, iter->cpu, NULL);
1798 } while (ent && ent->type == TRACE_CONT);
1799
1800 if (!ok)
1801 trace_seq_putc(s, '\n');
1802}
1803
1804static void test_cpu_buff_start(struct trace_iterator *iter) 1405static void test_cpu_buff_start(struct trace_iterator *iter)
1805{ 1406{
1806 struct trace_seq *s = &iter->seq; 1407 struct trace_seq *s = &iter->seq;
@@ -1824,17 +1425,14 @@ print_lat_fmt(struct trace_iterator *iter, unsigned int trace_idx, int cpu)
1824 struct trace_seq *s = &iter->seq; 1425 struct trace_seq *s = &iter->seq;
1825 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK); 1426 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1826 struct trace_entry *next_entry; 1427 struct trace_entry *next_entry;
1428 struct trace_event *event;
1827 unsigned long verbose = (trace_flags & TRACE_ITER_VERBOSE); 1429 unsigned long verbose = (trace_flags & TRACE_ITER_VERBOSE);
1828 struct trace_entry *entry = iter->ent; 1430 struct trace_entry *entry = iter->ent;
1829 unsigned long abs_usecs; 1431 unsigned long abs_usecs;
1830 unsigned long rel_usecs; 1432 unsigned long rel_usecs;
1831 u64 next_ts; 1433 u64 next_ts;
1832 char *comm; 1434 char *comm;
1833 int S, T; 1435 int ret;
1834 int i;
1835
1836 if (entry->type == TRACE_CONT)
1837 return TRACE_TYPE_HANDLED;
1838 1436
1839 test_cpu_buff_start(iter); 1437 test_cpu_buff_start(iter);
1840 1438
@@ -1859,96 +1457,16 @@ print_lat_fmt(struct trace_iterator *iter, unsigned int trace_idx, int cpu)
1859 lat_print_generic(s, entry, cpu); 1457 lat_print_generic(s, entry, cpu);
1860 lat_print_timestamp(s, abs_usecs, rel_usecs); 1458 lat_print_timestamp(s, abs_usecs, rel_usecs);
1861 } 1459 }
1862 switch (entry->type) {
1863 case TRACE_FN: {
1864 struct ftrace_entry *field;
1865
1866 trace_assign_type(field, entry);
1867
1868 seq_print_ip_sym(s, field->ip, sym_flags);
1869 trace_seq_puts(s, " (");
1870 seq_print_ip_sym(s, field->parent_ip, sym_flags);
1871 trace_seq_puts(s, ")\n");
1872 break;
1873 }
1874 case TRACE_CTX:
1875 case TRACE_WAKE: {
1876 struct ctx_switch_entry *field;
1877
1878 trace_assign_type(field, entry);
1879
1880 T = task_state_char(field->next_state);
1881 S = task_state_char(field->prev_state);
1882 comm = trace_find_cmdline(field->next_pid);
1883 trace_seq_printf(s, " %5d:%3d:%c %s [%03d] %5d:%3d:%c %s\n",
1884 field->prev_pid,
1885 field->prev_prio,
1886 S, entry->type == TRACE_CTX ? "==>" : " +",
1887 field->next_cpu,
1888 field->next_pid,
1889 field->next_prio,
1890 T, comm);
1891 break;
1892 }
1893 case TRACE_SPECIAL: {
1894 struct special_entry *field;
1895
1896 trace_assign_type(field, entry);
1897
1898 trace_seq_printf(s, "# %ld %ld %ld\n",
1899 field->arg1,
1900 field->arg2,
1901 field->arg3);
1902 break;
1903 }
1904 case TRACE_STACK: {
1905 struct stack_entry *field;
1906
1907 trace_assign_type(field, entry);
1908
1909 for (i = 0; i < FTRACE_STACK_ENTRIES; i++) {
1910 if (i)
1911 trace_seq_puts(s, " <= ");
1912 seq_print_ip_sym(s, field->caller[i], sym_flags);
1913 }
1914 trace_seq_puts(s, "\n");
1915 break;
1916 }
1917 case TRACE_PRINT: {
1918 struct print_entry *field;
1919
1920 trace_assign_type(field, entry);
1921 1460
1922 seq_print_ip_sym(s, field->ip, sym_flags); 1461 event = ftrace_find_event(entry->type);
1923 trace_seq_printf(s, ": %s", field->buf); 1462 if (event && event->latency_trace) {
1924 if (entry->flags & TRACE_FLAG_CONT) 1463 ret = event->latency_trace(s, entry, sym_flags);
1925 trace_seq_print_cont(s, iter); 1464 if (ret)
1926 break; 1465 return ret;
1927 } 1466 return TRACE_TYPE_HANDLED;
1928 case TRACE_BRANCH: {
1929 struct trace_branch *field;
1930
1931 trace_assign_type(field, entry);
1932
1933 trace_seq_printf(s, "[%s] %s:%s:%d\n",
1934 field->correct ? " ok " : " MISS ",
1935 field->func,
1936 field->file,
1937 field->line);
1938 break;
1939 } 1467 }
1940 case TRACE_USER_STACK: {
1941 struct userstack_entry *field;
1942
1943 trace_assign_type(field, entry);
1944 1468
1945 seq_print_userip_objs(field, s, sym_flags); 1469 trace_seq_printf(s, "Unknown type %d\n", entry->type);
1946 trace_seq_putc(s, '\n');
1947 break;
1948 }
1949 default:
1950 trace_seq_printf(s, "Unknown type %d\n", entry->type);
1951 }
1952 return TRACE_TYPE_HANDLED; 1470 return TRACE_TYPE_HANDLED;
1953} 1471}
1954 1472
@@ -1957,19 +1475,15 @@ static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
1957 struct trace_seq *s = &iter->seq; 1475 struct trace_seq *s = &iter->seq;
1958 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK); 1476 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1959 struct trace_entry *entry; 1477 struct trace_entry *entry;
1478 struct trace_event *event;
1960 unsigned long usec_rem; 1479 unsigned long usec_rem;
1961 unsigned long long t; 1480 unsigned long long t;
1962 unsigned long secs; 1481 unsigned long secs;
1963 char *comm; 1482 char *comm;
1964 int ret; 1483 int ret;
1965 int S, T;
1966 int i;
1967 1484
1968 entry = iter->ent; 1485 entry = iter->ent;
1969 1486
1970 if (entry->type == TRACE_CONT)
1971 return TRACE_TYPE_HANDLED;
1972
1973 test_cpu_buff_start(iter); 1487 test_cpu_buff_start(iter);
1974 1488
1975 comm = trace_find_cmdline(iter->ent->pid); 1489 comm = trace_find_cmdline(iter->ent->pid);
@@ -1988,129 +1502,17 @@ static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
1988 if (!ret) 1502 if (!ret)
1989 return TRACE_TYPE_PARTIAL_LINE; 1503 return TRACE_TYPE_PARTIAL_LINE;
1990 1504
1991 switch (entry->type) { 1505 event = ftrace_find_event(entry->type);
1992 case TRACE_FN: { 1506 if (event && event->trace) {
1993 struct ftrace_entry *field; 1507 ret = event->trace(s, entry, sym_flags);
1994 1508 if (ret)
1995 trace_assign_type(field, entry); 1509 return ret;
1996 1510 return TRACE_TYPE_HANDLED;
1997 ret = seq_print_ip_sym(s, field->ip, sym_flags);
1998 if (!ret)
1999 return TRACE_TYPE_PARTIAL_LINE;
2000 if ((sym_flags & TRACE_ITER_PRINT_PARENT) &&
2001 field->parent_ip) {
2002 ret = trace_seq_printf(s, " <-");
2003 if (!ret)
2004 return TRACE_TYPE_PARTIAL_LINE;
2005 ret = seq_print_ip_sym(s,
2006 field->parent_ip,
2007 sym_flags);
2008 if (!ret)
2009 return TRACE_TYPE_PARTIAL_LINE;
2010 }
2011 ret = trace_seq_printf(s, "\n");
2012 if (!ret)
2013 return TRACE_TYPE_PARTIAL_LINE;
2014 break;
2015 }
2016 case TRACE_CTX:
2017 case TRACE_WAKE: {
2018 struct ctx_switch_entry *field;
2019
2020 trace_assign_type(field, entry);
2021
2022 T = task_state_char(field->next_state);
2023 S = task_state_char(field->prev_state);
2024 ret = trace_seq_printf(s, " %5d:%3d:%c %s [%03d] %5d:%3d:%c\n",
2025 field->prev_pid,
2026 field->prev_prio,
2027 S,
2028 entry->type == TRACE_CTX ? "==>" : " +",
2029 field->next_cpu,
2030 field->next_pid,
2031 field->next_prio,
2032 T);
2033 if (!ret)
2034 return TRACE_TYPE_PARTIAL_LINE;
2035 break;
2036 }
2037 case TRACE_SPECIAL: {
2038 struct special_entry *field;
2039
2040 trace_assign_type(field, entry);
2041
2042 ret = trace_seq_printf(s, "# %ld %ld %ld\n",
2043 field->arg1,
2044 field->arg2,
2045 field->arg3);
2046 if (!ret)
2047 return TRACE_TYPE_PARTIAL_LINE;
2048 break;
2049 }
2050 case TRACE_STACK: {
2051 struct stack_entry *field;
2052
2053 trace_assign_type(field, entry);
2054
2055 for (i = 0; i < FTRACE_STACK_ENTRIES; i++) {
2056 if (i) {
2057 ret = trace_seq_puts(s, " <= ");
2058 if (!ret)
2059 return TRACE_TYPE_PARTIAL_LINE;
2060 }
2061 ret = seq_print_ip_sym(s, field->caller[i],
2062 sym_flags);
2063 if (!ret)
2064 return TRACE_TYPE_PARTIAL_LINE;
2065 }
2066 ret = trace_seq_puts(s, "\n");
2067 if (!ret)
2068 return TRACE_TYPE_PARTIAL_LINE;
2069 break;
2070 }
2071 case TRACE_PRINT: {
2072 struct print_entry *field;
2073
2074 trace_assign_type(field, entry);
2075
2076 seq_print_ip_sym(s, field->ip, sym_flags);
2077 trace_seq_printf(s, ": %s", field->buf);
2078 if (entry->flags & TRACE_FLAG_CONT)
2079 trace_seq_print_cont(s, iter);
2080 break;
2081 }
2082 case TRACE_GRAPH_RET: {
2083 return print_graph_function(iter);
2084 }
2085 case TRACE_GRAPH_ENT: {
2086 return print_graph_function(iter);
2087 }
2088 case TRACE_BRANCH: {
2089 struct trace_branch *field;
2090
2091 trace_assign_type(field, entry);
2092
2093 trace_seq_printf(s, "[%s] %s:%s:%d\n",
2094 field->correct ? " ok " : " MISS ",
2095 field->func,
2096 field->file,
2097 field->line);
2098 break;
2099 } 1511 }
2100 case TRACE_USER_STACK: { 1512 ret = trace_seq_printf(s, "Unknown type %d\n", entry->type);
2101 struct userstack_entry *field; 1513 if (!ret)
2102 1514 return TRACE_TYPE_PARTIAL_LINE;
2103 trace_assign_type(field, entry);
2104 1515
2105 ret = seq_print_userip_objs(field, s, sym_flags);
2106 if (!ret)
2107 return TRACE_TYPE_PARTIAL_LINE;
2108 ret = trace_seq_putc(s, '\n');
2109 if (!ret)
2110 return TRACE_TYPE_PARTIAL_LINE;
2111 break;
2112 }
2113 }
2114 return TRACE_TYPE_HANDLED; 1516 return TRACE_TYPE_HANDLED;
2115} 1517}
2116 1518
@@ -2118,152 +1520,47 @@ static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
2118{ 1520{
2119 struct trace_seq *s = &iter->seq; 1521 struct trace_seq *s = &iter->seq;
2120 struct trace_entry *entry; 1522 struct trace_entry *entry;
1523 struct trace_event *event;
2121 int ret; 1524 int ret;
2122 int S, T;
2123 1525
2124 entry = iter->ent; 1526 entry = iter->ent;
2125 1527
2126 if (entry->type == TRACE_CONT)
2127 return TRACE_TYPE_HANDLED;
2128
2129 ret = trace_seq_printf(s, "%d %d %llu ", 1528 ret = trace_seq_printf(s, "%d %d %llu ",
2130 entry->pid, iter->cpu, iter->ts); 1529 entry->pid, iter->cpu, iter->ts);
2131 if (!ret) 1530 if (!ret)
2132 return TRACE_TYPE_PARTIAL_LINE; 1531 return TRACE_TYPE_PARTIAL_LINE;
2133 1532
2134 switch (entry->type) { 1533 event = ftrace_find_event(entry->type);
2135 case TRACE_FN: { 1534 if (event && event->raw) {
2136 struct ftrace_entry *field; 1535 ret = event->raw(s, entry, 0);
2137 1536 if (ret)
2138 trace_assign_type(field, entry); 1537 return ret;
2139 1538 return TRACE_TYPE_HANDLED;
2140 ret = trace_seq_printf(s, "%x %x\n",
2141 field->ip,
2142 field->parent_ip);
2143 if (!ret)
2144 return TRACE_TYPE_PARTIAL_LINE;
2145 break;
2146 }
2147 case TRACE_CTX:
2148 case TRACE_WAKE: {
2149 struct ctx_switch_entry *field;
2150
2151 trace_assign_type(field, entry);
2152
2153 T = task_state_char(field->next_state);
2154 S = entry->type == TRACE_WAKE ? '+' :
2155 task_state_char(field->prev_state);
2156 ret = trace_seq_printf(s, "%d %d %c %d %d %d %c\n",
2157 field->prev_pid,
2158 field->prev_prio,
2159 S,
2160 field->next_cpu,
2161 field->next_pid,
2162 field->next_prio,
2163 T);
2164 if (!ret)
2165 return TRACE_TYPE_PARTIAL_LINE;
2166 break;
2167 }
2168 case TRACE_SPECIAL:
2169 case TRACE_USER_STACK:
2170 case TRACE_STACK: {
2171 struct special_entry *field;
2172
2173 trace_assign_type(field, entry);
2174
2175 ret = trace_seq_printf(s, "# %ld %ld %ld\n",
2176 field->arg1,
2177 field->arg2,
2178 field->arg3);
2179 if (!ret)
2180 return TRACE_TYPE_PARTIAL_LINE;
2181 break;
2182 } 1539 }
2183 case TRACE_PRINT: { 1540 ret = trace_seq_printf(s, "%d ?\n", entry->type);
2184 struct print_entry *field; 1541 if (!ret)
2185 1542 return TRACE_TYPE_PARTIAL_LINE;
2186 trace_assign_type(field, entry);
2187 1543
2188 trace_seq_printf(s, "# %lx %s", field->ip, field->buf);
2189 if (entry->flags & TRACE_FLAG_CONT)
2190 trace_seq_print_cont(s, iter);
2191 break;
2192 }
2193 }
2194 return TRACE_TYPE_HANDLED; 1544 return TRACE_TYPE_HANDLED;
2195} 1545}
2196 1546
2197#define SEQ_PUT_FIELD_RET(s, x) \
2198do { \
2199 if (!trace_seq_putmem(s, &(x), sizeof(x))) \
2200 return 0; \
2201} while (0)
2202
2203#define SEQ_PUT_HEX_FIELD_RET(s, x) \
2204do { \
2205 BUILD_BUG_ON(sizeof(x) > MAX_MEMHEX_BYTES); \
2206 if (!trace_seq_putmem_hex(s, &(x), sizeof(x))) \
2207 return 0; \
2208} while (0)
2209
2210static enum print_line_t print_hex_fmt(struct trace_iterator *iter) 1547static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
2211{ 1548{
2212 struct trace_seq *s = &iter->seq; 1549 struct trace_seq *s = &iter->seq;
2213 unsigned char newline = '\n'; 1550 unsigned char newline = '\n';
2214 struct trace_entry *entry; 1551 struct trace_entry *entry;
2215 int S, T; 1552 struct trace_event *event;
2216 1553
2217 entry = iter->ent; 1554 entry = iter->ent;
2218 1555
2219 if (entry->type == TRACE_CONT)
2220 return TRACE_TYPE_HANDLED;
2221
2222 SEQ_PUT_HEX_FIELD_RET(s, entry->pid); 1556 SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
2223 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu); 1557 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
2224 SEQ_PUT_HEX_FIELD_RET(s, iter->ts); 1558 SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
2225 1559
2226 switch (entry->type) { 1560 event = ftrace_find_event(entry->type);
2227 case TRACE_FN: { 1561 if (event && event->hex)
2228 struct ftrace_entry *field; 1562 event->hex(s, entry, 0);
2229
2230 trace_assign_type(field, entry);
2231 1563
2232 SEQ_PUT_HEX_FIELD_RET(s, field->ip);
2233 SEQ_PUT_HEX_FIELD_RET(s, field->parent_ip);
2234 break;
2235 }
2236 case TRACE_CTX:
2237 case TRACE_WAKE: {
2238 struct ctx_switch_entry *field;
2239
2240 trace_assign_type(field, entry);
2241
2242 T = task_state_char(field->next_state);
2243 S = entry->type == TRACE_WAKE ? '+' :
2244 task_state_char(field->prev_state);
2245 SEQ_PUT_HEX_FIELD_RET(s, field->prev_pid);
2246 SEQ_PUT_HEX_FIELD_RET(s, field->prev_prio);
2247 SEQ_PUT_HEX_FIELD_RET(s, S);
2248 SEQ_PUT_HEX_FIELD_RET(s, field->next_cpu);
2249 SEQ_PUT_HEX_FIELD_RET(s, field->next_pid);
2250 SEQ_PUT_HEX_FIELD_RET(s, field->next_prio);
2251 SEQ_PUT_HEX_FIELD_RET(s, T);
2252 break;
2253 }
2254 case TRACE_SPECIAL:
2255 case TRACE_USER_STACK:
2256 case TRACE_STACK: {
2257 struct special_entry *field;
2258
2259 trace_assign_type(field, entry);
2260
2261 SEQ_PUT_HEX_FIELD_RET(s, field->arg1);
2262 SEQ_PUT_HEX_FIELD_RET(s, field->arg2);
2263 SEQ_PUT_HEX_FIELD_RET(s, field->arg3);
2264 break;
2265 }
2266 }
2267 SEQ_PUT_FIELD_RET(s, newline); 1564 SEQ_PUT_FIELD_RET(s, newline);
2268 1565
2269 return TRACE_TYPE_HANDLED; 1566 return TRACE_TYPE_HANDLED;
@@ -2282,9 +1579,6 @@ static enum print_line_t print_printk_msg_only(struct trace_iterator *iter)
2282 if (!ret) 1579 if (!ret)
2283 return TRACE_TYPE_PARTIAL_LINE; 1580 return TRACE_TYPE_PARTIAL_LINE;
2284 1581
2285 if (entry->flags & TRACE_FLAG_CONT)
2286 trace_seq_print_cont(s, iter);
2287
2288 return TRACE_TYPE_HANDLED; 1582 return TRACE_TYPE_HANDLED;
2289} 1583}
2290 1584
@@ -2292,53 +1586,19 @@ static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
2292{ 1586{
2293 struct trace_seq *s = &iter->seq; 1587 struct trace_seq *s = &iter->seq;
2294 struct trace_entry *entry; 1588 struct trace_entry *entry;
1589 struct trace_event *event;
2295 1590
2296 entry = iter->ent; 1591 entry = iter->ent;
2297 1592
2298 if (entry->type == TRACE_CONT)
2299 return TRACE_TYPE_HANDLED;
2300
2301 SEQ_PUT_FIELD_RET(s, entry->pid); 1593 SEQ_PUT_FIELD_RET(s, entry->pid);
2302 SEQ_PUT_FIELD_RET(s, entry->cpu); 1594 SEQ_PUT_FIELD_RET(s, entry->cpu);
2303 SEQ_PUT_FIELD_RET(s, iter->ts); 1595 SEQ_PUT_FIELD_RET(s, iter->ts);
2304 1596
2305 switch (entry->type) { 1597 event = ftrace_find_event(entry->type);
2306 case TRACE_FN: { 1598 if (event && event->binary)
2307 struct ftrace_entry *field; 1599 event->binary(s, entry, 0);
2308
2309 trace_assign_type(field, entry);
2310
2311 SEQ_PUT_FIELD_RET(s, field->ip);
2312 SEQ_PUT_FIELD_RET(s, field->parent_ip);
2313 break;
2314 }
2315 case TRACE_CTX: {
2316 struct ctx_switch_entry *field;
2317
2318 trace_assign_type(field, entry);
2319 1600
2320 SEQ_PUT_FIELD_RET(s, field->prev_pid); 1601 return TRACE_TYPE_HANDLED;
2321 SEQ_PUT_FIELD_RET(s, field->prev_prio);
2322 SEQ_PUT_FIELD_RET(s, field->prev_state);
2323 SEQ_PUT_FIELD_RET(s, field->next_pid);
2324 SEQ_PUT_FIELD_RET(s, field->next_prio);
2325 SEQ_PUT_FIELD_RET(s, field->next_state);
2326 break;
2327 }
2328 case TRACE_SPECIAL:
2329 case TRACE_USER_STACK:
2330 case TRACE_STACK: {
2331 struct special_entry *field;
2332
2333 trace_assign_type(field, entry);
2334
2335 SEQ_PUT_FIELD_RET(s, field->arg1);
2336 SEQ_PUT_FIELD_RET(s, field->arg2);
2337 SEQ_PUT_FIELD_RET(s, field->arg3);
2338 break;
2339 }
2340 }
2341 return 1;
2342} 1602}
2343 1603
2344static int trace_empty(struct trace_iterator *iter) 1604static int trace_empty(struct trace_iterator *iter)
@@ -3691,6 +2951,15 @@ int __ftrace_printk(unsigned long ip, const char *fmt, ...)
3691} 2951}
3692EXPORT_SYMBOL_GPL(__ftrace_printk); 2952EXPORT_SYMBOL_GPL(__ftrace_printk);
3693 2953
2954int __ftrace_vprintk(unsigned long ip, const char *fmt, va_list ap)
2955{
2956 if (!(trace_flags & TRACE_ITER_PRINTK))
2957 return 0;
2958
2959 return trace_vprintk(ip, task_curr_ret_stack(current), fmt, ap);
2960}
2961EXPORT_SYMBOL_GPL(__ftrace_vprintk);
2962
3694static int trace_panic_handler(struct notifier_block *this, 2963static int trace_panic_handler(struct notifier_block *this,
3695 unsigned long event, void *unused) 2964 unsigned long event, void *unused)
3696{ 2965{
@@ -3878,7 +3147,6 @@ __init static int tracer_alloc_buffers(void)
3878#else 3147#else
3879 current_trace = &nop_trace; 3148 current_trace = &nop_trace;
3880#endif 3149#endif
3881
3882 /* All seems OK, enable tracing */ 3150 /* All seems OK, enable tracing */
3883 tracing_disabled = 0; 3151 tracing_disabled = 0;
3884 3152
diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h
index 4d3d381bfd95..e603a291134b 100644
--- a/kernel/trace/trace.h
+++ b/kernel/trace/trace.h
@@ -9,6 +9,7 @@
9#include <linux/mmiotrace.h> 9#include <linux/mmiotrace.h>
10#include <linux/ftrace.h> 10#include <linux/ftrace.h>
11#include <trace/boot.h> 11#include <trace/boot.h>
12#include <trace/kmemtrace.h>
12 13
13enum trace_type { 14enum trace_type {
14 __TRACE_FIRST_TYPE = 0, 15 __TRACE_FIRST_TYPE = 0,
@@ -16,7 +17,6 @@ enum trace_type {
16 TRACE_FN, 17 TRACE_FN,
17 TRACE_CTX, 18 TRACE_CTX,
18 TRACE_WAKE, 19 TRACE_WAKE,
19 TRACE_CONT,
20 TRACE_STACK, 20 TRACE_STACK,
21 TRACE_PRINT, 21 TRACE_PRINT,
22 TRACE_SPECIAL, 22 TRACE_SPECIAL,
@@ -29,9 +29,12 @@ enum trace_type {
29 TRACE_GRAPH_ENT, 29 TRACE_GRAPH_ENT,
30 TRACE_USER_STACK, 30 TRACE_USER_STACK,
31 TRACE_HW_BRANCHES, 31 TRACE_HW_BRANCHES,
32 TRACE_KMEM_ALLOC,
33 TRACE_KMEM_FREE,
32 TRACE_POWER, 34 TRACE_POWER,
35 TRACE_BLK,
33 36
34 __TRACE_LAST_TYPE 37 __TRACE_LAST_TYPE,
35}; 38};
36 39
37/* 40/*
@@ -170,6 +173,24 @@ struct trace_power {
170 struct power_trace state_data; 173 struct power_trace state_data;
171}; 174};
172 175
176struct kmemtrace_alloc_entry {
177 struct trace_entry ent;
178 enum kmemtrace_type_id type_id;
179 unsigned long call_site;
180 const void *ptr;
181 size_t bytes_req;
182 size_t bytes_alloc;
183 gfp_t gfp_flags;
184 int node;
185};
186
187struct kmemtrace_free_entry {
188 struct trace_entry ent;
189 enum kmemtrace_type_id type_id;
190 unsigned long call_site;
191 const void *ptr;
192};
193
173/* 194/*
174 * trace_flag_type is an enumeration that holds different 195 * trace_flag_type is an enumeration that holds different
175 * states when a trace occurs. These are: 196 * states when a trace occurs. These are:
@@ -178,7 +199,6 @@ struct trace_power {
178 * NEED_RESCED - reschedule is requested 199 * NEED_RESCED - reschedule is requested
179 * HARDIRQ - inside an interrupt handler 200 * HARDIRQ - inside an interrupt handler
180 * SOFTIRQ - inside a softirq handler 201 * SOFTIRQ - inside a softirq handler
181 * CONT - multiple entries hold the trace item
182 */ 202 */
183enum trace_flag_type { 203enum trace_flag_type {
184 TRACE_FLAG_IRQS_OFF = 0x01, 204 TRACE_FLAG_IRQS_OFF = 0x01,
@@ -186,7 +206,6 @@ enum trace_flag_type {
186 TRACE_FLAG_NEED_RESCHED = 0x04, 206 TRACE_FLAG_NEED_RESCHED = 0x04,
187 TRACE_FLAG_HARDIRQ = 0x08, 207 TRACE_FLAG_HARDIRQ = 0x08,
188 TRACE_FLAG_SOFTIRQ = 0x10, 208 TRACE_FLAG_SOFTIRQ = 0x10,
189 TRACE_FLAG_CONT = 0x20,
190}; 209};
191 210
192#define TRACE_BUF_SIZE 1024 211#define TRACE_BUF_SIZE 1024
@@ -262,7 +281,6 @@ extern void __ftrace_bad_type(void);
262 do { \ 281 do { \
263 IF_ASSIGN(var, ent, struct ftrace_entry, TRACE_FN); \ 282 IF_ASSIGN(var, ent, struct ftrace_entry, TRACE_FN); \
264 IF_ASSIGN(var, ent, struct ctx_switch_entry, 0); \ 283 IF_ASSIGN(var, ent, struct ctx_switch_entry, 0); \
265 IF_ASSIGN(var, ent, struct trace_field_cont, TRACE_CONT); \
266 IF_ASSIGN(var, ent, struct stack_entry, TRACE_STACK); \ 284 IF_ASSIGN(var, ent, struct stack_entry, TRACE_STACK); \
267 IF_ASSIGN(var, ent, struct userstack_entry, TRACE_USER_STACK);\ 285 IF_ASSIGN(var, ent, struct userstack_entry, TRACE_USER_STACK);\
268 IF_ASSIGN(var, ent, struct print_entry, TRACE_PRINT); \ 286 IF_ASSIGN(var, ent, struct print_entry, TRACE_PRINT); \
@@ -280,6 +298,10 @@ extern void __ftrace_bad_type(void);
280 TRACE_GRAPH_RET); \ 298 TRACE_GRAPH_RET); \
281 IF_ASSIGN(var, ent, struct hw_branch_entry, TRACE_HW_BRANCHES);\ 299 IF_ASSIGN(var, ent, struct hw_branch_entry, TRACE_HW_BRANCHES);\
282 IF_ASSIGN(var, ent, struct trace_power, TRACE_POWER); \ 300 IF_ASSIGN(var, ent, struct trace_power, TRACE_POWER); \
301 IF_ASSIGN(var, ent, struct kmemtrace_alloc_entry, \
302 TRACE_KMEM_ALLOC); \
303 IF_ASSIGN(var, ent, struct kmemtrace_free_entry, \
304 TRACE_KMEM_FREE); \
283 __ftrace_bad_type(); \ 305 __ftrace_bad_type(); \
284 } while (0) 306 } while (0)
285 307
@@ -313,6 +335,7 @@ struct tracer_flags {
313/* Makes more easy to define a tracer opt */ 335/* Makes more easy to define a tracer opt */
314#define TRACER_OPT(s, b) .name = #s, .bit = b 336#define TRACER_OPT(s, b) .name = #s, .bit = b
315 337
338
316/* 339/*
317 * A specific tracer, represented by methods that operate on a trace array: 340 * A specific tracer, represented by methods that operate on a trace array:
318 */ 341 */
@@ -340,6 +363,7 @@ struct tracer {
340 struct tracer *next; 363 struct tracer *next;
341 int print_max; 364 int print_max;
342 struct tracer_flags *flags; 365 struct tracer_flags *flags;
366 struct tracer_stat *stats;
343}; 367};
344 368
345struct trace_seq { 369struct trace_seq {
@@ -415,7 +439,6 @@ void trace_function(struct trace_array *tr,
415 439
416void trace_graph_return(struct ftrace_graph_ret *trace); 440void trace_graph_return(struct ftrace_graph_ret *trace);
417int trace_graph_entry(struct ftrace_graph_ent *trace); 441int trace_graph_entry(struct ftrace_graph_ent *trace);
418void trace_hw_branch(struct trace_array *tr, u64 from, u64 to);
419 442
420void tracing_start_cmdline_record(void); 443void tracing_start_cmdline_record(void);
421void tracing_stop_cmdline_record(void); 444void tracing_stop_cmdline_record(void);
@@ -434,15 +457,12 @@ void update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu);
434void update_max_tr_single(struct trace_array *tr, 457void update_max_tr_single(struct trace_array *tr,
435 struct task_struct *tsk, int cpu); 458 struct task_struct *tsk, int cpu);
436 459
437extern cycle_t ftrace_now(int cpu); 460void __trace_stack(struct trace_array *tr,
461 struct trace_array_cpu *data,
462 unsigned long flags,
463 int skip, int pc);
438 464
439#ifdef CONFIG_FUNCTION_TRACER 465extern cycle_t ftrace_now(int cpu);
440void tracing_start_function_trace(void);
441void tracing_stop_function_trace(void);
442#else
443# define tracing_start_function_trace() do { } while (0)
444# define tracing_stop_function_trace() do { } while (0)
445#endif
446 466
447#ifdef CONFIG_CONTEXT_SWITCH_TRACER 467#ifdef CONFIG_CONTEXT_SWITCH_TRACER
448typedef void 468typedef void
@@ -456,10 +476,10 @@ struct tracer_switch_ops {
456 void *private; 476 void *private;
457 struct tracer_switch_ops *next; 477 struct tracer_switch_ops *next;
458}; 478};
459
460char *trace_find_cmdline(int pid);
461#endif /* CONFIG_CONTEXT_SWITCH_TRACER */ 479#endif /* CONFIG_CONTEXT_SWITCH_TRACER */
462 480
481extern char *trace_find_cmdline(int pid);
482
463#ifdef CONFIG_DYNAMIC_FTRACE 483#ifdef CONFIG_DYNAMIC_FTRACE
464extern unsigned long ftrace_update_tot_cnt; 484extern unsigned long ftrace_update_tot_cnt;
465#define DYN_FTRACE_TEST_NAME trace_selftest_dynamic_test_func 485#define DYN_FTRACE_TEST_NAME trace_selftest_dynamic_test_func
@@ -488,15 +508,6 @@ extern int trace_selftest_startup_branch(struct tracer *trace,
488#endif /* CONFIG_FTRACE_STARTUP_TEST */ 508#endif /* CONFIG_FTRACE_STARTUP_TEST */
489 509
490extern void *head_page(struct trace_array_cpu *data); 510extern void *head_page(struct trace_array_cpu *data);
491extern int trace_seq_printf(struct trace_seq *s, const char *fmt, ...);
492extern void trace_seq_print_cont(struct trace_seq *s,
493 struct trace_iterator *iter);
494
495extern int
496seq_print_ip_sym(struct trace_seq *s, unsigned long ip,
497 unsigned long sym_flags);
498extern ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf,
499 size_t cnt);
500extern long ns2usecs(cycle_t nsec); 511extern long ns2usecs(cycle_t nsec);
501extern int 512extern int
502trace_vprintk(unsigned long ip, int depth, const char *fmt, va_list args); 513trace_vprintk(unsigned long ip, int depth, const char *fmt, va_list args);
diff --git a/kernel/trace/trace_boot.c b/kernel/trace/trace_boot.c
index 366c8c333e13..0e94b3d091f7 100644
--- a/kernel/trace/trace_boot.c
+++ b/kernel/trace/trace_boot.c
@@ -11,6 +11,7 @@
11#include <linux/kallsyms.h> 11#include <linux/kallsyms.h>
12 12
13#include "trace.h" 13#include "trace.h"
14#include "trace_output.h"
14 15
15static struct trace_array *boot_trace; 16static struct trace_array *boot_trace;
16static bool pre_initcalls_finished; 17static bool pre_initcalls_finished;
diff --git a/kernel/trace/trace_branch.c b/kernel/trace/trace_branch.c
index 6c00feb3bac7..1284145c8898 100644
--- a/kernel/trace/trace_branch.c
+++ b/kernel/trace/trace_branch.c
@@ -14,12 +14,17 @@
14#include <linux/hash.h> 14#include <linux/hash.h>
15#include <linux/fs.h> 15#include <linux/fs.h>
16#include <asm/local.h> 16#include <asm/local.h>
17
17#include "trace.h" 18#include "trace.h"
19#include "trace_stat.h"
20#include "trace_output.h"
18 21
19#ifdef CONFIG_BRANCH_TRACER 22#ifdef CONFIG_BRANCH_TRACER
20 23
24static struct tracer branch_trace;
21static int branch_tracing_enabled __read_mostly; 25static int branch_tracing_enabled __read_mostly;
22static DEFINE_MUTEX(branch_tracing_mutex); 26static DEFINE_MUTEX(branch_tracing_mutex);
27
23static struct trace_array *branch_tracer; 28static struct trace_array *branch_tracer;
24 29
25static void 30static void
@@ -128,11 +133,7 @@ static void stop_branch_trace(struct trace_array *tr)
128 133
129static int branch_trace_init(struct trace_array *tr) 134static int branch_trace_init(struct trace_array *tr)
130{ 135{
131 int cpu; 136 tracing_reset_online_cpus(tr);
132
133 for_each_online_cpu(cpu)
134 tracing_reset(tr, cpu);
135
136 start_branch_trace(tr); 137 start_branch_trace(tr);
137 return 0; 138 return 0;
138} 139}
@@ -142,22 +143,74 @@ static void branch_trace_reset(struct trace_array *tr)
142 stop_branch_trace(tr); 143 stop_branch_trace(tr);
143} 144}
144 145
145struct tracer branch_trace __read_mostly = 146static int
147trace_print_print(struct trace_seq *s, struct trace_entry *entry, int flags)
148{
149 struct print_entry *field;
150
151 trace_assign_type(field, entry);
152
153 if (seq_print_ip_sym(s, field->ip, flags))
154 goto partial;
155
156 if (trace_seq_printf(s, ": %s", field->buf))
157 goto partial;
158
159 partial:
160 return TRACE_TYPE_PARTIAL_LINE;
161}
162
163static int
164trace_branch_print(struct trace_seq *s, struct trace_entry *entry, int flags)
165{
166 struct trace_branch *field;
167
168 trace_assign_type(field, entry);
169
170 if (trace_seq_printf(s, "[%s] %s:%s:%d\n",
171 field->correct ? " ok " : " MISS ",
172 field->func,
173 field->file,
174 field->line))
175 return TRACE_TYPE_PARTIAL_LINE;
176
177 return 0;
178}
179
180
181static struct trace_event trace_branch_event = {
182 .type = TRACE_BRANCH,
183 .trace = trace_branch_print,
184 .latency_trace = trace_branch_print,
185 .raw = trace_nop_print,
186 .hex = trace_nop_print,
187 .binary = trace_nop_print,
188};
189
190static struct tracer branch_trace __read_mostly =
146{ 191{
147 .name = "branch", 192 .name = "branch",
148 .init = branch_trace_init, 193 .init = branch_trace_init,
149 .reset = branch_trace_reset, 194 .reset = branch_trace_reset,
150#ifdef CONFIG_FTRACE_SELFTEST 195#ifdef CONFIG_FTRACE_SELFTEST
151 .selftest = trace_selftest_startup_branch, 196 .selftest = trace_selftest_startup_branch,
152#endif 197#endif /* CONFIG_FTRACE_SELFTEST */
153}; 198};
154 199
155__init static int init_branch_trace(void) 200__init static int init_branch_tracer(void)
156{ 201{
202 int ret;
203
204 ret = register_ftrace_event(&trace_branch_event);
205 if (!ret) {
206 printk(KERN_WARNING "Warning: could not register "
207 "branch events\n");
208 return 1;
209 }
157 return register_tracer(&branch_trace); 210 return register_tracer(&branch_trace);
158} 211}
212device_initcall(init_branch_tracer);
159 213
160device_initcall(init_branch_trace);
161#else 214#else
162static inline 215static inline
163void trace_likely_condition(struct ftrace_branch_data *f, int val, int expect) 216void trace_likely_condition(struct ftrace_branch_data *f, int val, int expect)
@@ -183,66 +236,39 @@ void ftrace_likely_update(struct ftrace_branch_data *f, int val, int expect)
183} 236}
184EXPORT_SYMBOL(ftrace_likely_update); 237EXPORT_SYMBOL(ftrace_likely_update);
185 238
186struct ftrace_pointer { 239extern unsigned long __start_annotated_branch_profile[];
187 void *start; 240extern unsigned long __stop_annotated_branch_profile[];
188 void *stop;
189 int hit;
190};
191 241
192static void * 242static int annotated_branch_stat_headers(struct seq_file *m)
193t_next(struct seq_file *m, void *v, loff_t *pos)
194{ 243{
195 const struct ftrace_pointer *f = m->private; 244 seq_printf(m, " correct incorrect %% ");
196 struct ftrace_branch_data *p = v; 245 seq_printf(m, " Function "
197 246 " File Line\n"
198 (*pos)++; 247 " ------- --------- - "
199 248 " -------- "
200 if (v == (void *)1) 249 " ---- ----\n");
201 return f->start; 250 return 0;
202
203 ++p;
204
205 if ((void *)p >= (void *)f->stop)
206 return NULL;
207
208 return p;
209} 251}
210 252
211static void *t_start(struct seq_file *m, loff_t *pos) 253static inline long get_incorrect_percent(struct ftrace_branch_data *p)
212{ 254{
213 void *t = (void *)1; 255 long percent;
214 loff_t l = 0;
215
216 for (; t && l < *pos; t = t_next(m, t, &l))
217 ;
218 256
219 return t; 257 if (p->correct) {
220} 258 percent = p->incorrect * 100;
259 percent /= p->correct + p->incorrect;
260 } else
261 percent = p->incorrect ? 100 : -1;
221 262
222static void t_stop(struct seq_file *m, void *p) 263 return percent;
223{
224} 264}
225 265
226static int t_show(struct seq_file *m, void *v) 266static int branch_stat_show(struct seq_file *m, void *v)
227{ 267{
228 const struct ftrace_pointer *fp = m->private;
229 struct ftrace_branch_data *p = v; 268 struct ftrace_branch_data *p = v;
230 const char *f; 269 const char *f;
231 long percent; 270 long percent;
232 271
233 if (v == (void *)1) {
234 if (fp->hit)
235 seq_printf(m, " miss hit %% ");
236 else
237 seq_printf(m, " correct incorrect %% ");
238 seq_printf(m, " Function "
239 " File Line\n"
240 " ------- --------- - "
241 " -------- "
242 " ---- ----\n");
243 return 0;
244 }
245
246 /* Only print the file, not the path */ 272 /* Only print the file, not the path */
247 f = p->file + strlen(p->file); 273 f = p->file + strlen(p->file);
248 while (f >= p->file && *f != '/') 274 while (f >= p->file && *f != '/')
@@ -252,11 +278,7 @@ static int t_show(struct seq_file *m, void *v)
252 /* 278 /*
253 * The miss is overlayed on correct, and hit on incorrect. 279 * The miss is overlayed on correct, and hit on incorrect.
254 */ 280 */
255 if (p->correct) { 281 percent = get_incorrect_percent(p);
256 percent = p->incorrect * 100;
257 percent /= p->correct + p->incorrect;
258 } else
259 percent = p->incorrect ? 100 : -1;
260 282
261 seq_printf(m, "%8lu %8lu ", p->correct, p->incorrect); 283 seq_printf(m, "%8lu %8lu ", p->correct, p->incorrect);
262 if (percent < 0) 284 if (percent < 0)
@@ -267,76 +289,118 @@ static int t_show(struct seq_file *m, void *v)
267 return 0; 289 return 0;
268} 290}
269 291
270static struct seq_operations tracing_likely_seq_ops = { 292static void *annotated_branch_stat_start(void)
271 .start = t_start, 293{
272 .next = t_next, 294 return __start_annotated_branch_profile;
273 .stop = t_stop, 295}
274 .show = t_show, 296
297static void *
298annotated_branch_stat_next(void *v, int idx)
299{
300 struct ftrace_branch_data *p = v;
301
302 ++p;
303
304 if ((void *)p >= (void *)__stop_annotated_branch_profile)
305 return NULL;
306
307 return p;
308}
309
310static int annotated_branch_stat_cmp(void *p1, void *p2)
311{
312 struct ftrace_branch_data *a = p1;
313 struct ftrace_branch_data *b = p2;
314
315 long percent_a, percent_b;
316
317 percent_a = get_incorrect_percent(a);
318 percent_b = get_incorrect_percent(b);
319
320 if (percent_a < percent_b)
321 return -1;
322 if (percent_a > percent_b)
323 return 1;
324 else
325 return 0;
326}
327
328static struct tracer_stat annotated_branch_stats = {
329 .name = "branch_annotated",
330 .stat_start = annotated_branch_stat_start,
331 .stat_next = annotated_branch_stat_next,
332 .stat_cmp = annotated_branch_stat_cmp,
333 .stat_headers = annotated_branch_stat_headers,
334 .stat_show = branch_stat_show
275}; 335};
276 336
277static int tracing_branch_open(struct inode *inode, struct file *file) 337__init static int init_annotated_branch_stats(void)
278{ 338{
279 int ret; 339 int ret;
280 340
281 ret = seq_open(file, &tracing_likely_seq_ops); 341 ret = register_stat_tracer(&annotated_branch_stats);
282 if (!ret) { 342 if (!ret) {
283 struct seq_file *m = file->private_data; 343 printk(KERN_WARNING "Warning: could not register "
284 m->private = (void *)inode->i_private; 344 "annotated branches stats\n");
345 return 1;
285 } 346 }
286 347 return 0;
287 return ret;
288} 348}
289 349fs_initcall(init_annotated_branch_stats);
290static const struct file_operations tracing_branch_fops = {
291 .open = tracing_branch_open,
292 .read = seq_read,
293 .llseek = seq_lseek,
294};
295 350
296#ifdef CONFIG_PROFILE_ALL_BRANCHES 351#ifdef CONFIG_PROFILE_ALL_BRANCHES
352
297extern unsigned long __start_branch_profile[]; 353extern unsigned long __start_branch_profile[];
298extern unsigned long __stop_branch_profile[]; 354extern unsigned long __stop_branch_profile[];
299 355
300static const struct ftrace_pointer ftrace_branch_pos = { 356static int all_branch_stat_headers(struct seq_file *m)
301 .start = __start_branch_profile, 357{
302 .stop = __stop_branch_profile, 358 seq_printf(m, " miss hit %% ");
303 .hit = 1, 359 seq_printf(m, " Function "
304}; 360 " File Line\n"
361 " ------- --------- - "
362 " -------- "
363 " ---- ----\n");
364 return 0;
365}
305 366
306#endif /* CONFIG_PROFILE_ALL_BRANCHES */ 367static void *all_branch_stat_start(void)
368{
369 return __start_branch_profile;
370}
307 371
308extern unsigned long __start_annotated_branch_profile[]; 372static void *
309extern unsigned long __stop_annotated_branch_profile[]; 373all_branch_stat_next(void *v, int idx)
374{
375 struct ftrace_branch_data *p = v;
310 376
311static const struct ftrace_pointer ftrace_annotated_branch_pos = { 377 ++p;
312 .start = __start_annotated_branch_profile,
313 .stop = __stop_annotated_branch_profile,
314};
315 378
316static __init int ftrace_branch_init(void) 379 if ((void *)p >= (void *)__stop_branch_profile)
317{ 380 return NULL;
318 struct dentry *d_tracer;
319 struct dentry *entry;
320 381
321 d_tracer = tracing_init_dentry(); 382 return p;
383}
322 384
323 entry = debugfs_create_file("profile_annotated_branch", 0444, d_tracer, 385static struct tracer_stat all_branch_stats = {
324 (void *)&ftrace_annotated_branch_pos, 386 .name = "branch_all",
325 &tracing_branch_fops); 387 .stat_start = all_branch_stat_start,
326 if (!entry) 388 .stat_next = all_branch_stat_next,
327 pr_warning("Could not create debugfs " 389 .stat_headers = all_branch_stat_headers,
328 "'profile_annotatet_branch' entry\n"); 390 .stat_show = branch_stat_show
391};
329 392
330#ifdef CONFIG_PROFILE_ALL_BRANCHES 393__init static int all_annotated_branch_stats(void)
331 entry = debugfs_create_file("profile_branch", 0444, d_tracer, 394{
332 (void *)&ftrace_branch_pos, 395 int ret;
333 &tracing_branch_fops);
334 if (!entry)
335 pr_warning("Could not create debugfs"
336 " 'profile_branch' entry\n");
337#endif
338 396
397 ret = register_stat_tracer(&all_branch_stats);
398 if (!ret) {
399 printk(KERN_WARNING "Warning: could not register "
400 "all branches stats\n");
401 return 1;
402 }
339 return 0; 403 return 0;
340} 404}
341 405fs_initcall(all_annotated_branch_stats);
342device_initcall(ftrace_branch_init); 406#endif /* CONFIG_PROFILE_ALL_BRANCHES */
diff --git a/kernel/trace/trace_functions.c b/kernel/trace/trace_functions.c
index 9236d7e25a16..b3a320f8aba7 100644
--- a/kernel/trace/trace_functions.c
+++ b/kernel/trace/trace_functions.c
@@ -16,8 +16,17 @@
16 16
17#include "trace.h" 17#include "trace.h"
18 18
19/* function tracing enabled */
20static int ftrace_function_enabled;
21
22static struct trace_array *func_trace;
23
24static void tracing_start_function_trace(void);
25static void tracing_stop_function_trace(void);
26
19static void start_function_trace(struct trace_array *tr) 27static void start_function_trace(struct trace_array *tr)
20{ 28{
29 func_trace = tr;
21 tr->cpu = get_cpu(); 30 tr->cpu = get_cpu();
22 tracing_reset_online_cpus(tr); 31 tracing_reset_online_cpus(tr);
23 put_cpu(); 32 put_cpu();
@@ -48,14 +57,188 @@ static void function_trace_start(struct trace_array *tr)
48 tracing_reset_online_cpus(tr); 57 tracing_reset_online_cpus(tr);
49} 58}
50 59
60static void
61function_trace_call_preempt_only(unsigned long ip, unsigned long parent_ip)
62{
63 struct trace_array *tr = func_trace;
64 struct trace_array_cpu *data;
65 unsigned long flags;
66 long disabled;
67 int cpu, resched;
68 int pc;
69
70 if (unlikely(!ftrace_function_enabled))
71 return;
72
73 pc = preempt_count();
74 resched = ftrace_preempt_disable();
75 local_save_flags(flags);
76 cpu = raw_smp_processor_id();
77 data = tr->data[cpu];
78 disabled = atomic_inc_return(&data->disabled);
79
80 if (likely(disabled == 1))
81 trace_function(tr, data, ip, parent_ip, flags, pc);
82
83 atomic_dec(&data->disabled);
84 ftrace_preempt_enable(resched);
85}
86
87static void
88function_trace_call(unsigned long ip, unsigned long parent_ip)
89{
90 struct trace_array *tr = func_trace;
91 struct trace_array_cpu *data;
92 unsigned long flags;
93 long disabled;
94 int cpu;
95 int pc;
96
97 if (unlikely(!ftrace_function_enabled))
98 return;
99
100 /*
101 * Need to use raw, since this must be called before the
102 * recursive protection is performed.
103 */
104 local_irq_save(flags);
105 cpu = raw_smp_processor_id();
106 data = tr->data[cpu];
107 disabled = atomic_inc_return(&data->disabled);
108
109 if (likely(disabled == 1)) {
110 pc = preempt_count();
111 trace_function(tr, data, ip, parent_ip, flags, pc);
112 }
113
114 atomic_dec(&data->disabled);
115 local_irq_restore(flags);
116}
117
118static void
119function_stack_trace_call(unsigned long ip, unsigned long parent_ip)
120{
121 struct trace_array *tr = func_trace;
122 struct trace_array_cpu *data;
123 unsigned long flags;
124 long disabled;
125 int cpu;
126 int pc;
127
128 if (unlikely(!ftrace_function_enabled))
129 return;
130
131 /*
132 * Need to use raw, since this must be called before the
133 * recursive protection is performed.
134 */
135 local_irq_save(flags);
136 cpu = raw_smp_processor_id();
137 data = tr->data[cpu];
138 disabled = atomic_inc_return(&data->disabled);
139
140 if (likely(disabled == 1)) {
141 pc = preempt_count();
142 trace_function(tr, data, ip, parent_ip, flags, pc);
143 /*
144 * skip over 5 funcs:
145 * __ftrace_trace_stack,
146 * __trace_stack,
147 * function_stack_trace_call
148 * ftrace_list_func
149 * ftrace_call
150 */
151 __trace_stack(tr, data, flags, 5, pc);
152 }
153
154 atomic_dec(&data->disabled);
155 local_irq_restore(flags);
156}
157
158
159static struct ftrace_ops trace_ops __read_mostly =
160{
161 .func = function_trace_call,
162};
163
164static struct ftrace_ops trace_stack_ops __read_mostly =
165{
166 .func = function_stack_trace_call,
167};
168
169/* Our two options */
170enum {
171 TRACE_FUNC_OPT_STACK = 0x1,
172};
173
174static struct tracer_opt func_opts[] = {
175#ifdef CONFIG_STACKTRACE
176 { TRACER_OPT(func_stack_trace, TRACE_FUNC_OPT_STACK) },
177#endif
178 { } /* Always set a last empty entry */
179};
180
181static struct tracer_flags func_flags = {
182 .val = 0, /* By default: all flags disabled */
183 .opts = func_opts
184};
185
186static void tracing_start_function_trace(void)
187{
188 ftrace_function_enabled = 0;
189
190 if (trace_flags & TRACE_ITER_PREEMPTONLY)
191 trace_ops.func = function_trace_call_preempt_only;
192 else
193 trace_ops.func = function_trace_call;
194
195 if (func_flags.val & TRACE_FUNC_OPT_STACK)
196 register_ftrace_function(&trace_stack_ops);
197 else
198 register_ftrace_function(&trace_ops);
199
200 ftrace_function_enabled = 1;
201}
202
203static void tracing_stop_function_trace(void)
204{
205 ftrace_function_enabled = 0;
206 /* OK if they are not registered */
207 unregister_ftrace_function(&trace_stack_ops);
208 unregister_ftrace_function(&trace_ops);
209}
210
211static int func_set_flag(u32 old_flags, u32 bit, int set)
212{
213 if (bit == TRACE_FUNC_OPT_STACK) {
214 /* do nothing if already set */
215 if (!!set == !!(func_flags.val & TRACE_FUNC_OPT_STACK))
216 return 0;
217
218 if (set) {
219 unregister_ftrace_function(&trace_ops);
220 register_ftrace_function(&trace_stack_ops);
221 } else {
222 unregister_ftrace_function(&trace_stack_ops);
223 register_ftrace_function(&trace_ops);
224 }
225
226 return 0;
227 }
228
229 return -EINVAL;
230}
231
51static struct tracer function_trace __read_mostly = 232static struct tracer function_trace __read_mostly =
52{ 233{
53 .name = "function", 234 .name = "function",
54 .init = function_trace_init, 235 .init = function_trace_init,
55 .reset = function_trace_reset, 236 .reset = function_trace_reset,
56 .start = function_trace_start, 237 .start = function_trace_start,
238 .flags = &func_flags,
239 .set_flag = func_set_flag,
57#ifdef CONFIG_FTRACE_SELFTEST 240#ifdef CONFIG_FTRACE_SELFTEST
58 .selftest = trace_selftest_startup_function, 241 .selftest = trace_selftest_startup_function,
59#endif 242#endif
60}; 243};
61 244
diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c
index 930c08e5b38e..c97594d826bc 100644
--- a/kernel/trace/trace_functions_graph.c
+++ b/kernel/trace/trace_functions_graph.c
@@ -1,7 +1,7 @@
1/* 1/*
2 * 2 *
3 * Function graph tracer. 3 * Function graph tracer.
4 * Copyright (c) 2008 Frederic Weisbecker <fweisbec@gmail.com> 4 * Copyright (c) 2008-2009 Frederic Weisbecker <fweisbec@gmail.com>
5 * Mostly borrowed from function tracer which 5 * Mostly borrowed from function tracer which
6 * is Copyright (c) Steven Rostedt <srostedt@redhat.com> 6 * is Copyright (c) Steven Rostedt <srostedt@redhat.com>
7 * 7 *
@@ -12,6 +12,7 @@
12#include <linux/fs.h> 12#include <linux/fs.h>
13 13
14#include "trace.h" 14#include "trace.h"
15#include "trace_output.h"
15 16
16#define TRACE_GRAPH_INDENT 2 17#define TRACE_GRAPH_INDENT 2
17 18
@@ -20,9 +21,11 @@
20#define TRACE_GRAPH_PRINT_CPU 0x2 21#define TRACE_GRAPH_PRINT_CPU 0x2
21#define TRACE_GRAPH_PRINT_OVERHEAD 0x4 22#define TRACE_GRAPH_PRINT_OVERHEAD 0x4
22#define TRACE_GRAPH_PRINT_PROC 0x8 23#define TRACE_GRAPH_PRINT_PROC 0x8
24#define TRACE_GRAPH_PRINT_DURATION 0x10
25#define TRACE_GRAPH_PRINT_ABS_TIME 0X20
23 26
24static struct tracer_opt trace_opts[] = { 27static struct tracer_opt trace_opts[] = {
25 /* Display overruns ? */ 28 /* Display overruns? (for self-debug purpose) */
26 { TRACER_OPT(funcgraph-overrun, TRACE_GRAPH_PRINT_OVERRUN) }, 29 { TRACER_OPT(funcgraph-overrun, TRACE_GRAPH_PRINT_OVERRUN) },
27 /* Display CPU ? */ 30 /* Display CPU ? */
28 { TRACER_OPT(funcgraph-cpu, TRACE_GRAPH_PRINT_CPU) }, 31 { TRACER_OPT(funcgraph-cpu, TRACE_GRAPH_PRINT_CPU) },
@@ -30,29 +33,30 @@ static struct tracer_opt trace_opts[] = {
30 { TRACER_OPT(funcgraph-overhead, TRACE_GRAPH_PRINT_OVERHEAD) }, 33 { TRACER_OPT(funcgraph-overhead, TRACE_GRAPH_PRINT_OVERHEAD) },
31 /* Display proc name/pid */ 34 /* Display proc name/pid */
32 { TRACER_OPT(funcgraph-proc, TRACE_GRAPH_PRINT_PROC) }, 35 { TRACER_OPT(funcgraph-proc, TRACE_GRAPH_PRINT_PROC) },
36 /* Display duration of execution */
37 { TRACER_OPT(funcgraph-duration, TRACE_GRAPH_PRINT_DURATION) },
38 /* Display absolute time of an entry */
39 { TRACER_OPT(funcgraph-abstime, TRACE_GRAPH_PRINT_ABS_TIME) },
33 { } /* Empty entry */ 40 { } /* Empty entry */
34}; 41};
35 42
36static struct tracer_flags tracer_flags = { 43static struct tracer_flags tracer_flags = {
37 /* Don't display overruns and proc by default */ 44 /* Don't display overruns and proc by default */
38 .val = TRACE_GRAPH_PRINT_CPU | TRACE_GRAPH_PRINT_OVERHEAD, 45 .val = TRACE_GRAPH_PRINT_CPU | TRACE_GRAPH_PRINT_OVERHEAD |
46 TRACE_GRAPH_PRINT_DURATION,
39 .opts = trace_opts 47 .opts = trace_opts
40}; 48};
41 49
42/* pid on the last trace processed */ 50/* pid on the last trace processed */
43static pid_t last_pid[NR_CPUS] = { [0 ... NR_CPUS-1] = -1 }; 51
44 52
45static int graph_trace_init(struct trace_array *tr) 53static int graph_trace_init(struct trace_array *tr)
46{ 54{
47 int cpu, ret; 55 int ret = register_ftrace_graph(&trace_graph_return,
48
49 for_each_online_cpu(cpu)
50 tracing_reset(tr, cpu);
51
52 ret = register_ftrace_graph(&trace_graph_return,
53 &trace_graph_entry); 56 &trace_graph_entry);
54 if (ret) 57 if (ret)
55 return ret; 58 return ret;
59 tracing_reset_online_cpus(tr);
56 tracing_start_cmdline_record(); 60 tracing_start_cmdline_record();
57 61
58 return 0; 62 return 0;
@@ -153,17 +157,25 @@ print_graph_proc(struct trace_seq *s, pid_t pid)
153 157
154/* If the pid changed since the last trace, output this event */ 158/* If the pid changed since the last trace, output this event */
155static enum print_line_t 159static enum print_line_t
156verif_pid(struct trace_seq *s, pid_t pid, int cpu) 160verif_pid(struct trace_seq *s, pid_t pid, int cpu, pid_t *last_pids_cpu)
157{ 161{
158 pid_t prev_pid; 162 pid_t prev_pid;
163 pid_t *last_pid;
159 int ret; 164 int ret;
160 165
161 if (last_pid[cpu] != -1 && last_pid[cpu] == pid) 166 if (!last_pids_cpu)
162 return TRACE_TYPE_HANDLED; 167 return TRACE_TYPE_HANDLED;
163 168
164 prev_pid = last_pid[cpu]; 169 last_pid = per_cpu_ptr(last_pids_cpu, cpu);
165 last_pid[cpu] = pid; 170
171 if (*last_pid == pid)
172 return TRACE_TYPE_HANDLED;
166 173
174 prev_pid = *last_pid;
175 *last_pid = pid;
176
177 if (prev_pid == -1)
178 return TRACE_TYPE_HANDLED;
167/* 179/*
168 * Context-switch trace line: 180 * Context-switch trace line:
169 181
@@ -231,9 +243,34 @@ trace_branch_is_leaf(struct trace_iterator *iter,
231 return true; 243 return true;
232} 244}
233 245
246/* Signal a overhead of time execution to the output */
247static int
248print_graph_overhead(unsigned long long duration, struct trace_seq *s)
249{
250 /* If duration disappear, we don't need anything */
251 if (!(tracer_flags.val & TRACE_GRAPH_PRINT_DURATION))
252 return 1;
253
254 /* Non nested entry or return */
255 if (duration == -1)
256 return trace_seq_printf(s, " ");
257
258 if (tracer_flags.val & TRACE_GRAPH_PRINT_OVERHEAD) {
259 /* Duration exceeded 100 msecs */
260 if (duration > 100000ULL)
261 return trace_seq_printf(s, "! ");
262
263 /* Duration exceeded 10 msecs */
264 if (duration > 10000ULL)
265 return trace_seq_printf(s, "+ ");
266 }
267
268 return trace_seq_printf(s, " ");
269}
270
234static enum print_line_t 271static enum print_line_t
235print_graph_irq(struct trace_seq *s, unsigned long addr, 272print_graph_irq(struct trace_seq *s, unsigned long addr,
236 enum trace_type type, int cpu, pid_t pid) 273 enum trace_type type, int cpu, pid_t pid)
237{ 274{
238 int ret; 275 int ret;
239 276
@@ -241,35 +278,40 @@ print_graph_irq(struct trace_seq *s, unsigned long addr,
241 addr >= (unsigned long)__irqentry_text_end) 278 addr >= (unsigned long)__irqentry_text_end)
242 return TRACE_TYPE_UNHANDLED; 279 return TRACE_TYPE_UNHANDLED;
243 280
244 if (type == TRACE_GRAPH_ENT) { 281 /* Cpu */
245 ret = trace_seq_printf(s, "==========> | "); 282 if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU) {
246 } else { 283 ret = print_graph_cpu(s, cpu);
247 /* Cpu */ 284 if (ret == TRACE_TYPE_PARTIAL_LINE)
248 if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU) { 285 return TRACE_TYPE_PARTIAL_LINE;
249 ret = print_graph_cpu(s, cpu); 286 }
250 if (ret == TRACE_TYPE_PARTIAL_LINE) 287 /* Proc */
251 return TRACE_TYPE_PARTIAL_LINE; 288 if (tracer_flags.val & TRACE_GRAPH_PRINT_PROC) {
252 } 289 ret = print_graph_proc(s, pid);
253 /* Proc */ 290 if (ret == TRACE_TYPE_PARTIAL_LINE)
254 if (tracer_flags.val & TRACE_GRAPH_PRINT_PROC) { 291 return TRACE_TYPE_PARTIAL_LINE;
255 ret = print_graph_proc(s, pid); 292 ret = trace_seq_printf(s, " | ");
256 if (ret == TRACE_TYPE_PARTIAL_LINE) 293 if (!ret)
257 return TRACE_TYPE_PARTIAL_LINE; 294 return TRACE_TYPE_PARTIAL_LINE;
295 }
258 296
259 ret = trace_seq_printf(s, " | "); 297 /* No overhead */
260 if (!ret) 298 ret = print_graph_overhead(-1, s);
261 return TRACE_TYPE_PARTIAL_LINE; 299 if (!ret)
262 } 300 return TRACE_TYPE_PARTIAL_LINE;
263 301
264 /* No overhead */ 302 if (type == TRACE_GRAPH_ENT)
265 if (tracer_flags.val & TRACE_GRAPH_PRINT_OVERHEAD) { 303 ret = trace_seq_printf(s, "==========>");
266 ret = trace_seq_printf(s, " "); 304 else
267 if (!ret) 305 ret = trace_seq_printf(s, "<==========");
268 return TRACE_TYPE_PARTIAL_LINE; 306
269 } 307 if (!ret)
308 return TRACE_TYPE_PARTIAL_LINE;
309
310 /* Don't close the duration column if haven't one */
311 if (tracer_flags.val & TRACE_GRAPH_PRINT_DURATION)
312 trace_seq_printf(s, " |");
313 ret = trace_seq_printf(s, "\n");
270 314
271 ret = trace_seq_printf(s, "<========== |\n");
272 }
273 if (!ret) 315 if (!ret)
274 return TRACE_TYPE_PARTIAL_LINE; 316 return TRACE_TYPE_PARTIAL_LINE;
275 return TRACE_TYPE_HANDLED; 317 return TRACE_TYPE_HANDLED;
@@ -288,7 +330,7 @@ print_graph_duration(unsigned long long duration, struct trace_seq *s)
288 sprintf(msecs_str, "%lu", (unsigned long) duration); 330 sprintf(msecs_str, "%lu", (unsigned long) duration);
289 331
290 /* Print msecs */ 332 /* Print msecs */
291 ret = trace_seq_printf(s, msecs_str); 333 ret = trace_seq_printf(s, "%s", msecs_str);
292 if (!ret) 334 if (!ret)
293 return TRACE_TYPE_PARTIAL_LINE; 335 return TRACE_TYPE_PARTIAL_LINE;
294 336
@@ -321,19 +363,15 @@ print_graph_duration(unsigned long long duration, struct trace_seq *s)
321 363
322} 364}
323 365
324/* Signal a overhead of time execution to the output */ 366static int print_graph_abs_time(u64 t, struct trace_seq *s)
325static int
326print_graph_overhead(unsigned long long duration, struct trace_seq *s)
327{ 367{
328 /* Duration exceeded 100 msecs */ 368 unsigned long usecs_rem;
329 if (duration > 100000ULL)
330 return trace_seq_printf(s, "! ");
331 369
332 /* Duration exceeded 10 msecs */ 370 usecs_rem = do_div(t, 1000000000);
333 if (duration > 10000ULL) 371 usecs_rem /= 1000;
334 return trace_seq_printf(s, "+ ");
335 372
336 return trace_seq_printf(s, " "); 373 return trace_seq_printf(s, "%5lu.%06lu | ",
374 (unsigned long)t, usecs_rem);
337} 375}
338 376
339/* Case of a leaf function on its call entry */ 377/* Case of a leaf function on its call entry */
@@ -356,16 +394,16 @@ print_graph_entry_leaf(struct trace_iterator *iter,
356 duration = graph_ret->rettime - graph_ret->calltime; 394 duration = graph_ret->rettime - graph_ret->calltime;
357 395
358 /* Overhead */ 396 /* Overhead */
359 if (tracer_flags.val & TRACE_GRAPH_PRINT_OVERHEAD) { 397 ret = print_graph_overhead(duration, s);
360 ret = print_graph_overhead(duration, s); 398 if (!ret)
361 if (!ret) 399 return TRACE_TYPE_PARTIAL_LINE;
362 return TRACE_TYPE_PARTIAL_LINE;
363 }
364 400
365 /* Duration */ 401 /* Duration */
366 ret = print_graph_duration(duration, s); 402 if (tracer_flags.val & TRACE_GRAPH_PRINT_DURATION) {
367 if (ret == TRACE_TYPE_PARTIAL_LINE) 403 ret = print_graph_duration(duration, s);
368 return TRACE_TYPE_PARTIAL_LINE; 404 if (ret == TRACE_TYPE_PARTIAL_LINE)
405 return TRACE_TYPE_PARTIAL_LINE;
406 }
369 407
370 /* Function */ 408 /* Function */
371 for (i = 0; i < call->depth * TRACE_GRAPH_INDENT; i++) { 409 for (i = 0; i < call->depth * TRACE_GRAPH_INDENT; i++) {
@@ -394,25 +432,17 @@ print_graph_entry_nested(struct ftrace_graph_ent_entry *entry,
394 struct ftrace_graph_ent *call = &entry->graph_ent; 432 struct ftrace_graph_ent *call = &entry->graph_ent;
395 433
396 /* No overhead */ 434 /* No overhead */
397 if (tracer_flags.val & TRACE_GRAPH_PRINT_OVERHEAD) { 435 ret = print_graph_overhead(-1, s);
398 ret = trace_seq_printf(s, " "); 436 if (!ret)
399 if (!ret) 437 return TRACE_TYPE_PARTIAL_LINE;
400 return TRACE_TYPE_PARTIAL_LINE;
401 }
402 438
403 /* Interrupt */ 439 /* No time */
404 ret = print_graph_irq(s, call->func, TRACE_GRAPH_ENT, cpu, pid); 440 if (tracer_flags.val & TRACE_GRAPH_PRINT_DURATION) {
405 if (ret == TRACE_TYPE_UNHANDLED) {
406 /* No time */
407 ret = trace_seq_printf(s, " | "); 441 ret = trace_seq_printf(s, " | ");
408 if (!ret) 442 if (!ret)
409 return TRACE_TYPE_PARTIAL_LINE; 443 return TRACE_TYPE_PARTIAL_LINE;
410 } else {
411 if (ret == TRACE_TYPE_PARTIAL_LINE)
412 return TRACE_TYPE_PARTIAL_LINE;
413 } 444 }
414 445
415
416 /* Function */ 446 /* Function */
417 for (i = 0; i < call->depth * TRACE_GRAPH_INDENT; i++) { 447 for (i = 0; i < call->depth * TRACE_GRAPH_INDENT; i++) {
418 ret = trace_seq_printf(s, " "); 448 ret = trace_seq_printf(s, " ");
@@ -433,15 +463,30 @@ print_graph_entry_nested(struct ftrace_graph_ent_entry *entry,
433 463
434static enum print_line_t 464static enum print_line_t
435print_graph_entry(struct ftrace_graph_ent_entry *field, struct trace_seq *s, 465print_graph_entry(struct ftrace_graph_ent_entry *field, struct trace_seq *s,
436 struct trace_iterator *iter, int cpu) 466 struct trace_iterator *iter)
437{ 467{
438 int ret; 468 int ret;
469 int cpu = iter->cpu;
470 pid_t *last_entry = iter->private;
439 struct trace_entry *ent = iter->ent; 471 struct trace_entry *ent = iter->ent;
472 struct ftrace_graph_ent *call = &field->graph_ent;
440 473
441 /* Pid */ 474 /* Pid */
442 if (verif_pid(s, ent->pid, cpu) == TRACE_TYPE_PARTIAL_LINE) 475 if (verif_pid(s, ent->pid, cpu, last_entry) == TRACE_TYPE_PARTIAL_LINE)
476 return TRACE_TYPE_PARTIAL_LINE;
477
478 /* Interrupt */
479 ret = print_graph_irq(s, call->func, TRACE_GRAPH_ENT, cpu, ent->pid);
480 if (ret == TRACE_TYPE_PARTIAL_LINE)
443 return TRACE_TYPE_PARTIAL_LINE; 481 return TRACE_TYPE_PARTIAL_LINE;
444 482
483 /* Absolute time */
484 if (tracer_flags.val & TRACE_GRAPH_PRINT_ABS_TIME) {
485 ret = print_graph_abs_time(iter->ts, s);
486 if (!ret)
487 return TRACE_TYPE_PARTIAL_LINE;
488 }
489
445 /* Cpu */ 490 /* Cpu */
446 if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU) { 491 if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU) {
447 ret = print_graph_cpu(s, cpu); 492 ret = print_graph_cpu(s, cpu);
@@ -469,16 +514,25 @@ print_graph_entry(struct ftrace_graph_ent_entry *field, struct trace_seq *s,
469 514
470static enum print_line_t 515static enum print_line_t
471print_graph_return(struct ftrace_graph_ret *trace, struct trace_seq *s, 516print_graph_return(struct ftrace_graph_ret *trace, struct trace_seq *s,
472 struct trace_entry *ent, int cpu) 517 struct trace_entry *ent, struct trace_iterator *iter)
473{ 518{
474 int i; 519 int i;
475 int ret; 520 int ret;
521 int cpu = iter->cpu;
522 pid_t *last_pid = iter->private;
476 unsigned long long duration = trace->rettime - trace->calltime; 523 unsigned long long duration = trace->rettime - trace->calltime;
477 524
478 /* Pid */ 525 /* Pid */
479 if (verif_pid(s, ent->pid, cpu) == TRACE_TYPE_PARTIAL_LINE) 526 if (verif_pid(s, ent->pid, cpu, last_pid) == TRACE_TYPE_PARTIAL_LINE)
480 return TRACE_TYPE_PARTIAL_LINE; 527 return TRACE_TYPE_PARTIAL_LINE;
481 528
529 /* Absolute time */
530 if (tracer_flags.val & TRACE_GRAPH_PRINT_ABS_TIME) {
531 ret = print_graph_abs_time(iter->ts, s);
532 if (!ret)
533 return TRACE_TYPE_PARTIAL_LINE;
534 }
535
482 /* Cpu */ 536 /* Cpu */
483 if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU) { 537 if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU) {
484 ret = print_graph_cpu(s, cpu); 538 ret = print_graph_cpu(s, cpu);
@@ -498,16 +552,16 @@ print_graph_return(struct ftrace_graph_ret *trace, struct trace_seq *s,
498 } 552 }
499 553
500 /* Overhead */ 554 /* Overhead */
501 if (tracer_flags.val & TRACE_GRAPH_PRINT_OVERHEAD) { 555 ret = print_graph_overhead(duration, s);
502 ret = print_graph_overhead(duration, s); 556 if (!ret)
503 if (!ret) 557 return TRACE_TYPE_PARTIAL_LINE;
504 return TRACE_TYPE_PARTIAL_LINE;
505 }
506 558
507 /* Duration */ 559 /* Duration */
508 ret = print_graph_duration(duration, s); 560 if (tracer_flags.val & TRACE_GRAPH_PRINT_DURATION) {
509 if (ret == TRACE_TYPE_PARTIAL_LINE) 561 ret = print_graph_duration(duration, s);
510 return TRACE_TYPE_PARTIAL_LINE; 562 if (ret == TRACE_TYPE_PARTIAL_LINE)
563 return TRACE_TYPE_PARTIAL_LINE;
564 }
511 565
512 /* Closing brace */ 566 /* Closing brace */
513 for (i = 0; i < trace->depth * TRACE_GRAPH_INDENT; i++) { 567 for (i = 0; i < trace->depth * TRACE_GRAPH_INDENT; i++) {
@@ -541,14 +595,23 @@ print_graph_comment(struct print_entry *trace, struct trace_seq *s,
541{ 595{
542 int i; 596 int i;
543 int ret; 597 int ret;
598 int cpu = iter->cpu;
599 pid_t *last_pid = iter->private;
600
601 /* Absolute time */
602 if (tracer_flags.val & TRACE_GRAPH_PRINT_ABS_TIME) {
603 ret = print_graph_abs_time(iter->ts, s);
604 if (!ret)
605 return TRACE_TYPE_PARTIAL_LINE;
606 }
544 607
545 /* Pid */ 608 /* Pid */
546 if (verif_pid(s, ent->pid, iter->cpu) == TRACE_TYPE_PARTIAL_LINE) 609 if (verif_pid(s, ent->pid, cpu, last_pid) == TRACE_TYPE_PARTIAL_LINE)
547 return TRACE_TYPE_PARTIAL_LINE; 610 return TRACE_TYPE_PARTIAL_LINE;
548 611
549 /* Cpu */ 612 /* Cpu */
550 if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU) { 613 if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU) {
551 ret = print_graph_cpu(s, iter->cpu); 614 ret = print_graph_cpu(s, cpu);
552 if (ret == TRACE_TYPE_PARTIAL_LINE) 615 if (ret == TRACE_TYPE_PARTIAL_LINE)
553 return TRACE_TYPE_PARTIAL_LINE; 616 return TRACE_TYPE_PARTIAL_LINE;
554 } 617 }
@@ -565,17 +628,17 @@ print_graph_comment(struct print_entry *trace, struct trace_seq *s,
565 } 628 }
566 629
567 /* No overhead */ 630 /* No overhead */
568 if (tracer_flags.val & TRACE_GRAPH_PRINT_OVERHEAD) { 631 ret = print_graph_overhead(-1, s);
569 ret = trace_seq_printf(s, " "); 632 if (!ret)
633 return TRACE_TYPE_PARTIAL_LINE;
634
635 /* No time */
636 if (tracer_flags.val & TRACE_GRAPH_PRINT_DURATION) {
637 ret = trace_seq_printf(s, " | ");
570 if (!ret) 638 if (!ret)
571 return TRACE_TYPE_PARTIAL_LINE; 639 return TRACE_TYPE_PARTIAL_LINE;
572 } 640 }
573 641
574 /* No time */
575 ret = trace_seq_printf(s, " | ");
576 if (!ret)
577 return TRACE_TYPE_PARTIAL_LINE;
578
579 /* Indentation */ 642 /* Indentation */
580 if (trace->depth > 0) 643 if (trace->depth > 0)
581 for (i = 0; i < (trace->depth + 1) * TRACE_GRAPH_INDENT; i++) { 644 for (i = 0; i < (trace->depth + 1) * TRACE_GRAPH_INDENT; i++) {
@@ -589,8 +652,11 @@ print_graph_comment(struct print_entry *trace, struct trace_seq *s,
589 if (!ret) 652 if (!ret)
590 return TRACE_TYPE_PARTIAL_LINE; 653 return TRACE_TYPE_PARTIAL_LINE;
591 654
592 if (ent->flags & TRACE_FLAG_CONT) 655 /* Strip ending newline */
593 trace_seq_print_cont(s, iter); 656 if (s->buffer[s->len - 1] == '\n') {
657 s->buffer[s->len - 1] = '\0';
658 s->len--;
659 }
594 660
595 ret = trace_seq_printf(s, " */\n"); 661 ret = trace_seq_printf(s, " */\n");
596 if (!ret) 662 if (!ret)
@@ -610,13 +676,12 @@ print_graph_function(struct trace_iterator *iter)
610 case TRACE_GRAPH_ENT: { 676 case TRACE_GRAPH_ENT: {
611 struct ftrace_graph_ent_entry *field; 677 struct ftrace_graph_ent_entry *field;
612 trace_assign_type(field, entry); 678 trace_assign_type(field, entry);
613 return print_graph_entry(field, s, iter, 679 return print_graph_entry(field, s, iter);
614 iter->cpu);
615 } 680 }
616 case TRACE_GRAPH_RET: { 681 case TRACE_GRAPH_RET: {
617 struct ftrace_graph_ret_entry *field; 682 struct ftrace_graph_ret_entry *field;
618 trace_assign_type(field, entry); 683 trace_assign_type(field, entry);
619 return print_graph_return(&field->ret, s, entry, iter->cpu); 684 return print_graph_return(&field->ret, s, entry, iter);
620 } 685 }
621 case TRACE_PRINT: { 686 case TRACE_PRINT: {
622 struct print_entry *field; 687 struct print_entry *field;
@@ -632,28 +697,55 @@ static void print_graph_headers(struct seq_file *s)
632{ 697{
633 /* 1st line */ 698 /* 1st line */
634 seq_printf(s, "# "); 699 seq_printf(s, "# ");
700 if (tracer_flags.val & TRACE_GRAPH_PRINT_ABS_TIME)
701 seq_printf(s, " TIME ");
635 if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU) 702 if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU)
636 seq_printf(s, "CPU "); 703 seq_printf(s, "CPU");
637 if (tracer_flags.val & TRACE_GRAPH_PRINT_PROC) 704 if (tracer_flags.val & TRACE_GRAPH_PRINT_PROC)
638 seq_printf(s, "TASK/PID "); 705 seq_printf(s, " TASK/PID ");
639 if (tracer_flags.val & TRACE_GRAPH_PRINT_OVERHEAD) 706 if (tracer_flags.val & TRACE_GRAPH_PRINT_DURATION)
640 seq_printf(s, "OVERHEAD/"); 707 seq_printf(s, " DURATION ");
641 seq_printf(s, "DURATION FUNCTION CALLS\n"); 708 seq_printf(s, " FUNCTION CALLS\n");
642 709
643 /* 2nd line */ 710 /* 2nd line */
644 seq_printf(s, "# "); 711 seq_printf(s, "# ");
712 if (tracer_flags.val & TRACE_GRAPH_PRINT_ABS_TIME)
713 seq_printf(s, " | ");
645 if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU) 714 if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU)
646 seq_printf(s, "| "); 715 seq_printf(s, "| ");
647 if (tracer_flags.val & TRACE_GRAPH_PRINT_PROC) 716 if (tracer_flags.val & TRACE_GRAPH_PRINT_PROC)
648 seq_printf(s, "| | "); 717 seq_printf(s, " | | ");
649 if (tracer_flags.val & TRACE_GRAPH_PRINT_OVERHEAD) { 718 if (tracer_flags.val & TRACE_GRAPH_PRINT_DURATION)
650 seq_printf(s, "| "); 719 seq_printf(s, " | | ");
651 seq_printf(s, "| | | | |\n"); 720 seq_printf(s, " | | | |\n");
652 } else 721}
653 seq_printf(s, " | | | | |\n"); 722
723static void graph_trace_open(struct trace_iterator *iter)
724{
725 /* pid on the last trace processed */
726 pid_t *last_pid = alloc_percpu(pid_t);
727 int cpu;
728
729 if (!last_pid)
730 pr_warning("function graph tracer: not enough memory\n");
731 else
732 for_each_possible_cpu(cpu) {
733 pid_t *pid = per_cpu_ptr(last_pid, cpu);
734 *pid = -1;
735 }
736
737 iter->private = last_pid;
654} 738}
739
740static void graph_trace_close(struct trace_iterator *iter)
741{
742 percpu_free(iter->private);
743}
744
655static struct tracer graph_trace __read_mostly = { 745static struct tracer graph_trace __read_mostly = {
656 .name = "function_graph", 746 .name = "function_graph",
747 .open = graph_trace_open,
748 .close = graph_trace_close,
657 .init = graph_trace_init, 749 .init = graph_trace_init,
658 .reset = graph_trace_reset, 750 .reset = graph_trace_reset,
659 .print_line = print_graph_function, 751 .print_line = print_graph_function,
diff --git a/kernel/trace/trace_hw_branches.c b/kernel/trace/trace_hw_branches.c
index 649df22d435f..fff3545fc866 100644
--- a/kernel/trace/trace_hw_branches.c
+++ b/kernel/trace/trace_hw_branches.c
@@ -1,7 +1,8 @@
1/* 1/*
2 * h/w branch tracer for x86 based on bts 2 * h/w branch tracer for x86 based on bts
3 * 3 *
4 * Copyright (C) 2008 Markus Metzger <markus.t.metzger@gmail.com> 4 * Copyright (C) 2008-2009 Intel Corporation.
5 * Markus Metzger <markus.t.metzger@gmail.com>, 2008-2009
5 * 6 *
6 */ 7 */
7 8
@@ -10,21 +11,44 @@
10#include <linux/debugfs.h> 11#include <linux/debugfs.h>
11#include <linux/ftrace.h> 12#include <linux/ftrace.h>
12#include <linux/kallsyms.h> 13#include <linux/kallsyms.h>
14#include <linux/mutex.h>
15#include <linux/cpu.h>
16#include <linux/smp.h>
13 17
14#include <asm/ds.h> 18#include <asm/ds.h>
15 19
16#include "trace.h" 20#include "trace.h"
21#include "trace_output.h"
17 22
18 23
19#define SIZEOF_BTS (1 << 13) 24#define SIZEOF_BTS (1 << 13)
20 25
26/* The tracer mutex protects the below per-cpu tracer array.
27 It needs to be held to:
28 - start tracing on all cpus
29 - stop tracing on all cpus
30 - start tracing on a single hotplug cpu
31 - stop tracing on a single hotplug cpu
32 - read the trace from all cpus
33 - read the trace from a single cpu
34*/
35static DEFINE_MUTEX(bts_tracer_mutex);
21static DEFINE_PER_CPU(struct bts_tracer *, tracer); 36static DEFINE_PER_CPU(struct bts_tracer *, tracer);
22static DEFINE_PER_CPU(unsigned char[SIZEOF_BTS], buffer); 37static DEFINE_PER_CPU(unsigned char[SIZEOF_BTS], buffer);
23 38
24#define this_tracer per_cpu(tracer, smp_processor_id()) 39#define this_tracer per_cpu(tracer, smp_processor_id())
25#define this_buffer per_cpu(buffer, smp_processor_id()) 40#define this_buffer per_cpu(buffer, smp_processor_id())
26 41
42static int __read_mostly trace_hw_branches_enabled;
43static struct trace_array *hw_branch_trace __read_mostly;
27 44
45
46/*
47 * Start tracing on the current cpu.
48 * The argument is ignored.
49 *
50 * pre: bts_tracer_mutex must be locked.
51 */
28static void bts_trace_start_cpu(void *arg) 52static void bts_trace_start_cpu(void *arg)
29{ 53{
30 if (this_tracer) 54 if (this_tracer)
@@ -42,14 +66,20 @@ static void bts_trace_start_cpu(void *arg)
42 66
43static void bts_trace_start(struct trace_array *tr) 67static void bts_trace_start(struct trace_array *tr)
44{ 68{
45 int cpu; 69 mutex_lock(&bts_tracer_mutex);
46 70
47 tracing_reset_online_cpus(tr); 71 on_each_cpu(bts_trace_start_cpu, NULL, 1);
72 trace_hw_branches_enabled = 1;
48 73
49 for_each_cpu(cpu, cpu_possible_mask) 74 mutex_unlock(&bts_tracer_mutex);
50 smp_call_function_single(cpu, bts_trace_start_cpu, NULL, 1);
51} 75}
52 76
77/*
78 * Start tracing on the current cpu.
79 * The argument is ignored.
80 *
81 * pre: bts_tracer_mutex must be locked.
82 */
53static void bts_trace_stop_cpu(void *arg) 83static void bts_trace_stop_cpu(void *arg)
54{ 84{
55 if (this_tracer) { 85 if (this_tracer) {
@@ -60,26 +90,63 @@ static void bts_trace_stop_cpu(void *arg)
60 90
61static void bts_trace_stop(struct trace_array *tr) 91static void bts_trace_stop(struct trace_array *tr)
62{ 92{
63 int cpu; 93 mutex_lock(&bts_tracer_mutex);
94
95 trace_hw_branches_enabled = 0;
96 on_each_cpu(bts_trace_stop_cpu, NULL, 1);
64 97
65 for_each_cpu(cpu, cpu_possible_mask) 98 mutex_unlock(&bts_tracer_mutex);
99}
100
101static int __cpuinit bts_hotcpu_handler(struct notifier_block *nfb,
102 unsigned long action, void *hcpu)
103{
104 unsigned int cpu = (unsigned long)hcpu;
105
106 mutex_lock(&bts_tracer_mutex);
107
108 if (!trace_hw_branches_enabled)
109 goto out;
110
111 switch (action) {
112 case CPU_ONLINE:
113 case CPU_DOWN_FAILED:
114 smp_call_function_single(cpu, bts_trace_start_cpu, NULL, 1);
115 break;
116 case CPU_DOWN_PREPARE:
66 smp_call_function_single(cpu, bts_trace_stop_cpu, NULL, 1); 117 smp_call_function_single(cpu, bts_trace_stop_cpu, NULL, 1);
118 break;
119 }
120
121 out:
122 mutex_unlock(&bts_tracer_mutex);
123 return NOTIFY_DONE;
67} 124}
68 125
126static struct notifier_block bts_hotcpu_notifier __cpuinitdata = {
127 .notifier_call = bts_hotcpu_handler
128};
129
69static int bts_trace_init(struct trace_array *tr) 130static int bts_trace_init(struct trace_array *tr)
70{ 131{
132 hw_branch_trace = tr;
133
134 register_hotcpu_notifier(&bts_hotcpu_notifier);
71 tracing_reset_online_cpus(tr); 135 tracing_reset_online_cpus(tr);
72 bts_trace_start(tr); 136 bts_trace_start(tr);
73 137
74 return 0; 138 return 0;
75} 139}
76 140
141static void bts_trace_reset(struct trace_array *tr)
142{
143 bts_trace_stop(tr);
144 unregister_hotcpu_notifier(&bts_hotcpu_notifier);
145}
146
77static void bts_trace_print_header(struct seq_file *m) 147static void bts_trace_print_header(struct seq_file *m)
78{ 148{
79 seq_puts(m, 149 seq_puts(m, "# CPU# TO <- FROM\n");
80 "# CPU# FROM TO FUNCTION\n");
81 seq_puts(m,
82 "# | | | |\n");
83} 150}
84 151
85static enum print_line_t bts_trace_print_line(struct trace_iterator *iter) 152static enum print_line_t bts_trace_print_line(struct trace_iterator *iter)
@@ -87,15 +154,15 @@ static enum print_line_t bts_trace_print_line(struct trace_iterator *iter)
87 struct trace_entry *entry = iter->ent; 154 struct trace_entry *entry = iter->ent;
88 struct trace_seq *seq = &iter->seq; 155 struct trace_seq *seq = &iter->seq;
89 struct hw_branch_entry *it; 156 struct hw_branch_entry *it;
157 unsigned long symflags = TRACE_ITER_SYM_OFFSET;
90 158
91 trace_assign_type(it, entry); 159 trace_assign_type(it, entry);
92 160
93 if (entry->type == TRACE_HW_BRANCHES) { 161 if (entry->type == TRACE_HW_BRANCHES) {
94 if (trace_seq_printf(seq, "%4d ", entry->cpu) && 162 if (trace_seq_printf(seq, "%4d ", entry->cpu) &&
95 trace_seq_printf(seq, "0x%016llx -> 0x%016llx ", 163 seq_print_ip_sym(seq, it->to, symflags) &&
96 it->from, it->to) && 164 trace_seq_printf(seq, "\t <- ") &&
97 (!it->from || 165 seq_print_ip_sym(seq, it->from, symflags) &&
98 seq_print_ip_sym(seq, it->from, /* sym_flags = */ 0)) &&
99 trace_seq_printf(seq, "\n")) 166 trace_seq_printf(seq, "\n"))
100 return TRACE_TYPE_HANDLED; 167 return TRACE_TYPE_HANDLED;
101 return TRACE_TYPE_PARTIAL_LINE;; 168 return TRACE_TYPE_PARTIAL_LINE;;
@@ -103,26 +170,42 @@ static enum print_line_t bts_trace_print_line(struct trace_iterator *iter)
103 return TRACE_TYPE_UNHANDLED; 170 return TRACE_TYPE_UNHANDLED;
104} 171}
105 172
106void trace_hw_branch(struct trace_array *tr, u64 from, u64 to) 173void trace_hw_branch(u64 from, u64 to)
107{ 174{
175 struct trace_array *tr = hw_branch_trace;
108 struct ring_buffer_event *event; 176 struct ring_buffer_event *event;
109 struct hw_branch_entry *entry; 177 struct hw_branch_entry *entry;
110 unsigned long irq; 178 unsigned long irq1, irq2;
179 int cpu;
111 180
112 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry), &irq); 181 if (unlikely(!tr))
113 if (!event)
114 return; 182 return;
183
184 if (unlikely(!trace_hw_branches_enabled))
185 return;
186
187 local_irq_save(irq1);
188 cpu = raw_smp_processor_id();
189 if (atomic_inc_return(&tr->data[cpu]->disabled) != 1)
190 goto out;
191
192 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry), &irq2);
193 if (!event)
194 goto out;
115 entry = ring_buffer_event_data(event); 195 entry = ring_buffer_event_data(event);
116 tracing_generic_entry_update(&entry->ent, 0, from); 196 tracing_generic_entry_update(&entry->ent, 0, from);
117 entry->ent.type = TRACE_HW_BRANCHES; 197 entry->ent.type = TRACE_HW_BRANCHES;
118 entry->ent.cpu = smp_processor_id(); 198 entry->ent.cpu = cpu;
119 entry->from = from; 199 entry->from = from;
120 entry->to = to; 200 entry->to = to;
121 ring_buffer_unlock_commit(tr->buffer, event, irq); 201 ring_buffer_unlock_commit(tr->buffer, event, irq2);
202
203 out:
204 atomic_dec(&tr->data[cpu]->disabled);
205 local_irq_restore(irq1);
122} 206}
123 207
124static void trace_bts_at(struct trace_array *tr, 208static void trace_bts_at(const struct bts_trace *trace, void *at)
125 const struct bts_trace *trace, void *at)
126{ 209{
127 struct bts_struct bts; 210 struct bts_struct bts;
128 int err = 0; 211 int err = 0;
@@ -137,18 +220,29 @@ static void trace_bts_at(struct trace_array *tr,
137 220
138 switch (bts.qualifier) { 221 switch (bts.qualifier) {
139 case BTS_BRANCH: 222 case BTS_BRANCH:
140 trace_hw_branch(tr, bts.variant.lbr.from, bts.variant.lbr.to); 223 trace_hw_branch(bts.variant.lbr.from, bts.variant.lbr.to);
141 break; 224 break;
142 } 225 }
143} 226}
144 227
228/*
229 * Collect the trace on the current cpu and write it into the ftrace buffer.
230 *
231 * pre: bts_tracer_mutex must be locked
232 */
145static void trace_bts_cpu(void *arg) 233static void trace_bts_cpu(void *arg)
146{ 234{
147 struct trace_array *tr = (struct trace_array *) arg; 235 struct trace_array *tr = (struct trace_array *) arg;
148 const struct bts_trace *trace; 236 const struct bts_trace *trace;
149 unsigned char *at; 237 unsigned char *at;
150 238
151 if (!this_tracer) 239 if (unlikely(!tr))
240 return;
241
242 if (unlikely(atomic_read(&tr->data[raw_smp_processor_id()]->disabled)))
243 return;
244
245 if (unlikely(!this_tracer))
152 return; 246 return;
153 247
154 ds_suspend_bts(this_tracer); 248 ds_suspend_bts(this_tracer);
@@ -158,11 +252,11 @@ static void trace_bts_cpu(void *arg)
158 252
159 for (at = trace->ds.top; (void *)at < trace->ds.end; 253 for (at = trace->ds.top; (void *)at < trace->ds.end;
160 at += trace->ds.size) 254 at += trace->ds.size)
161 trace_bts_at(tr, trace, at); 255 trace_bts_at(trace, at);
162 256
163 for (at = trace->ds.begin; (void *)at < trace->ds.top; 257 for (at = trace->ds.begin; (void *)at < trace->ds.top;
164 at += trace->ds.size) 258 at += trace->ds.size)
165 trace_bts_at(tr, trace, at); 259 trace_bts_at(trace, at);
166 260
167out: 261out:
168 ds_resume_bts(this_tracer); 262 ds_resume_bts(this_tracer);
@@ -170,22 +264,38 @@ out:
170 264
171static void trace_bts_prepare(struct trace_iterator *iter) 265static void trace_bts_prepare(struct trace_iterator *iter)
172{ 266{
173 int cpu; 267 mutex_lock(&bts_tracer_mutex);
268
269 on_each_cpu(trace_bts_cpu, iter->tr, 1);
270
271 mutex_unlock(&bts_tracer_mutex);
272}
273
274static void trace_bts_close(struct trace_iterator *iter)
275{
276 tracing_reset_online_cpus(iter->tr);
277}
278
279void trace_hw_branch_oops(void)
280{
281 mutex_lock(&bts_tracer_mutex);
282
283 trace_bts_cpu(hw_branch_trace);
174 284
175 for_each_cpu(cpu, cpu_possible_mask) 285 mutex_unlock(&bts_tracer_mutex);
176 smp_call_function_single(cpu, trace_bts_cpu, iter->tr, 1);
177} 286}
178 287
179struct tracer bts_tracer __read_mostly = 288struct tracer bts_tracer __read_mostly =
180{ 289{
181 .name = "hw-branch-tracer", 290 .name = "hw-branch-tracer",
182 .init = bts_trace_init, 291 .init = bts_trace_init,
183 .reset = bts_trace_stop, 292 .reset = bts_trace_reset,
184 .print_header = bts_trace_print_header, 293 .print_header = bts_trace_print_header,
185 .print_line = bts_trace_print_line, 294 .print_line = bts_trace_print_line,
186 .start = bts_trace_start, 295 .start = bts_trace_start,
187 .stop = bts_trace_stop, 296 .stop = bts_trace_stop,
188 .open = trace_bts_prepare 297 .open = trace_bts_prepare,
298 .close = trace_bts_close
189}; 299};
190 300
191__init static int init_bts_trace(void) 301__init static int init_bts_trace(void)
diff --git a/kernel/trace/trace_irqsoff.c b/kernel/trace/trace_irqsoff.c
index 62a78d943534..ed344b022a14 100644
--- a/kernel/trace/trace_irqsoff.c
+++ b/kernel/trace/trace_irqsoff.c
@@ -353,28 +353,18 @@ void trace_preempt_off(unsigned long a0, unsigned long a1)
353} 353}
354#endif /* CONFIG_PREEMPT_TRACER */ 354#endif /* CONFIG_PREEMPT_TRACER */
355 355
356/*
357 * save_tracer_enabled is used to save the state of the tracer_enabled
358 * variable when we disable it when we open a trace output file.
359 */
360static int save_tracer_enabled;
361
362static void start_irqsoff_tracer(struct trace_array *tr) 356static void start_irqsoff_tracer(struct trace_array *tr)
363{ 357{
364 register_ftrace_function(&trace_ops); 358 register_ftrace_function(&trace_ops);
365 if (tracing_is_enabled()) { 359 if (tracing_is_enabled())
366 tracer_enabled = 1; 360 tracer_enabled = 1;
367 save_tracer_enabled = 1; 361 else
368 } else {
369 tracer_enabled = 0; 362 tracer_enabled = 0;
370 save_tracer_enabled = 0;
371 }
372} 363}
373 364
374static void stop_irqsoff_tracer(struct trace_array *tr) 365static void stop_irqsoff_tracer(struct trace_array *tr)
375{ 366{
376 tracer_enabled = 0; 367 tracer_enabled = 0;
377 save_tracer_enabled = 0;
378 unregister_ftrace_function(&trace_ops); 368 unregister_ftrace_function(&trace_ops);
379} 369}
380 370
@@ -395,25 +385,11 @@ static void irqsoff_tracer_reset(struct trace_array *tr)
395static void irqsoff_tracer_start(struct trace_array *tr) 385static void irqsoff_tracer_start(struct trace_array *tr)
396{ 386{
397 tracer_enabled = 1; 387 tracer_enabled = 1;
398 save_tracer_enabled = 1;
399} 388}
400 389
401static void irqsoff_tracer_stop(struct trace_array *tr) 390static void irqsoff_tracer_stop(struct trace_array *tr)
402{ 391{
403 tracer_enabled = 0; 392 tracer_enabled = 0;
404 save_tracer_enabled = 0;
405}
406
407static void irqsoff_tracer_open(struct trace_iterator *iter)
408{
409 /* stop the trace while dumping */
410 tracer_enabled = 0;
411}
412
413static void irqsoff_tracer_close(struct trace_iterator *iter)
414{
415 /* restart tracing */
416 tracer_enabled = save_tracer_enabled;
417} 393}
418 394
419#ifdef CONFIG_IRQSOFF_TRACER 395#ifdef CONFIG_IRQSOFF_TRACER
@@ -431,8 +407,6 @@ static struct tracer irqsoff_tracer __read_mostly =
431 .reset = irqsoff_tracer_reset, 407 .reset = irqsoff_tracer_reset,
432 .start = irqsoff_tracer_start, 408 .start = irqsoff_tracer_start,
433 .stop = irqsoff_tracer_stop, 409 .stop = irqsoff_tracer_stop,
434 .open = irqsoff_tracer_open,
435 .close = irqsoff_tracer_close,
436 .print_max = 1, 410 .print_max = 1,
437#ifdef CONFIG_FTRACE_SELFTEST 411#ifdef CONFIG_FTRACE_SELFTEST
438 .selftest = trace_selftest_startup_irqsoff, 412 .selftest = trace_selftest_startup_irqsoff,
@@ -459,8 +433,6 @@ static struct tracer preemptoff_tracer __read_mostly =
459 .reset = irqsoff_tracer_reset, 433 .reset = irqsoff_tracer_reset,
460 .start = irqsoff_tracer_start, 434 .start = irqsoff_tracer_start,
461 .stop = irqsoff_tracer_stop, 435 .stop = irqsoff_tracer_stop,
462 .open = irqsoff_tracer_open,
463 .close = irqsoff_tracer_close,
464 .print_max = 1, 436 .print_max = 1,
465#ifdef CONFIG_FTRACE_SELFTEST 437#ifdef CONFIG_FTRACE_SELFTEST
466 .selftest = trace_selftest_startup_preemptoff, 438 .selftest = trace_selftest_startup_preemptoff,
@@ -489,8 +461,6 @@ static struct tracer preemptirqsoff_tracer __read_mostly =
489 .reset = irqsoff_tracer_reset, 461 .reset = irqsoff_tracer_reset,
490 .start = irqsoff_tracer_start, 462 .start = irqsoff_tracer_start,
491 .stop = irqsoff_tracer_stop, 463 .stop = irqsoff_tracer_stop,
492 .open = irqsoff_tracer_open,
493 .close = irqsoff_tracer_close,
494 .print_max = 1, 464 .print_max = 1,
495#ifdef CONFIG_FTRACE_SELFTEST 465#ifdef CONFIG_FTRACE_SELFTEST
496 .selftest = trace_selftest_startup_preemptirqsoff, 466 .selftest = trace_selftest_startup_preemptirqsoff,
diff --git a/kernel/trace/trace_mmiotrace.c b/kernel/trace/trace_mmiotrace.c
index fffcb069f1dc..ec78e244242e 100644
--- a/kernel/trace/trace_mmiotrace.c
+++ b/kernel/trace/trace_mmiotrace.c
@@ -9,8 +9,10 @@
9#include <linux/kernel.h> 9#include <linux/kernel.h>
10#include <linux/mmiotrace.h> 10#include <linux/mmiotrace.h>
11#include <linux/pci.h> 11#include <linux/pci.h>
12#include <asm/atomic.h>
12 13
13#include "trace.h" 14#include "trace.h"
15#include "trace_output.h"
14 16
15struct header_iter { 17struct header_iter {
16 struct pci_dev *dev; 18 struct pci_dev *dev;
@@ -19,6 +21,7 @@ struct header_iter {
19static struct trace_array *mmio_trace_array; 21static struct trace_array *mmio_trace_array;
20static bool overrun_detected; 22static bool overrun_detected;
21static unsigned long prev_overruns; 23static unsigned long prev_overruns;
24static atomic_t dropped_count;
22 25
23static void mmio_reset_data(struct trace_array *tr) 26static void mmio_reset_data(struct trace_array *tr)
24{ 27{
@@ -121,11 +124,11 @@ static void mmio_close(struct trace_iterator *iter)
121 124
122static unsigned long count_overruns(struct trace_iterator *iter) 125static unsigned long count_overruns(struct trace_iterator *iter)
123{ 126{
124 unsigned long cnt = 0; 127 unsigned long cnt = atomic_xchg(&dropped_count, 0);
125 unsigned long over = ring_buffer_overruns(iter->tr->buffer); 128 unsigned long over = ring_buffer_overruns(iter->tr->buffer);
126 129
127 if (over > prev_overruns) 130 if (over > prev_overruns)
128 cnt = over - prev_overruns; 131 cnt += over - prev_overruns;
129 prev_overruns = over; 132 prev_overruns = over;
130 return cnt; 133 return cnt;
131} 134}
@@ -181,21 +184,22 @@ static enum print_line_t mmio_print_rw(struct trace_iterator *iter)
181 switch (rw->opcode) { 184 switch (rw->opcode) {
182 case MMIO_READ: 185 case MMIO_READ:
183 ret = trace_seq_printf(s, 186 ret = trace_seq_printf(s,
184 "R %d %lu.%06lu %d 0x%llx 0x%lx 0x%lx %d\n", 187 "R %d %u.%06lu %d 0x%llx 0x%lx 0x%lx %d\n",
185 rw->width, secs, usec_rem, rw->map_id, 188 rw->width, secs, usec_rem, rw->map_id,
186 (unsigned long long)rw->phys, 189 (unsigned long long)rw->phys,
187 rw->value, rw->pc, 0); 190 rw->value, rw->pc, 0);
188 break; 191 break;
189 case MMIO_WRITE: 192 case MMIO_WRITE:
190 ret = trace_seq_printf(s, 193 ret = trace_seq_printf(s,
191 "W %d %lu.%06lu %d 0x%llx 0x%lx 0x%lx %d\n", 194 "W %d %u.%06lu %d 0x%llx 0x%lx 0x%lx %d\n",
192 rw->width, secs, usec_rem, rw->map_id, 195 rw->width, secs, usec_rem, rw->map_id,
193 (unsigned long long)rw->phys, 196 (unsigned long long)rw->phys,
194 rw->value, rw->pc, 0); 197 rw->value, rw->pc, 0);
195 break; 198 break;
196 case MMIO_UNKNOWN_OP: 199 case MMIO_UNKNOWN_OP:
197 ret = trace_seq_printf(s, 200 ret = trace_seq_printf(s,
198 "UNKNOWN %lu.%06lu %d 0x%llx %02x,%02x,%02x 0x%lx %d\n", 201 "UNKNOWN %u.%06lu %d 0x%llx %02lx,%02lx,"
202 "%02lx 0x%lx %d\n",
199 secs, usec_rem, rw->map_id, 203 secs, usec_rem, rw->map_id,
200 (unsigned long long)rw->phys, 204 (unsigned long long)rw->phys,
201 (rw->value >> 16) & 0xff, (rw->value >> 8) & 0xff, 205 (rw->value >> 16) & 0xff, (rw->value >> 8) & 0xff,
@@ -227,14 +231,14 @@ static enum print_line_t mmio_print_map(struct trace_iterator *iter)
227 switch (m->opcode) { 231 switch (m->opcode) {
228 case MMIO_PROBE: 232 case MMIO_PROBE:
229 ret = trace_seq_printf(s, 233 ret = trace_seq_printf(s,
230 "MAP %lu.%06lu %d 0x%llx 0x%lx 0x%lx 0x%lx %d\n", 234 "MAP %u.%06lu %d 0x%llx 0x%lx 0x%lx 0x%lx %d\n",
231 secs, usec_rem, m->map_id, 235 secs, usec_rem, m->map_id,
232 (unsigned long long)m->phys, m->virt, m->len, 236 (unsigned long long)m->phys, m->virt, m->len,
233 0UL, 0); 237 0UL, 0);
234 break; 238 break;
235 case MMIO_UNPROBE: 239 case MMIO_UNPROBE:
236 ret = trace_seq_printf(s, 240 ret = trace_seq_printf(s,
237 "UNMAP %lu.%06lu %d 0x%lx %d\n", 241 "UNMAP %u.%06lu %d 0x%lx %d\n",
238 secs, usec_rem, m->map_id, 0UL, 0); 242 secs, usec_rem, m->map_id, 0UL, 0);
239 break; 243 break;
240 default: 244 default:
@@ -258,13 +262,10 @@ static enum print_line_t mmio_print_mark(struct trace_iterator *iter)
258 int ret; 262 int ret;
259 263
260 /* The trailing newline must be in the message. */ 264 /* The trailing newline must be in the message. */
261 ret = trace_seq_printf(s, "MARK %lu.%06lu %s", secs, usec_rem, msg); 265 ret = trace_seq_printf(s, "MARK %u.%06lu %s", secs, usec_rem, msg);
262 if (!ret) 266 if (!ret)
263 return TRACE_TYPE_PARTIAL_LINE; 267 return TRACE_TYPE_PARTIAL_LINE;
264 268
265 if (entry->flags & TRACE_FLAG_CONT)
266 trace_seq_print_cont(s, iter);
267
268 return TRACE_TYPE_HANDLED; 269 return TRACE_TYPE_HANDLED;
269} 270}
270 271
@@ -310,8 +311,10 @@ static void __trace_mmiotrace_rw(struct trace_array *tr,
310 311
311 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry), 312 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
312 &irq_flags); 313 &irq_flags);
313 if (!event) 314 if (!event) {
315 atomic_inc(&dropped_count);
314 return; 316 return;
317 }
315 entry = ring_buffer_event_data(event); 318 entry = ring_buffer_event_data(event);
316 tracing_generic_entry_update(&entry->ent, 0, preempt_count()); 319 tracing_generic_entry_update(&entry->ent, 0, preempt_count());
317 entry->ent.type = TRACE_MMIO_RW; 320 entry->ent.type = TRACE_MMIO_RW;
@@ -338,8 +341,10 @@ static void __trace_mmiotrace_map(struct trace_array *tr,
338 341
339 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry), 342 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
340 &irq_flags); 343 &irq_flags);
341 if (!event) 344 if (!event) {
345 atomic_inc(&dropped_count);
342 return; 346 return;
347 }
343 entry = ring_buffer_event_data(event); 348 entry = ring_buffer_event_data(event);
344 tracing_generic_entry_update(&entry->ent, 0, preempt_count()); 349 tracing_generic_entry_update(&entry->ent, 0, preempt_count());
345 entry->ent.type = TRACE_MMIO_MAP; 350 entry->ent.type = TRACE_MMIO_MAP;
diff --git a/kernel/trace/trace_nop.c b/kernel/trace/trace_nop.c
index b9767acd30ac..087b6cbf4ea5 100644
--- a/kernel/trace/trace_nop.c
+++ b/kernel/trace/trace_nop.c
@@ -47,12 +47,8 @@ static void stop_nop_trace(struct trace_array *tr)
47 47
48static int nop_trace_init(struct trace_array *tr) 48static int nop_trace_init(struct trace_array *tr)
49{ 49{
50 int cpu;
51 ctx_trace = tr; 50 ctx_trace = tr;
52 51 tracing_reset_online_cpus(tr);
53 for_each_online_cpu(cpu)
54 tracing_reset(tr, cpu);
55
56 start_nop_trace(tr); 52 start_nop_trace(tr);
57 return 0; 53 return 0;
58} 54}
diff --git a/kernel/trace/trace_output.c b/kernel/trace/trace_output.c
new file mode 100644
index 000000000000..1a4e144a9f8f
--- /dev/null
+++ b/kernel/trace/trace_output.c
@@ -0,0 +1,829 @@
1/*
2 * trace_output.c
3 *
4 * Copyright (C) 2008 Red Hat Inc, Steven Rostedt <srostedt@redhat.com>
5 *
6 */
7
8#include <linux/module.h>
9#include <linux/mutex.h>
10#include <linux/ftrace.h>
11
12#include "trace_output.h"
13
14/* must be a power of 2 */
15#define EVENT_HASHSIZE 128
16
17static DEFINE_MUTEX(trace_event_mutex);
18static struct hlist_head event_hash[EVENT_HASHSIZE] __read_mostly;
19
20static int next_event_type = __TRACE_LAST_TYPE + 1;
21
22/**
23 * trace_seq_printf - sequence printing of trace information
24 * @s: trace sequence descriptor
25 * @fmt: printf format string
26 *
27 * The tracer may use either sequence operations or its own
28 * copy to user routines. To simplify formating of a trace
29 * trace_seq_printf is used to store strings into a special
30 * buffer (@s). Then the output may be either used by
31 * the sequencer or pulled into another buffer.
32 */
33int
34trace_seq_printf(struct trace_seq *s, const char *fmt, ...)
35{
36 int len = (PAGE_SIZE - 1) - s->len;
37 va_list ap;
38 int ret;
39
40 if (!len)
41 return 0;
42
43 va_start(ap, fmt);
44 ret = vsnprintf(s->buffer + s->len, len, fmt, ap);
45 va_end(ap);
46
47 /* If we can't write it all, don't bother writing anything */
48 if (ret >= len)
49 return 0;
50
51 s->len += ret;
52
53 return len;
54}
55
56/**
57 * trace_seq_puts - trace sequence printing of simple string
58 * @s: trace sequence descriptor
59 * @str: simple string to record
60 *
61 * The tracer may use either the sequence operations or its own
62 * copy to user routines. This function records a simple string
63 * into a special buffer (@s) for later retrieval by a sequencer
64 * or other mechanism.
65 */
66int trace_seq_puts(struct trace_seq *s, const char *str)
67{
68 int len = strlen(str);
69
70 if (len > ((PAGE_SIZE - 1) - s->len))
71 return 0;
72
73 memcpy(s->buffer + s->len, str, len);
74 s->len += len;
75
76 return len;
77}
78
79int trace_seq_putc(struct trace_seq *s, unsigned char c)
80{
81 if (s->len >= (PAGE_SIZE - 1))
82 return 0;
83
84 s->buffer[s->len++] = c;
85
86 return 1;
87}
88
89int trace_seq_putmem(struct trace_seq *s, void *mem, size_t len)
90{
91 if (len > ((PAGE_SIZE - 1) - s->len))
92 return 0;
93
94 memcpy(s->buffer + s->len, mem, len);
95 s->len += len;
96
97 return len;
98}
99
100int trace_seq_putmem_hex(struct trace_seq *s, void *mem, size_t len)
101{
102 unsigned char hex[HEX_CHARS];
103 unsigned char *data = mem;
104 int i, j;
105
106#ifdef __BIG_ENDIAN
107 for (i = 0, j = 0; i < len; i++) {
108#else
109 for (i = len-1, j = 0; i >= 0; i--) {
110#endif
111 hex[j++] = hex_asc_hi(data[i]);
112 hex[j++] = hex_asc_lo(data[i]);
113 }
114 hex[j++] = ' ';
115
116 return trace_seq_putmem(s, hex, j);
117}
118
119int trace_seq_path(struct trace_seq *s, struct path *path)
120{
121 unsigned char *p;
122
123 if (s->len >= (PAGE_SIZE - 1))
124 return 0;
125 p = d_path(path, s->buffer + s->len, PAGE_SIZE - s->len);
126 if (!IS_ERR(p)) {
127 p = mangle_path(s->buffer + s->len, p, "\n");
128 if (p) {
129 s->len = p - s->buffer;
130 return 1;
131 }
132 } else {
133 s->buffer[s->len++] = '?';
134 return 1;
135 }
136
137 return 0;
138}
139
140#ifdef CONFIG_KRETPROBES
141static inline const char *kretprobed(const char *name)
142{
143 static const char tramp_name[] = "kretprobe_trampoline";
144 int size = sizeof(tramp_name);
145
146 if (strncmp(tramp_name, name, size) == 0)
147 return "[unknown/kretprobe'd]";
148 return name;
149}
150#else
151static inline const char *kretprobed(const char *name)
152{
153 return name;
154}
155#endif /* CONFIG_KRETPROBES */
156
157static int
158seq_print_sym_short(struct trace_seq *s, const char *fmt, unsigned long address)
159{
160#ifdef CONFIG_KALLSYMS
161 char str[KSYM_SYMBOL_LEN];
162 const char *name;
163
164 kallsyms_lookup(address, NULL, NULL, NULL, str);
165
166 name = kretprobed(str);
167
168 return trace_seq_printf(s, fmt, name);
169#endif
170 return 1;
171}
172
173static int
174seq_print_sym_offset(struct trace_seq *s, const char *fmt,
175 unsigned long address)
176{
177#ifdef CONFIG_KALLSYMS
178 char str[KSYM_SYMBOL_LEN];
179 const char *name;
180
181 sprint_symbol(str, address);
182 name = kretprobed(str);
183
184 return trace_seq_printf(s, fmt, name);
185#endif
186 return 1;
187}
188
189#ifndef CONFIG_64BIT
190# define IP_FMT "%08lx"
191#else
192# define IP_FMT "%016lx"
193#endif
194
195int seq_print_user_ip(struct trace_seq *s, struct mm_struct *mm,
196 unsigned long ip, unsigned long sym_flags)
197{
198 struct file *file = NULL;
199 unsigned long vmstart = 0;
200 int ret = 1;
201
202 if (mm) {
203 const struct vm_area_struct *vma;
204
205 down_read(&mm->mmap_sem);
206 vma = find_vma(mm, ip);
207 if (vma) {
208 file = vma->vm_file;
209 vmstart = vma->vm_start;
210 }
211 if (file) {
212 ret = trace_seq_path(s, &file->f_path);
213 if (ret)
214 ret = trace_seq_printf(s, "[+0x%lx]",
215 ip - vmstart);
216 }
217 up_read(&mm->mmap_sem);
218 }
219 if (ret && ((sym_flags & TRACE_ITER_SYM_ADDR) || !file))
220 ret = trace_seq_printf(s, " <" IP_FMT ">", ip);
221 return ret;
222}
223
224int
225seq_print_userip_objs(const struct userstack_entry *entry, struct trace_seq *s,
226 unsigned long sym_flags)
227{
228 struct mm_struct *mm = NULL;
229 int ret = 1;
230 unsigned int i;
231
232 if (trace_flags & TRACE_ITER_SYM_USEROBJ) {
233 struct task_struct *task;
234 /*
235 * we do the lookup on the thread group leader,
236 * since individual threads might have already quit!
237 */
238 rcu_read_lock();
239 task = find_task_by_vpid(entry->ent.tgid);
240 if (task)
241 mm = get_task_mm(task);
242 rcu_read_unlock();
243 }
244
245 for (i = 0; i < FTRACE_STACK_ENTRIES; i++) {
246 unsigned long ip = entry->caller[i];
247
248 if (ip == ULONG_MAX || !ret)
249 break;
250 if (i && ret)
251 ret = trace_seq_puts(s, " <- ");
252 if (!ip) {
253 if (ret)
254 ret = trace_seq_puts(s, "??");
255 continue;
256 }
257 if (!ret)
258 break;
259 if (ret)
260 ret = seq_print_user_ip(s, mm, ip, sym_flags);
261 }
262
263 if (mm)
264 mmput(mm);
265 return ret;
266}
267
268int
269seq_print_ip_sym(struct trace_seq *s, unsigned long ip, unsigned long sym_flags)
270{
271 int ret;
272
273 if (!ip)
274 return trace_seq_printf(s, "0");
275
276 if (sym_flags & TRACE_ITER_SYM_OFFSET)
277 ret = seq_print_sym_offset(s, "%s", ip);
278 else
279 ret = seq_print_sym_short(s, "%s", ip);
280
281 if (!ret)
282 return 0;
283
284 if (sym_flags & TRACE_ITER_SYM_ADDR)
285 ret = trace_seq_printf(s, " <" IP_FMT ">", ip);
286 return ret;
287}
288
289static const char state_to_char[] = TASK_STATE_TO_CHAR_STR;
290
291static int task_state_char(unsigned long state)
292{
293 int bit = state ? __ffs(state) + 1 : 0;
294
295 return bit < sizeof(state_to_char) - 1 ? state_to_char[bit] : '?';
296}
297
298/**
299 * ftrace_find_event - find a registered event
300 * @type: the type of event to look for
301 *
302 * Returns an event of type @type otherwise NULL
303 */
304struct trace_event *ftrace_find_event(int type)
305{
306 struct trace_event *event;
307 struct hlist_node *n;
308 unsigned key;
309
310 key = type & (EVENT_HASHSIZE - 1);
311
312 hlist_for_each_entry_rcu(event, n, &event_hash[key], node) {
313 if (event->type == type)
314 return event;
315 }
316
317 return NULL;
318}
319
320/**
321 * register_ftrace_event - register output for an event type
322 * @event: the event type to register
323 *
324 * Event types are stored in a hash and this hash is used to
325 * find a way to print an event. If the @event->type is set
326 * then it will use that type, otherwise it will assign a
327 * type to use.
328 *
329 * If you assign your own type, please make sure it is added
330 * to the trace_type enum in trace.h, to avoid collisions
331 * with the dynamic types.
332 *
333 * Returns the event type number or zero on error.
334 */
335int register_ftrace_event(struct trace_event *event)
336{
337 unsigned key;
338 int ret = 0;
339
340 mutex_lock(&trace_event_mutex);
341
342 if (!event->type)
343 event->type = next_event_type++;
344 else if (event->type > __TRACE_LAST_TYPE) {
345 printk(KERN_WARNING "Need to add type to trace.h\n");
346 WARN_ON(1);
347 }
348
349 if (ftrace_find_event(event->type))
350 goto out;
351
352 key = event->type & (EVENT_HASHSIZE - 1);
353
354 hlist_add_head_rcu(&event->node, &event_hash[key]);
355
356 ret = event->type;
357 out:
358 mutex_unlock(&trace_event_mutex);
359
360 return ret;
361}
362
363/**
364 * unregister_ftrace_event - remove a no longer used event
365 * @event: the event to remove
366 */
367int unregister_ftrace_event(struct trace_event *event)
368{
369 mutex_lock(&trace_event_mutex);
370 hlist_del(&event->node);
371 mutex_unlock(&trace_event_mutex);
372
373 return 0;
374}
375
376/*
377 * Standard events
378 */
379
380int
381trace_nop_print(struct trace_seq *s, struct trace_entry *entry, int flags)
382{
383 return 0;
384}
385
386/* TRACE_FN */
387static int
388trace_fn_latency(struct trace_seq *s, struct trace_entry *entry, int flags)
389{
390 struct ftrace_entry *field;
391
392 trace_assign_type(field, entry);
393
394 if (!seq_print_ip_sym(s, field->ip, flags))
395 goto partial;
396 if (!trace_seq_puts(s, " ("))
397 goto partial;
398 if (!seq_print_ip_sym(s, field->parent_ip, flags))
399 goto partial;
400 if (!trace_seq_puts(s, ")\n"))
401 goto partial;
402
403 return 0;
404
405 partial:
406 return TRACE_TYPE_PARTIAL_LINE;
407}
408
409static int
410trace_fn_trace(struct trace_seq *s, struct trace_entry *entry, int flags)
411{
412 struct ftrace_entry *field;
413
414 trace_assign_type(field, entry);
415
416 if (!seq_print_ip_sym(s, field->ip, flags))
417 goto partial;
418
419 if ((flags & TRACE_ITER_PRINT_PARENT) && field->parent_ip) {
420 if (!trace_seq_printf(s, " <-"))
421 goto partial;
422 if (!seq_print_ip_sym(s,
423 field->parent_ip,
424 flags))
425 goto partial;
426 }
427 if (!trace_seq_printf(s, "\n"))
428 goto partial;
429
430 return 0;
431
432 partial:
433 return TRACE_TYPE_PARTIAL_LINE;
434}
435
436static int
437trace_fn_raw(struct trace_seq *s, struct trace_entry *entry, int flags)
438{
439 struct ftrace_entry *field;
440
441 trace_assign_type(field, entry);
442
443 if (!trace_seq_printf(s, "%lx %lx\n",
444 field->ip,
445 field->parent_ip))
446 return TRACE_TYPE_PARTIAL_LINE;
447
448 return 0;
449}
450
451static int
452trace_fn_hex(struct trace_seq *s, struct trace_entry *entry, int flags)
453{
454 struct ftrace_entry *field;
455
456 trace_assign_type(field, entry);
457
458 SEQ_PUT_HEX_FIELD_RET(s, field->ip);
459 SEQ_PUT_HEX_FIELD_RET(s, field->parent_ip);
460
461 return 0;
462}
463
464static int
465trace_fn_bin(struct trace_seq *s, struct trace_entry *entry, int flags)
466{
467 struct ftrace_entry *field;
468
469 trace_assign_type(field, entry);
470
471 SEQ_PUT_FIELD_RET(s, field->ip);
472 SEQ_PUT_FIELD_RET(s, field->parent_ip);
473
474 return 0;
475}
476
477static struct trace_event trace_fn_event = {
478 .type = TRACE_FN,
479 .trace = trace_fn_trace,
480 .latency_trace = trace_fn_latency,
481 .raw = trace_fn_raw,
482 .hex = trace_fn_hex,
483 .binary = trace_fn_bin,
484};
485
486/* TRACE_CTX an TRACE_WAKE */
487static int
488trace_ctxwake_print(struct trace_seq *s, struct trace_entry *entry, int flags,
489 char *delim)
490{
491 struct ctx_switch_entry *field;
492 char *comm;
493 int S, T;
494
495 trace_assign_type(field, entry);
496
497 T = task_state_char(field->next_state);
498 S = task_state_char(field->prev_state);
499 comm = trace_find_cmdline(field->next_pid);
500 if (!trace_seq_printf(s, " %5d:%3d:%c %s [%03d] %5d:%3d:%c %s\n",
501 field->prev_pid,
502 field->prev_prio,
503 S, delim,
504 field->next_cpu,
505 field->next_pid,
506 field->next_prio,
507 T, comm))
508 return TRACE_TYPE_PARTIAL_LINE;
509
510 return 0;
511}
512
513static int
514trace_ctx_print(struct trace_seq *s, struct trace_entry *entry, int flags)
515{
516 return trace_ctxwake_print(s, entry, flags, "==>");
517}
518
519static int
520trace_wake_print(struct trace_seq *s, struct trace_entry *entry, int flags)
521{
522 return trace_ctxwake_print(s, entry, flags, " +");
523}
524
525static int
526trace_ctxwake_raw(struct trace_seq *s, struct trace_entry *entry, int flags,
527 char S)
528{
529 struct ctx_switch_entry *field;
530 int T;
531
532 trace_assign_type(field, entry);
533
534 if (!S)
535 task_state_char(field->prev_state);
536 T = task_state_char(field->next_state);
537 if (!trace_seq_printf(s, "%d %d %c %d %d %d %c\n",
538 field->prev_pid,
539 field->prev_prio,
540 S,
541 field->next_cpu,
542 field->next_pid,
543 field->next_prio,
544 T))
545 return TRACE_TYPE_PARTIAL_LINE;
546
547 return 0;
548}
549
550static int
551trace_ctx_raw(struct trace_seq *s, struct trace_entry *entry, int flags)
552{
553 return trace_ctxwake_raw(s, entry, flags, 0);
554}
555
556static int
557trace_wake_raw(struct trace_seq *s, struct trace_entry *entry, int flags)
558{
559 return trace_ctxwake_raw(s, entry, flags, '+');
560}
561
562
563static int
564trace_ctxwake_hex(struct trace_seq *s, struct trace_entry *entry, int flags,
565 char S)
566{
567 struct ctx_switch_entry *field;
568 int T;
569
570 trace_assign_type(field, entry);
571
572 if (!S)
573 task_state_char(field->prev_state);
574 T = task_state_char(field->next_state);
575
576 SEQ_PUT_HEX_FIELD_RET(s, field->prev_pid);
577 SEQ_PUT_HEX_FIELD_RET(s, field->prev_prio);
578 SEQ_PUT_HEX_FIELD_RET(s, S);
579 SEQ_PUT_HEX_FIELD_RET(s, field->next_cpu);
580 SEQ_PUT_HEX_FIELD_RET(s, field->next_pid);
581 SEQ_PUT_HEX_FIELD_RET(s, field->next_prio);
582 SEQ_PUT_HEX_FIELD_RET(s, T);
583
584 return 0;
585}
586
587static int
588trace_ctx_hex(struct trace_seq *s, struct trace_entry *entry, int flags)
589{
590 return trace_ctxwake_hex(s, entry, flags, 0);
591}
592
593static int
594trace_wake_hex(struct trace_seq *s, struct trace_entry *entry, int flags)
595{
596 return trace_ctxwake_hex(s, entry, flags, '+');
597}
598
599static int
600trace_ctxwake_bin(struct trace_seq *s, struct trace_entry *entry, int flags)
601{
602 struct ctx_switch_entry *field;
603
604 trace_assign_type(field, entry);
605
606 SEQ_PUT_FIELD_RET(s, field->prev_pid);
607 SEQ_PUT_FIELD_RET(s, field->prev_prio);
608 SEQ_PUT_FIELD_RET(s, field->prev_state);
609 SEQ_PUT_FIELD_RET(s, field->next_pid);
610 SEQ_PUT_FIELD_RET(s, field->next_prio);
611 SEQ_PUT_FIELD_RET(s, field->next_state);
612
613 return 0;
614}
615
616static struct trace_event trace_ctx_event = {
617 .type = TRACE_CTX,
618 .trace = trace_ctx_print,
619 .latency_trace = trace_ctx_print,
620 .raw = trace_ctx_raw,
621 .hex = trace_ctx_hex,
622 .binary = trace_ctxwake_bin,
623};
624
625static struct trace_event trace_wake_event = {
626 .type = TRACE_WAKE,
627 .trace = trace_wake_print,
628 .latency_trace = trace_wake_print,
629 .raw = trace_wake_raw,
630 .hex = trace_wake_hex,
631 .binary = trace_ctxwake_bin,
632};
633
634/* TRACE_SPECIAL */
635static int
636trace_special_print(struct trace_seq *s, struct trace_entry *entry, int flags)
637{
638 struct special_entry *field;
639
640 trace_assign_type(field, entry);
641
642 if (!trace_seq_printf(s, "# %ld %ld %ld\n",
643 field->arg1,
644 field->arg2,
645 field->arg3))
646 return TRACE_TYPE_PARTIAL_LINE;
647
648 return 0;
649}
650
651static int
652trace_special_hex(struct trace_seq *s, struct trace_entry *entry, int flags)
653{
654 struct special_entry *field;
655
656 trace_assign_type(field, entry);
657
658 SEQ_PUT_HEX_FIELD_RET(s, field->arg1);
659 SEQ_PUT_HEX_FIELD_RET(s, field->arg2);
660 SEQ_PUT_HEX_FIELD_RET(s, field->arg3);
661
662 return 0;
663}
664
665static int
666trace_special_bin(struct trace_seq *s, struct trace_entry *entry, int flags)
667{
668 struct special_entry *field;
669
670 trace_assign_type(field, entry);
671
672 SEQ_PUT_FIELD_RET(s, field->arg1);
673 SEQ_PUT_FIELD_RET(s, field->arg2);
674 SEQ_PUT_FIELD_RET(s, field->arg3);
675
676 return 0;
677}
678
679static struct trace_event trace_special_event = {
680 .type = TRACE_SPECIAL,
681 .trace = trace_special_print,
682 .latency_trace = trace_special_print,
683 .raw = trace_special_print,
684 .hex = trace_special_hex,
685 .binary = trace_special_bin,
686};
687
688/* TRACE_STACK */
689
690static int
691trace_stack_print(struct trace_seq *s, struct trace_entry *entry, int flags)
692{
693 struct stack_entry *field;
694 int i;
695
696 trace_assign_type(field, entry);
697
698 for (i = 0; i < FTRACE_STACK_ENTRIES; i++) {
699 if (i) {
700 if (!trace_seq_puts(s, " <= "))
701 goto partial;
702
703 if (!seq_print_ip_sym(s, field->caller[i], flags))
704 goto partial;
705 }
706 if (!trace_seq_puts(s, "\n"))
707 goto partial;
708 }
709
710 return 0;
711
712 partial:
713 return TRACE_TYPE_PARTIAL_LINE;
714}
715
716static struct trace_event trace_stack_event = {
717 .type = TRACE_STACK,
718 .trace = trace_stack_print,
719 .latency_trace = trace_stack_print,
720 .raw = trace_special_print,
721 .hex = trace_special_hex,
722 .binary = trace_special_bin,
723};
724
725/* TRACE_USER_STACK */
726static int
727trace_user_stack_print(struct trace_seq *s, struct trace_entry *entry,
728 int flags)
729{
730 struct userstack_entry *field;
731
732 trace_assign_type(field, entry);
733
734 if (!seq_print_userip_objs(field, s, flags))
735 goto partial;
736
737 if (!trace_seq_putc(s, '\n'))
738 goto partial;
739
740 return 0;
741
742 partial:
743 return TRACE_TYPE_PARTIAL_LINE;
744}
745
746static struct trace_event trace_user_stack_event = {
747 .type = TRACE_USER_STACK,
748 .trace = trace_user_stack_print,
749 .latency_trace = trace_user_stack_print,
750 .raw = trace_special_print,
751 .hex = trace_special_hex,
752 .binary = trace_special_bin,
753};
754
755/* TRACE_PRINT */
756static int
757trace_print_print(struct trace_seq *s, struct trace_entry *entry, int flags)
758{
759 struct print_entry *field;
760
761 trace_assign_type(field, entry);
762
763 if (!seq_print_ip_sym(s, field->ip, flags))
764 goto partial;
765
766 if (!trace_seq_printf(s, ": %s", field->buf))
767 goto partial;
768
769 return 0;
770
771 partial:
772 return TRACE_TYPE_PARTIAL_LINE;
773}
774
775static int
776trace_print_raw(struct trace_seq *s, struct trace_entry *entry, int flags)
777{
778 struct print_entry *field;
779
780 trace_assign_type(field, entry);
781
782 if (!trace_seq_printf(s, "# %lx %s", field->ip, field->buf))
783 goto partial;
784
785 return 0;
786
787 partial:
788 return TRACE_TYPE_PARTIAL_LINE;
789}
790
791static struct trace_event trace_print_event = {
792 .type = TRACE_PRINT,
793 .trace = trace_print_print,
794 .latency_trace = trace_print_print,
795 .raw = trace_print_raw,
796 .hex = trace_nop_print,
797 .binary = trace_nop_print,
798};
799
800static struct trace_event *events[] __initdata = {
801 &trace_fn_event,
802 &trace_ctx_event,
803 &trace_wake_event,
804 &trace_special_event,
805 &trace_stack_event,
806 &trace_user_stack_event,
807 &trace_print_event,
808 NULL
809};
810
811__init static int init_events(void)
812{
813 struct trace_event *event;
814 int i, ret;
815
816 for (i = 0; events[i]; i++) {
817 event = events[i];
818
819 ret = register_ftrace_event(event);
820 if (!ret) {
821 printk(KERN_WARNING "event %d failed to register\n",
822 event->type);
823 WARN_ON_ONCE(1);
824 }
825 }
826
827 return 0;
828}
829device_initcall(init_events);
diff --git a/kernel/trace/trace_output.h b/kernel/trace/trace_output.h
new file mode 100644
index 000000000000..1cbab5e3dc99
--- /dev/null
+++ b/kernel/trace/trace_output.h
@@ -0,0 +1,60 @@
1#ifndef __TRACE_EVENTS_H
2#define __TRACE_EVENTS_H
3
4#include "trace.h"
5
6typedef int (*trace_print_func)(struct trace_seq *s, struct trace_entry *entry,
7 int flags);
8
9struct trace_event {
10 struct hlist_node node;
11 int type;
12 trace_print_func trace;
13 trace_print_func latency_trace;
14 trace_print_func raw;
15 trace_print_func hex;
16 trace_print_func binary;
17};
18
19extern int trace_seq_printf(struct trace_seq *s, const char *fmt, ...)
20 __attribute__ ((format (printf, 2, 3)));
21extern int
22seq_print_ip_sym(struct trace_seq *s, unsigned long ip,
23 unsigned long sym_flags);
24extern ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf,
25 size_t cnt);
26int trace_seq_puts(struct trace_seq *s, const char *str);
27int trace_seq_putc(struct trace_seq *s, unsigned char c);
28int trace_seq_putmem(struct trace_seq *s, void *mem, size_t len);
29int trace_seq_putmem_hex(struct trace_seq *s, void *mem, size_t len);
30int trace_seq_path(struct trace_seq *s, struct path *path);
31int seq_print_userip_objs(const struct userstack_entry *entry,
32 struct trace_seq *s, unsigned long sym_flags);
33int seq_print_user_ip(struct trace_seq *s, struct mm_struct *mm,
34 unsigned long ip, unsigned long sym_flags);
35
36struct trace_event *ftrace_find_event(int type);
37int register_ftrace_event(struct trace_event *event);
38int unregister_ftrace_event(struct trace_event *event);
39
40int
41trace_nop_print(struct trace_seq *s, struct trace_entry *entry, int flags);
42
43#define MAX_MEMHEX_BYTES 8
44#define HEX_CHARS (MAX_MEMHEX_BYTES*2 + 1)
45
46#define SEQ_PUT_FIELD_RET(s, x) \
47do { \
48 if (!trace_seq_putmem(s, &(x), sizeof(x))) \
49 return TRACE_TYPE_PARTIAL_LINE; \
50} while (0)
51
52#define SEQ_PUT_HEX_FIELD_RET(s, x) \
53do { \
54 BUILD_BUG_ON(sizeof(x) > MAX_MEMHEX_BYTES); \
55 if (!trace_seq_putmem_hex(s, &(x), sizeof(x))) \
56 return TRACE_TYPE_PARTIAL_LINE; \
57} while (0)
58
59#endif
60
diff --git a/kernel/trace/trace_power.c b/kernel/trace/trace_power.c
index 7bda248daf55..faa6ab7a1f5c 100644
--- a/kernel/trace/trace_power.c
+++ b/kernel/trace/trace_power.c
@@ -16,6 +16,7 @@
16#include <linux/module.h> 16#include <linux/module.h>
17 17
18#include "trace.h" 18#include "trace.h"
19#include "trace_output.h"
19 20
20static struct trace_array *power_trace; 21static struct trace_array *power_trace;
21static int __read_mostly trace_power_enabled; 22static int __read_mostly trace_power_enabled;
diff --git a/kernel/trace/trace_sched_wakeup.c b/kernel/trace/trace_sched_wakeup.c
index 42ae1e77b6b3..a48c9b4b0c85 100644
--- a/kernel/trace/trace_sched_wakeup.c
+++ b/kernel/trace/trace_sched_wakeup.c
@@ -25,6 +25,7 @@ static int __read_mostly tracer_enabled;
25static struct task_struct *wakeup_task; 25static struct task_struct *wakeup_task;
26static int wakeup_cpu; 26static int wakeup_cpu;
27static unsigned wakeup_prio = -1; 27static unsigned wakeup_prio = -1;
28static int wakeup_rt;
28 29
29static raw_spinlock_t wakeup_lock = 30static raw_spinlock_t wakeup_lock =
30 (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED; 31 (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
@@ -152,6 +153,7 @@ probe_wakeup_sched_switch(struct rq *rq, struct task_struct *prev,
152 goto out_unlock; 153 goto out_unlock;
153 154
154 trace_function(wakeup_trace, data, CALLER_ADDR1, CALLER_ADDR2, flags, pc); 155 trace_function(wakeup_trace, data, CALLER_ADDR1, CALLER_ADDR2, flags, pc);
156 tracing_sched_switch_trace(wakeup_trace, data, prev, next, flags, pc);
155 157
156 /* 158 /*
157 * usecs conversion is slow so we try to delay the conversion 159 * usecs conversion is slow so we try to delay the conversion
@@ -182,13 +184,10 @@ out:
182 184
183static void __wakeup_reset(struct trace_array *tr) 185static void __wakeup_reset(struct trace_array *tr)
184{ 186{
185 struct trace_array_cpu *data;
186 int cpu; 187 int cpu;
187 188
188 for_each_possible_cpu(cpu) { 189 for_each_possible_cpu(cpu)
189 data = tr->data[cpu];
190 tracing_reset(tr, cpu); 190 tracing_reset(tr, cpu);
191 }
192 191
193 wakeup_cpu = -1; 192 wakeup_cpu = -1;
194 wakeup_prio = -1; 193 wakeup_prio = -1;
@@ -213,6 +212,7 @@ static void wakeup_reset(struct trace_array *tr)
213static void 212static void
214probe_wakeup(struct rq *rq, struct task_struct *p, int success) 213probe_wakeup(struct rq *rq, struct task_struct *p, int success)
215{ 214{
215 struct trace_array_cpu *data;
216 int cpu = smp_processor_id(); 216 int cpu = smp_processor_id();
217 unsigned long flags; 217 unsigned long flags;
218 long disabled; 218 long disabled;
@@ -224,7 +224,7 @@ probe_wakeup(struct rq *rq, struct task_struct *p, int success)
224 tracing_record_cmdline(p); 224 tracing_record_cmdline(p);
225 tracing_record_cmdline(current); 225 tracing_record_cmdline(current);
226 226
227 if (likely(!rt_task(p)) || 227 if ((wakeup_rt && !rt_task(p)) ||
228 p->prio >= wakeup_prio || 228 p->prio >= wakeup_prio ||
229 p->prio >= current->prio) 229 p->prio >= current->prio)
230 return; 230 return;
@@ -252,9 +252,12 @@ probe_wakeup(struct rq *rq, struct task_struct *p, int success)
252 252
253 local_save_flags(flags); 253 local_save_flags(flags);
254 254
255 wakeup_trace->data[wakeup_cpu]->preempt_timestamp = ftrace_now(cpu); 255 data = wakeup_trace->data[wakeup_cpu];
256 trace_function(wakeup_trace, wakeup_trace->data[wakeup_cpu], 256 data->preempt_timestamp = ftrace_now(cpu);
257 CALLER_ADDR1, CALLER_ADDR2, flags, pc); 257 tracing_sched_wakeup_trace(wakeup_trace, data, p, current,
258 flags, pc);
259 trace_function(wakeup_trace, data, CALLER_ADDR1, CALLER_ADDR2,
260 flags, pc);
258 261
259out_locked: 262out_locked:
260 __raw_spin_unlock(&wakeup_lock); 263 __raw_spin_unlock(&wakeup_lock);
@@ -262,12 +265,6 @@ out:
262 atomic_dec(&wakeup_trace->data[cpu]->disabled); 265 atomic_dec(&wakeup_trace->data[cpu]->disabled);
263} 266}
264 267
265/*
266 * save_tracer_enabled is used to save the state of the tracer_enabled
267 * variable when we disable it when we open a trace output file.
268 */
269static int save_tracer_enabled;
270
271static void start_wakeup_tracer(struct trace_array *tr) 268static void start_wakeup_tracer(struct trace_array *tr)
272{ 269{
273 int ret; 270 int ret;
@@ -306,13 +303,10 @@ static void start_wakeup_tracer(struct trace_array *tr)
306 303
307 register_ftrace_function(&trace_ops); 304 register_ftrace_function(&trace_ops);
308 305
309 if (tracing_is_enabled()) { 306 if (tracing_is_enabled())
310 tracer_enabled = 1; 307 tracer_enabled = 1;
311 save_tracer_enabled = 1; 308 else
312 } else {
313 tracer_enabled = 0; 309 tracer_enabled = 0;
314 save_tracer_enabled = 0;
315 }
316 310
317 return; 311 return;
318fail_deprobe_wake_new: 312fail_deprobe_wake_new:
@@ -324,14 +318,13 @@ fail_deprobe:
324static void stop_wakeup_tracer(struct trace_array *tr) 318static void stop_wakeup_tracer(struct trace_array *tr)
325{ 319{
326 tracer_enabled = 0; 320 tracer_enabled = 0;
327 save_tracer_enabled = 0;
328 unregister_ftrace_function(&trace_ops); 321 unregister_ftrace_function(&trace_ops);
329 unregister_trace_sched_switch(probe_wakeup_sched_switch); 322 unregister_trace_sched_switch(probe_wakeup_sched_switch);
330 unregister_trace_sched_wakeup_new(probe_wakeup); 323 unregister_trace_sched_wakeup_new(probe_wakeup);
331 unregister_trace_sched_wakeup(probe_wakeup); 324 unregister_trace_sched_wakeup(probe_wakeup);
332} 325}
333 326
334static int wakeup_tracer_init(struct trace_array *tr) 327static int __wakeup_tracer_init(struct trace_array *tr)
335{ 328{
336 tracing_max_latency = 0; 329 tracing_max_latency = 0;
337 wakeup_trace = tr; 330 wakeup_trace = tr;
@@ -339,6 +332,18 @@ static int wakeup_tracer_init(struct trace_array *tr)
339 return 0; 332 return 0;
340} 333}
341 334
335static int wakeup_tracer_init(struct trace_array *tr)
336{
337 wakeup_rt = 0;
338 return __wakeup_tracer_init(tr);
339}
340
341static int wakeup_rt_tracer_init(struct trace_array *tr)
342{
343 wakeup_rt = 1;
344 return __wakeup_tracer_init(tr);
345}
346
342static void wakeup_tracer_reset(struct trace_array *tr) 347static void wakeup_tracer_reset(struct trace_array *tr)
343{ 348{
344 stop_wakeup_tracer(tr); 349 stop_wakeup_tracer(tr);
@@ -350,28 +355,11 @@ static void wakeup_tracer_start(struct trace_array *tr)
350{ 355{
351 wakeup_reset(tr); 356 wakeup_reset(tr);
352 tracer_enabled = 1; 357 tracer_enabled = 1;
353 save_tracer_enabled = 1;
354} 358}
355 359
356static void wakeup_tracer_stop(struct trace_array *tr) 360static void wakeup_tracer_stop(struct trace_array *tr)
357{ 361{
358 tracer_enabled = 0; 362 tracer_enabled = 0;
359 save_tracer_enabled = 0;
360}
361
362static void wakeup_tracer_open(struct trace_iterator *iter)
363{
364 /* stop the trace while dumping */
365 tracer_enabled = 0;
366}
367
368static void wakeup_tracer_close(struct trace_iterator *iter)
369{
370 /* forget about any processes we were recording */
371 if (save_tracer_enabled) {
372 wakeup_reset(iter->tr);
373 tracer_enabled = 1;
374 }
375} 363}
376 364
377static struct tracer wakeup_tracer __read_mostly = 365static struct tracer wakeup_tracer __read_mostly =
@@ -381,8 +369,19 @@ static struct tracer wakeup_tracer __read_mostly =
381 .reset = wakeup_tracer_reset, 369 .reset = wakeup_tracer_reset,
382 .start = wakeup_tracer_start, 370 .start = wakeup_tracer_start,
383 .stop = wakeup_tracer_stop, 371 .stop = wakeup_tracer_stop,
384 .open = wakeup_tracer_open, 372 .print_max = 1,
385 .close = wakeup_tracer_close, 373#ifdef CONFIG_FTRACE_SELFTEST
374 .selftest = trace_selftest_startup_wakeup,
375#endif
376};
377
378static struct tracer wakeup_rt_tracer __read_mostly =
379{
380 .name = "wakeup_rt",
381 .init = wakeup_rt_tracer_init,
382 .reset = wakeup_tracer_reset,
383 .start = wakeup_tracer_start,
384 .stop = wakeup_tracer_stop,
386 .print_max = 1, 385 .print_max = 1,
387#ifdef CONFIG_FTRACE_SELFTEST 386#ifdef CONFIG_FTRACE_SELFTEST
388 .selftest = trace_selftest_startup_wakeup, 387 .selftest = trace_selftest_startup_wakeup,
@@ -397,6 +396,10 @@ __init static int init_wakeup_tracer(void)
397 if (ret) 396 if (ret)
398 return ret; 397 return ret;
399 398
399 ret = register_tracer(&wakeup_rt_tracer);
400 if (ret)
401 return ret;
402
400 return 0; 403 return 0;
401} 404}
402device_initcall(init_wakeup_tracer); 405device_initcall(init_wakeup_tracer);
diff --git a/kernel/trace/trace_selftest.c b/kernel/trace/trace_selftest.c
index 88c8eb70f54a..5013812578b1 100644
--- a/kernel/trace/trace_selftest.c
+++ b/kernel/trace/trace_selftest.c
@@ -9,7 +9,6 @@ static inline int trace_valid_entry(struct trace_entry *entry)
9 case TRACE_FN: 9 case TRACE_FN:
10 case TRACE_CTX: 10 case TRACE_CTX:
11 case TRACE_WAKE: 11 case TRACE_WAKE:
12 case TRACE_CONT:
13 case TRACE_STACK: 12 case TRACE_STACK:
14 case TRACE_PRINT: 13 case TRACE_PRINT:
15 case TRACE_SPECIAL: 14 case TRACE_SPECIAL:
diff --git a/kernel/trace/trace_stat.c b/kernel/trace/trace_stat.c
new file mode 100644
index 000000000000..eae9cef39291
--- /dev/null
+++ b/kernel/trace/trace_stat.c
@@ -0,0 +1,319 @@
1/*
2 * Infrastructure for statistic tracing (histogram output).
3 *
4 * Copyright (C) 2008 Frederic Weisbecker <fweisbec@gmail.com>
5 *
6 * Based on the code from trace_branch.c which is
7 * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com>
8 *
9 */
10
11
12#include <linux/list.h>
13#include <linux/debugfs.h>
14#include "trace_stat.h"
15#include "trace.h"
16
17
18/* List of stat entries from a tracer */
19struct trace_stat_list {
20 struct list_head list;
21 void *stat;
22};
23
24/* A stat session is the stats output in one file */
25struct tracer_stat_session {
26 struct list_head session_list;
27 struct tracer_stat *ts;
28 struct list_head stat_list;
29 struct mutex stat_mutex;
30 struct dentry *file;
31};
32
33/* All of the sessions currently in use. Each stat file embeed one session */
34static LIST_HEAD(all_stat_sessions);
35static DEFINE_MUTEX(all_stat_sessions_mutex);
36
37/* The root directory for all stat files */
38static struct dentry *stat_dir;
39
40
41static void reset_stat_session(struct tracer_stat_session *session)
42{
43 struct trace_stat_list *node, *next;
44
45 list_for_each_entry_safe(node, next, &session->stat_list, list)
46 kfree(node);
47
48 INIT_LIST_HEAD(&session->stat_list);
49}
50
51static void destroy_session(struct tracer_stat_session *session)
52{
53 debugfs_remove(session->file);
54 reset_stat_session(session);
55 mutex_destroy(&session->stat_mutex);
56 kfree(session);
57}
58
59/*
60 * For tracers that don't provide a stat_cmp callback.
61 * This one will force an immediate insertion on tail of
62 * the list.
63 */
64static int dummy_cmp(void *p1, void *p2)
65{
66 return 1;
67}
68
69/*
70 * Initialize the stat list at each trace_stat file opening.
71 * All of these copies and sorting are required on all opening
72 * since the stats could have changed between two file sessions.
73 */
74static int stat_seq_init(struct tracer_stat_session *session)
75{
76 struct trace_stat_list *iter_entry, *new_entry;
77 struct tracer_stat *ts = session->ts;
78 void *prev_stat;
79 int ret = 0;
80 int i;
81
82 mutex_lock(&session->stat_mutex);
83 reset_stat_session(session);
84
85 if (!ts->stat_cmp)
86 ts->stat_cmp = dummy_cmp;
87
88 /*
89 * The first entry. Actually this is the second, but the first
90 * one (the stat_list head) is pointless.
91 */
92 new_entry = kmalloc(sizeof(struct trace_stat_list), GFP_KERNEL);
93 if (!new_entry) {
94 ret = -ENOMEM;
95 goto exit;
96 }
97
98 INIT_LIST_HEAD(&new_entry->list);
99
100 list_add(&new_entry->list, &session->stat_list);
101
102 new_entry->stat = ts->stat_start();
103 prev_stat = new_entry->stat;
104
105 /*
106 * Iterate over the tracer stat entries and store them in a sorted
107 * list.
108 */
109 for (i = 1; ; i++) {
110 new_entry = kmalloc(sizeof(struct trace_stat_list), GFP_KERNEL);
111 if (!new_entry) {
112 ret = -ENOMEM;
113 goto exit_free_list;
114 }
115
116 INIT_LIST_HEAD(&new_entry->list);
117 new_entry->stat = ts->stat_next(prev_stat, i);
118
119 /* End of insertion */
120 if (!new_entry->stat)
121 break;
122
123 list_for_each_entry(iter_entry, &session->stat_list, list) {
124
125 /* Insertion with a descendent sorting */
126 if (ts->stat_cmp(new_entry->stat,
127 iter_entry->stat) > 0) {
128
129 list_add_tail(&new_entry->list,
130 &iter_entry->list);
131 break;
132
133 /* The current smaller value */
134 } else if (list_is_last(&iter_entry->list,
135 &session->stat_list)) {
136 list_add(&new_entry->list, &iter_entry->list);
137 break;
138 }
139 }
140
141 prev_stat = new_entry->stat;
142 }
143exit:
144 mutex_unlock(&session->stat_mutex);
145 return ret;
146
147exit_free_list:
148 reset_stat_session(session);
149 mutex_unlock(&session->stat_mutex);
150 return ret;
151}
152
153
154static void *stat_seq_start(struct seq_file *s, loff_t *pos)
155{
156 struct tracer_stat_session *session = s->private;
157
158 /* Prevent from tracer switch or stat_list modification */
159 mutex_lock(&session->stat_mutex);
160
161 /* If we are in the beginning of the file, print the headers */
162 if (!*pos && session->ts->stat_headers)
163 session->ts->stat_headers(s);
164
165 return seq_list_start(&session->stat_list, *pos);
166}
167
168static void *stat_seq_next(struct seq_file *s, void *p, loff_t *pos)
169{
170 struct tracer_stat_session *session = s->private;
171
172 return seq_list_next(p, &session->stat_list, pos);
173}
174
175static void stat_seq_stop(struct seq_file *s, void *p)
176{
177 struct tracer_stat_session *session = s->private;
178 mutex_unlock(&session->stat_mutex);
179}
180
181static int stat_seq_show(struct seq_file *s, void *v)
182{
183 struct tracer_stat_session *session = s->private;
184 struct trace_stat_list *l = list_entry(v, struct trace_stat_list, list);
185
186 return session->ts->stat_show(s, l->stat);
187}
188
189static const struct seq_operations trace_stat_seq_ops = {
190 .start = stat_seq_start,
191 .next = stat_seq_next,
192 .stop = stat_seq_stop,
193 .show = stat_seq_show
194};
195
196/* The session stat is refilled and resorted at each stat file opening */
197static int tracing_stat_open(struct inode *inode, struct file *file)
198{
199 int ret;
200
201 struct tracer_stat_session *session = inode->i_private;
202
203 ret = seq_open(file, &trace_stat_seq_ops);
204 if (!ret) {
205 struct seq_file *m = file->private_data;
206 m->private = session;
207 ret = stat_seq_init(session);
208 }
209
210 return ret;
211}
212
213/*
214 * Avoid consuming memory with our now useless list.
215 */
216static int tracing_stat_release(struct inode *i, struct file *f)
217{
218 struct tracer_stat_session *session = i->i_private;
219
220 mutex_lock(&session->stat_mutex);
221 reset_stat_session(session);
222 mutex_unlock(&session->stat_mutex);
223
224 return 0;
225}
226
227static const struct file_operations tracing_stat_fops = {
228 .open = tracing_stat_open,
229 .read = seq_read,
230 .llseek = seq_lseek,
231 .release = tracing_stat_release
232};
233
234static int tracing_stat_init(void)
235{
236 struct dentry *d_tracing;
237
238 d_tracing = tracing_init_dentry();
239
240 stat_dir = debugfs_create_dir("trace_stat", d_tracing);
241 if (!stat_dir)
242 pr_warning("Could not create debugfs "
243 "'trace_stat' entry\n");
244 return 0;
245}
246
247static int init_stat_file(struct tracer_stat_session *session)
248{
249 if (!stat_dir && tracing_stat_init())
250 return -ENODEV;
251
252 session->file = debugfs_create_file(session->ts->name, 0644,
253 stat_dir,
254 session, &tracing_stat_fops);
255 if (!session->file)
256 return -ENOMEM;
257 return 0;
258}
259
260int register_stat_tracer(struct tracer_stat *trace)
261{
262 struct tracer_stat_session *session, *node, *tmp;
263 int ret;
264
265 if (!trace)
266 return -EINVAL;
267
268 if (!trace->stat_start || !trace->stat_next || !trace->stat_show)
269 return -EINVAL;
270
271 /* Already registered? */
272 mutex_lock(&all_stat_sessions_mutex);
273 list_for_each_entry_safe(node, tmp, &all_stat_sessions, session_list) {
274 if (node->ts == trace) {
275 mutex_unlock(&all_stat_sessions_mutex);
276 return -EINVAL;
277 }
278 }
279 mutex_unlock(&all_stat_sessions_mutex);
280
281 /* Init the session */
282 session = kmalloc(sizeof(struct tracer_stat_session), GFP_KERNEL);
283 if (!session)
284 return -ENOMEM;
285
286 session->ts = trace;
287 INIT_LIST_HEAD(&session->session_list);
288 INIT_LIST_HEAD(&session->stat_list);
289 mutex_init(&session->stat_mutex);
290 session->file = NULL;
291
292 ret = init_stat_file(session);
293 if (ret) {
294 destroy_session(session);
295 return ret;
296 }
297
298 /* Register */
299 mutex_lock(&all_stat_sessions_mutex);
300 list_add_tail(&session->session_list, &all_stat_sessions);
301 mutex_unlock(&all_stat_sessions_mutex);
302
303 return 0;
304}
305
306void unregister_stat_tracer(struct tracer_stat *trace)
307{
308 struct tracer_stat_session *node, *tmp;
309
310 mutex_lock(&all_stat_sessions_mutex);
311 list_for_each_entry_safe(node, tmp, &all_stat_sessions, session_list) {
312 if (node->ts == trace) {
313 list_del(&node->session_list);
314 destroy_session(node);
315 break;
316 }
317 }
318 mutex_unlock(&all_stat_sessions_mutex);
319}
diff --git a/kernel/trace/trace_stat.h b/kernel/trace/trace_stat.h
new file mode 100644
index 000000000000..202274cf7f3d
--- /dev/null
+++ b/kernel/trace/trace_stat.h
@@ -0,0 +1,31 @@
1#ifndef __TRACE_STAT_H
2#define __TRACE_STAT_H
3
4#include <linux/seq_file.h>
5
6/*
7 * If you want to provide a stat file (one-shot statistics), fill
8 * an iterator with stat_start/stat_next and a stat_show callbacks.
9 * The others callbacks are optional.
10 */
11struct tracer_stat {
12 /* The name of your stat file */
13 const char *name;
14 /* Iteration over statistic entries */
15 void *(*stat_start)(void);
16 void *(*stat_next)(void *prev, int idx);
17 /* Compare two entries for stats sorting */
18 int (*stat_cmp)(void *p1, void *p2);
19 /* Print a stat entry */
20 int (*stat_show)(struct seq_file *s, void *p);
21 /* Print the headers of your stat entries */
22 int (*stat_headers)(struct seq_file *s);
23};
24
25/*
26 * Destroy or create a stat file
27 */
28extern int register_stat_tracer(struct tracer_stat *trace);
29extern void unregister_stat_tracer(struct tracer_stat *trace);
30
31#endif /* __TRACE_STAT_H */
diff --git a/kernel/trace/trace_workqueue.c b/kernel/trace/trace_workqueue.c
new file mode 100644
index 000000000000..4664990fe9c5
--- /dev/null
+++ b/kernel/trace/trace_workqueue.c
@@ -0,0 +1,281 @@
1/*
2 * Workqueue statistical tracer.
3 *
4 * Copyright (C) 2008 Frederic Weisbecker <fweisbec@gmail.com>
5 *
6 */
7
8
9#include <trace/workqueue.h>
10#include <linux/list.h>
11#include <linux/percpu.h>
12#include "trace_stat.h"
13#include "trace.h"
14
15
16/* A cpu workqueue thread */
17struct cpu_workqueue_stats {
18 struct list_head list;
19/* Useful to know if we print the cpu headers */
20 bool first_entry;
21 int cpu;
22 pid_t pid;
23/* Can be inserted from interrupt or user context, need to be atomic */
24 atomic_t inserted;
25/*
26 * Don't need to be atomic, works are serialized in a single workqueue thread
27 * on a single CPU.
28 */
29 unsigned int executed;
30};
31
32/* List of workqueue threads on one cpu */
33struct workqueue_global_stats {
34 struct list_head list;
35 spinlock_t lock;
36};
37
38/* Don't need a global lock because allocated before the workqueues, and
39 * never freed.
40 */
41static DEFINE_PER_CPU(struct workqueue_global_stats, all_workqueue_stat);
42#define workqueue_cpu_stat(cpu) (&per_cpu(all_workqueue_stat, cpu))
43
44/* Insertion of a work */
45static void
46probe_workqueue_insertion(struct task_struct *wq_thread,
47 struct work_struct *work)
48{
49 int cpu = cpumask_first(&wq_thread->cpus_allowed);
50 struct cpu_workqueue_stats *node, *next;
51 unsigned long flags;
52
53 spin_lock_irqsave(&workqueue_cpu_stat(cpu)->lock, flags);
54 list_for_each_entry_safe(node, next, &workqueue_cpu_stat(cpu)->list,
55 list) {
56 if (node->pid == wq_thread->pid) {
57 atomic_inc(&node->inserted);
58 goto found;
59 }
60 }
61 pr_debug("trace_workqueue: entry not found\n");
62found:
63 spin_unlock_irqrestore(&workqueue_cpu_stat(cpu)->lock, flags);
64}
65
66/* Execution of a work */
67static void
68probe_workqueue_execution(struct task_struct *wq_thread,
69 struct work_struct *work)
70{
71 int cpu = cpumask_first(&wq_thread->cpus_allowed);
72 struct cpu_workqueue_stats *node, *next;
73 unsigned long flags;
74
75 spin_lock_irqsave(&workqueue_cpu_stat(cpu)->lock, flags);
76 list_for_each_entry_safe(node, next, &workqueue_cpu_stat(cpu)->list,
77 list) {
78 if (node->pid == wq_thread->pid) {
79 node->executed++;
80 goto found;
81 }
82 }
83 pr_debug("trace_workqueue: entry not found\n");
84found:
85 spin_unlock_irqrestore(&workqueue_cpu_stat(cpu)->lock, flags);
86}
87
88/* Creation of a cpu workqueue thread */
89static void probe_workqueue_creation(struct task_struct *wq_thread, int cpu)
90{
91 struct cpu_workqueue_stats *cws;
92 unsigned long flags;
93
94 WARN_ON(cpu < 0 || cpu >= num_possible_cpus());
95
96 /* Workqueues are sometimes created in atomic context */
97 cws = kzalloc(sizeof(struct cpu_workqueue_stats), GFP_ATOMIC);
98 if (!cws) {
99 pr_warning("trace_workqueue: not enough memory\n");
100 return;
101 }
102 tracing_record_cmdline(wq_thread);
103
104 INIT_LIST_HEAD(&cws->list);
105 cws->cpu = cpu;
106
107 cws->pid = wq_thread->pid;
108
109 spin_lock_irqsave(&workqueue_cpu_stat(cpu)->lock, flags);
110 if (list_empty(&workqueue_cpu_stat(cpu)->list))
111 cws->first_entry = true;
112 list_add_tail(&cws->list, &workqueue_cpu_stat(cpu)->list);
113 spin_unlock_irqrestore(&workqueue_cpu_stat(cpu)->lock, flags);
114}
115
116/* Destruction of a cpu workqueue thread */
117static void probe_workqueue_destruction(struct task_struct *wq_thread)
118{
119 /* Workqueue only execute on one cpu */
120 int cpu = cpumask_first(&wq_thread->cpus_allowed);
121 struct cpu_workqueue_stats *node, *next;
122 unsigned long flags;
123
124 spin_lock_irqsave(&workqueue_cpu_stat(cpu)->lock, flags);
125 list_for_each_entry_safe(node, next, &workqueue_cpu_stat(cpu)->list,
126 list) {
127 if (node->pid == wq_thread->pid) {
128 list_del(&node->list);
129 kfree(node);
130 goto found;
131 }
132 }
133
134 pr_debug("trace_workqueue: don't find workqueue to destroy\n");
135found:
136 spin_unlock_irqrestore(&workqueue_cpu_stat(cpu)->lock, flags);
137
138}
139
140static struct cpu_workqueue_stats *workqueue_stat_start_cpu(int cpu)
141{
142 unsigned long flags;
143 struct cpu_workqueue_stats *ret = NULL;
144
145
146 spin_lock_irqsave(&workqueue_cpu_stat(cpu)->lock, flags);
147
148 if (!list_empty(&workqueue_cpu_stat(cpu)->list))
149 ret = list_entry(workqueue_cpu_stat(cpu)->list.next,
150 struct cpu_workqueue_stats, list);
151
152 spin_unlock_irqrestore(&workqueue_cpu_stat(cpu)->lock, flags);
153
154 return ret;
155}
156
157static void *workqueue_stat_start(void)
158{
159 int cpu;
160 void *ret = NULL;
161
162 for_each_possible_cpu(cpu) {
163 ret = workqueue_stat_start_cpu(cpu);
164 if (ret)
165 return ret;
166 }
167 return NULL;
168}
169
170static void *workqueue_stat_next(void *prev, int idx)
171{
172 struct cpu_workqueue_stats *prev_cws = prev;
173 int cpu = prev_cws->cpu;
174 unsigned long flags;
175 void *ret = NULL;
176
177 spin_lock_irqsave(&workqueue_cpu_stat(cpu)->lock, flags);
178 if (list_is_last(&prev_cws->list, &workqueue_cpu_stat(cpu)->list)) {
179 spin_unlock_irqrestore(&workqueue_cpu_stat(cpu)->lock, flags);
180 for (++cpu ; cpu < num_possible_cpus(); cpu++) {
181 ret = workqueue_stat_start_cpu(cpu);
182 if (ret)
183 return ret;
184 }
185 return NULL;
186 }
187 spin_unlock_irqrestore(&workqueue_cpu_stat(cpu)->lock, flags);
188
189 return list_entry(prev_cws->list.next, struct cpu_workqueue_stats,
190 list);
191}
192
193static int workqueue_stat_show(struct seq_file *s, void *p)
194{
195 struct cpu_workqueue_stats *cws = p;
196 unsigned long flags;
197 int cpu = cws->cpu;
198
199 seq_printf(s, "%3d %6d %6u %s\n", cws->cpu,
200 atomic_read(&cws->inserted),
201 cws->executed,
202 trace_find_cmdline(cws->pid));
203
204 spin_lock_irqsave(&workqueue_cpu_stat(cpu)->lock, flags);
205 if (&cws->list == workqueue_cpu_stat(cpu)->list.next)
206 seq_printf(s, "\n");
207 spin_unlock_irqrestore(&workqueue_cpu_stat(cpu)->lock, flags);
208
209 return 0;
210}
211
212static int workqueue_stat_headers(struct seq_file *s)
213{
214 seq_printf(s, "# CPU INSERTED EXECUTED NAME\n");
215 seq_printf(s, "# | | | |\n\n");
216 return 0;
217}
218
219struct tracer_stat workqueue_stats __read_mostly = {
220 .name = "workqueues",
221 .stat_start = workqueue_stat_start,
222 .stat_next = workqueue_stat_next,
223 .stat_show = workqueue_stat_show,
224 .stat_headers = workqueue_stat_headers
225};
226
227
228int __init stat_workqueue_init(void)
229{
230 if (register_stat_tracer(&workqueue_stats)) {
231 pr_warning("Unable to register workqueue stat tracer\n");
232 return 1;
233 }
234
235 return 0;
236}
237fs_initcall(stat_workqueue_init);
238
239/*
240 * Workqueues are created very early, just after pre-smp initcalls.
241 * So we must register our tracepoints at this stage.
242 */
243int __init trace_workqueue_early_init(void)
244{
245 int ret, cpu;
246
247 ret = register_trace_workqueue_insertion(probe_workqueue_insertion);
248 if (ret)
249 goto out;
250
251 ret = register_trace_workqueue_execution(probe_workqueue_execution);
252 if (ret)
253 goto no_insertion;
254
255 ret = register_trace_workqueue_creation(probe_workqueue_creation);
256 if (ret)
257 goto no_execution;
258
259 ret = register_trace_workqueue_destruction(probe_workqueue_destruction);
260 if (ret)
261 goto no_creation;
262
263 for_each_possible_cpu(cpu) {
264 spin_lock_init(&workqueue_cpu_stat(cpu)->lock);
265 INIT_LIST_HEAD(&workqueue_cpu_stat(cpu)->list);
266 }
267
268 return 0;
269
270no_creation:
271 unregister_trace_workqueue_creation(probe_workqueue_creation);
272no_execution:
273 unregister_trace_workqueue_execution(probe_workqueue_execution);
274no_insertion:
275 unregister_trace_workqueue_insertion(probe_workqueue_insertion);
276out:
277 pr_warning("trace_workqueue: unable to trace workqueues\n");
278
279 return 1;
280}
281early_initcall(trace_workqueue_early_init);
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index 1f0c509b40d3..e53ee18ef431 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -33,6 +33,7 @@
33#include <linux/kallsyms.h> 33#include <linux/kallsyms.h>
34#include <linux/debug_locks.h> 34#include <linux/debug_locks.h>
35#include <linux/lockdep.h> 35#include <linux/lockdep.h>
36#include <trace/workqueue.h>
36 37
37/* 38/*
38 * The per-CPU workqueue (if single thread, we always use the first 39 * The per-CPU workqueue (if single thread, we always use the first
@@ -125,9 +126,13 @@ struct cpu_workqueue_struct *get_wq_data(struct work_struct *work)
125 return (void *) (atomic_long_read(&work->data) & WORK_STRUCT_WQ_DATA_MASK); 126 return (void *) (atomic_long_read(&work->data) & WORK_STRUCT_WQ_DATA_MASK);
126} 127}
127 128
129DEFINE_TRACE(workqueue_insertion);
130
128static void insert_work(struct cpu_workqueue_struct *cwq, 131static void insert_work(struct cpu_workqueue_struct *cwq,
129 struct work_struct *work, struct list_head *head) 132 struct work_struct *work, struct list_head *head)
130{ 133{
134 trace_workqueue_insertion(cwq->thread, work);
135
131 set_wq_data(work, cwq); 136 set_wq_data(work, cwq);
132 /* 137 /*
133 * Ensure that we get the right work->data if we see the 138 * Ensure that we get the right work->data if we see the
@@ -259,6 +264,8 @@ int queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
259} 264}
260EXPORT_SYMBOL_GPL(queue_delayed_work_on); 265EXPORT_SYMBOL_GPL(queue_delayed_work_on);
261 266
267DEFINE_TRACE(workqueue_execution);
268
262static void run_workqueue(struct cpu_workqueue_struct *cwq) 269static void run_workqueue(struct cpu_workqueue_struct *cwq)
263{ 270{
264 spin_lock_irq(&cwq->lock); 271 spin_lock_irq(&cwq->lock);
@@ -284,7 +291,7 @@ static void run_workqueue(struct cpu_workqueue_struct *cwq)
284 */ 291 */
285 struct lockdep_map lockdep_map = work->lockdep_map; 292 struct lockdep_map lockdep_map = work->lockdep_map;
286#endif 293#endif
287 294 trace_workqueue_execution(cwq->thread, work);
288 cwq->current_work = work; 295 cwq->current_work = work;
289 list_del_init(cwq->worklist.next); 296 list_del_init(cwq->worklist.next);
290 spin_unlock_irq(&cwq->lock); 297 spin_unlock_irq(&cwq->lock);
@@ -765,6 +772,8 @@ init_cpu_workqueue(struct workqueue_struct *wq, int cpu)
765 return cwq; 772 return cwq;
766} 773}
767 774
775DEFINE_TRACE(workqueue_creation);
776
768static int create_workqueue_thread(struct cpu_workqueue_struct *cwq, int cpu) 777static int create_workqueue_thread(struct cpu_workqueue_struct *cwq, int cpu)
769{ 778{
770 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 }; 779 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
@@ -787,6 +796,8 @@ static int create_workqueue_thread(struct cpu_workqueue_struct *cwq, int cpu)
787 sched_setscheduler_nocheck(p, SCHED_FIFO, &param); 796 sched_setscheduler_nocheck(p, SCHED_FIFO, &param);
788 cwq->thread = p; 797 cwq->thread = p;
789 798
799 trace_workqueue_creation(cwq->thread, cpu);
800
790 return 0; 801 return 0;
791} 802}
792 803
@@ -868,6 +879,8 @@ struct workqueue_struct *__create_workqueue_key(const char *name,
868} 879}
869EXPORT_SYMBOL_GPL(__create_workqueue_key); 880EXPORT_SYMBOL_GPL(__create_workqueue_key);
870 881
882DEFINE_TRACE(workqueue_destruction);
883
871static void cleanup_workqueue_thread(struct cpu_workqueue_struct *cwq) 884static void cleanup_workqueue_thread(struct cpu_workqueue_struct *cwq)
872{ 885{
873 /* 886 /*
@@ -891,6 +904,7 @@ static void cleanup_workqueue_thread(struct cpu_workqueue_struct *cwq)
891 * checks list_empty(), and a "normal" queue_work() can't use 904 * checks list_empty(), and a "normal" queue_work() can't use
892 * a dead CPU. 905 * a dead CPU.
893 */ 906 */
907 trace_workqueue_destruction(cwq->thread);
894 kthread_stop(cwq->thread); 908 kthread_stop(cwq->thread);
895 cwq->thread = NULL; 909 cwq->thread = NULL;
896} 910}
generated by cgit v1.2.2 (git 2.25.0) at 2025-01-29 20:57:19 -0500