aboutsummaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/extable.c4
-rw-r--r--kernel/module.c2
-rw-r--r--kernel/relay.c4
-rw-r--r--kernel/sched.c8
-rw-r--r--kernel/softirq.c13
-rw-r--r--kernel/trace/Kconfig72
-rw-r--r--kernel/trace/Makefile5
-rw-r--r--kernel/trace/blktrace.c1538
-rw-r--r--kernel/trace/ftrace.c983
-rw-r--r--kernel/trace/kmemtrace.c339
-rw-r--r--kernel/trace/ring_buffer.c168
-rw-r--r--kernel/trace/trace.c1533
-rw-r--r--kernel/trace/trace.h130
-rw-r--r--kernel/trace/trace_boot.c36
-rw-r--r--kernel/trace/trace_branch.c279
-rw-r--r--kernel/trace/trace_functions.c369
-rw-r--r--kernel/trace/trace_functions_graph.c474
-rw-r--r--kernel/trace/trace_hw_branches.c179
-rw-r--r--kernel/trace/trace_irqsoff.c46
-rw-r--r--kernel/trace/trace_mmiotrace.c41
-rw-r--r--kernel/trace/trace_nop.c5
-rw-r--r--kernel/trace/trace_output.c919
-rw-r--r--kernel/trace/trace_output.h62
-rw-r--r--kernel/trace/trace_power.c188
-rw-r--r--kernel/trace/trace_sched_switch.c15
-rw-r--r--kernel/trace/trace_sched_wakeup.c88
-rw-r--r--kernel/trace/trace_selftest.c85
-rw-r--r--kernel/trace/trace_stat.c319
-rw-r--r--kernel/trace/trace_stat.h31
-rw-r--r--kernel/trace/trace_sysprof.c21
-rw-r--r--kernel/trace/trace_workqueue.c281
-rw-r--r--kernel/workqueue.c16
32 files changed, 6293 insertions, 1960 deletions
diff --git a/kernel/extable.c b/kernel/extable.c
index e136ed8d82ba..0df6253730be 100644
--- a/kernel/extable.c
+++ b/kernel/extable.c
@@ -41,7 +41,7 @@ const struct exception_table_entry *search_exception_tables(unsigned long addr)
41 return e; 41 return e;
42} 42}
43 43
44__notrace_funcgraph int core_kernel_text(unsigned long addr) 44int core_kernel_text(unsigned long addr)
45{ 45{
46 if (addr >= (unsigned long)_stext && 46 if (addr >= (unsigned long)_stext &&
47 addr <= (unsigned long)_etext) 47 addr <= (unsigned long)_etext)
@@ -54,7 +54,7 @@ __notrace_funcgraph int core_kernel_text(unsigned long addr)
54 return 0; 54 return 0;
55} 55}
56 56
57__notrace_funcgraph int __kernel_text_address(unsigned long addr) 57int __kernel_text_address(unsigned long addr)
58{ 58{
59 if (core_kernel_text(addr)) 59 if (core_kernel_text(addr))
60 return 1; 60 return 1;
diff --git a/kernel/module.c b/kernel/module.c
index ba22484a987e..22d7379709da 100644
--- a/kernel/module.c
+++ b/kernel/module.c
@@ -2735,7 +2735,7 @@ int is_module_address(unsigned long addr)
2735 2735
2736 2736
2737/* Is this a valid kernel address? */ 2737/* Is this a valid kernel address? */
2738__notrace_funcgraph struct module *__module_text_address(unsigned long addr) 2738struct module *__module_text_address(unsigned long addr)
2739{ 2739{
2740 struct module *mod; 2740 struct module *mod;
2741 2741
diff --git a/kernel/relay.c b/kernel/relay.c
index 9d79b7854fa6..edc0ba6d8160 100644
--- a/kernel/relay.c
+++ b/kernel/relay.c
@@ -677,9 +677,7 @@ int relay_late_setup_files(struct rchan *chan,
677 */ 677 */
678 for_each_online_cpu(i) { 678 for_each_online_cpu(i) {
679 if (unlikely(!chan->buf[i])) { 679 if (unlikely(!chan->buf[i])) {
680 printk(KERN_ERR "relay_late_setup_files: CPU %u " 680 WARN_ONCE(1, KERN_ERR "CPU has no buffer!\n");
681 "has no buffer, it must have!\n", i);
682 BUG();
683 err = -EINVAL; 681 err = -EINVAL;
684 break; 682 break;
685 } 683 }
diff --git a/kernel/sched.c b/kernel/sched.c
index 410eec404133..2492b9b3fed4 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -4404,10 +4404,7 @@ void scheduler_tick(void)
4404#endif 4404#endif
4405} 4405}
4406 4406
4407#if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \ 4407unsigned long get_parent_ip(unsigned long addr)
4408 defined(CONFIG_PREEMPT_TRACER))
4409
4410static inline unsigned long get_parent_ip(unsigned long addr)
4411{ 4408{
4412 if (in_lock_functions(addr)) { 4409 if (in_lock_functions(addr)) {
4413 addr = CALLER_ADDR2; 4410 addr = CALLER_ADDR2;
@@ -4417,6 +4414,9 @@ static inline unsigned long get_parent_ip(unsigned long addr)
4417 return addr; 4414 return addr;
4418} 4415}
4419 4416
4417#if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \
4418 defined(CONFIG_PREEMPT_TRACER))
4419
4420void __kprobes add_preempt_count(int val) 4420void __kprobes add_preempt_count(int val)
4421{ 4421{
4422#ifdef CONFIG_DEBUG_PREEMPT 4422#ifdef CONFIG_DEBUG_PREEMPT
diff --git a/kernel/softirq.c b/kernel/softirq.c
index bdbe9de9cd8d..6edfc2c11d99 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -21,6 +21,7 @@
21#include <linux/freezer.h> 21#include <linux/freezer.h>
22#include <linux/kthread.h> 22#include <linux/kthread.h>
23#include <linux/rcupdate.h> 23#include <linux/rcupdate.h>
24#include <linux/ftrace.h>
24#include <linux/smp.h> 25#include <linux/smp.h>
25#include <linux/tick.h> 26#include <linux/tick.h>
26 27
@@ -79,13 +80,23 @@ static void __local_bh_disable(unsigned long ip)
79 WARN_ON_ONCE(in_irq()); 80 WARN_ON_ONCE(in_irq());
80 81
81 raw_local_irq_save(flags); 82 raw_local_irq_save(flags);
82 add_preempt_count(SOFTIRQ_OFFSET); 83 /*
84 * The preempt tracer hooks into add_preempt_count and will break
85 * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
86 * is set and before current->softirq_enabled is cleared.
87 * We must manually increment preempt_count here and manually
88 * call the trace_preempt_off later.
89 */
90 preempt_count() += SOFTIRQ_OFFSET;
83 /* 91 /*
84 * Were softirqs turned off above: 92 * Were softirqs turned off above:
85 */ 93 */
86 if (softirq_count() == SOFTIRQ_OFFSET) 94 if (softirq_count() == SOFTIRQ_OFFSET)
87 trace_softirqs_off(ip); 95 trace_softirqs_off(ip);
88 raw_local_irq_restore(flags); 96 raw_local_irq_restore(flags);
97
98 if (preempt_count() == SOFTIRQ_OFFSET)
99 trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
89} 100}
90#else /* !CONFIG_TRACE_IRQFLAGS */ 101#else /* !CONFIG_TRACE_IRQFLAGS */
91static inline void __local_bh_disable(unsigned long ip) 102static inline void __local_bh_disable(unsigned long ip)
diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig
index 34e707e5ab87..07877f4b5233 100644
--- a/kernel/trace/Kconfig
+++ b/kernel/trace/Kconfig
@@ -9,6 +9,9 @@ config USER_STACKTRACE_SUPPORT
9config NOP_TRACER 9config NOP_TRACER
10 bool 10 bool
11 11
12config HAVE_FTRACE_NMI_ENTER
13 bool
14
12config HAVE_FUNCTION_TRACER 15config HAVE_FUNCTION_TRACER
13 bool 16 bool
14 17
@@ -37,6 +40,11 @@ config TRACER_MAX_TRACE
37config RING_BUFFER 40config RING_BUFFER
38 bool 41 bool
39 42
43config FTRACE_NMI_ENTER
44 bool
45 depends on HAVE_FTRACE_NMI_ENTER
46 default y
47
40config TRACING 48config TRACING
41 bool 49 bool
42 select DEBUG_FS 50 select DEBUG_FS
@@ -127,6 +135,7 @@ config SYSPROF_TRACER
127 bool "Sysprof Tracer" 135 bool "Sysprof Tracer"
128 depends on X86 136 depends on X86
129 select TRACING 137 select TRACING
138 select CONTEXT_SWITCH_TRACER
130 help 139 help
131 This tracer provides the trace needed by the 'Sysprof' userspace 140 This tracer provides the trace needed by the 'Sysprof' userspace
132 tool. 141 tool.
@@ -165,9 +174,8 @@ config BOOT_TRACER
165 representation of the delays during initcalls - but the raw 174 representation of the delays during initcalls - but the raw
166 /debug/tracing/trace text output is readable too. 175 /debug/tracing/trace text output is readable too.
167 176
168 ( Note that tracing self tests can't be enabled if this tracer is 177 You must pass in ftrace=initcall to the kernel command line
169 selected, because the self-tests are an initcall as well and that 178 to enable this on bootup.
170 would invalidate the boot trace. )
171 179
172config TRACE_BRANCH_PROFILING 180config TRACE_BRANCH_PROFILING
173 bool "Trace likely/unlikely profiler" 181 bool "Trace likely/unlikely profiler"
@@ -266,6 +274,62 @@ config HW_BRANCH_TRACER
266 This tracer records all branches on the system in a circular 274 This tracer records all branches on the system in a circular
267 buffer giving access to the last N branches for each cpu. 275 buffer giving access to the last N branches for each cpu.
268 276
277config KMEMTRACE
278 bool "Trace SLAB allocations"
279 select TRACING
280 help
281 kmemtrace provides tracing for slab allocator functions, such as
282 kmalloc, kfree, kmem_cache_alloc, kmem_cache_free etc.. Collected
283 data is then fed to the userspace application in order to analyse
284 allocation hotspots, internal fragmentation and so on, making it
285 possible to see how well an allocator performs, as well as debug
286 and profile kernel code.
287
288 This requires an userspace application to use. See
289 Documentation/vm/kmemtrace.txt for more information.
290
291 Saying Y will make the kernel somewhat larger and slower. However,
292 if you disable kmemtrace at run-time or boot-time, the performance
293 impact is minimal (depending on the arch the kernel is built for).
294
295 If unsure, say N.
296
297config WORKQUEUE_TRACER
298 bool "Trace workqueues"
299 select TRACING
300 help
301 The workqueue tracer provides some statistical informations
302 about each cpu workqueue thread such as the number of the
303 works inserted and executed since their creation. It can help
304 to evaluate the amount of work each of them have to perform.
305 For example it can help a developer to decide whether he should
306 choose a per cpu workqueue instead of a singlethreaded one.
307
308config BLK_DEV_IO_TRACE
309 bool "Support for tracing block io actions"
310 depends on SYSFS
311 depends on BLOCK
312 select RELAY
313 select DEBUG_FS
314 select TRACEPOINTS
315 select TRACING
316 select STACKTRACE
317 help
318 Say Y here if you want to be able to trace the block layer actions
319 on a given queue. Tracing allows you to see any traffic happening
320 on a block device queue. For more information (and the userspace
321 support tools needed), fetch the blktrace tools from:
322
323 git://git.kernel.dk/blktrace.git
324
325 Tracing also is possible using the ftrace interface, e.g.:
326
327 echo 1 > /sys/block/sda/sda1/trace/enable
328 echo blk > /sys/kernel/debug/tracing/current_tracer
329 cat /sys/kernel/debug/tracing/trace_pipe
330
331 If unsure, say N.
332
269config DYNAMIC_FTRACE 333config DYNAMIC_FTRACE
270 bool "enable/disable ftrace tracepoints dynamically" 334 bool "enable/disable ftrace tracepoints dynamically"
271 depends on FUNCTION_TRACER 335 depends on FUNCTION_TRACER
@@ -296,7 +360,7 @@ config FTRACE_SELFTEST
296 360
297config FTRACE_STARTUP_TEST 361config FTRACE_STARTUP_TEST
298 bool "Perform a startup test on ftrace" 362 bool "Perform a startup test on ftrace"
299 depends on TRACING && DEBUG_KERNEL && !BOOT_TRACER 363 depends on TRACING && DEBUG_KERNEL
300 select FTRACE_SELFTEST 364 select FTRACE_SELFTEST
301 help 365 help
302 This option performs a series of startup tests on ftrace. On bootup 366 This option performs a series of startup tests on ftrace. On bootup
diff --git a/kernel/trace/Makefile b/kernel/trace/Makefile
index 349d5a93653f..627090bc262d 100644
--- a/kernel/trace/Makefile
+++ b/kernel/trace/Makefile
@@ -19,6 +19,8 @@ obj-$(CONFIG_FUNCTION_TRACER) += libftrace.o
19obj-$(CONFIG_RING_BUFFER) += ring_buffer.o 19obj-$(CONFIG_RING_BUFFER) += ring_buffer.o
20 20
21obj-$(CONFIG_TRACING) += trace.o 21obj-$(CONFIG_TRACING) += trace.o
22obj-$(CONFIG_TRACING) += trace_output.o
23obj-$(CONFIG_TRACING) += trace_stat.o
22obj-$(CONFIG_CONTEXT_SWITCH_TRACER) += trace_sched_switch.o 24obj-$(CONFIG_CONTEXT_SWITCH_TRACER) += trace_sched_switch.o
23obj-$(CONFIG_SYSPROF_TRACER) += trace_sysprof.o 25obj-$(CONFIG_SYSPROF_TRACER) += trace_sysprof.o
24obj-$(CONFIG_FUNCTION_TRACER) += trace_functions.o 26obj-$(CONFIG_FUNCTION_TRACER) += trace_functions.o
@@ -33,5 +35,8 @@ obj-$(CONFIG_FUNCTION_GRAPH_TRACER) += trace_functions_graph.o
33obj-$(CONFIG_TRACE_BRANCH_PROFILING) += trace_branch.o 35obj-$(CONFIG_TRACE_BRANCH_PROFILING) += trace_branch.o
34obj-$(CONFIG_HW_BRANCH_TRACER) += trace_hw_branches.o 36obj-$(CONFIG_HW_BRANCH_TRACER) += trace_hw_branches.o
35obj-$(CONFIG_POWER_TRACER) += trace_power.o 37obj-$(CONFIG_POWER_TRACER) += trace_power.o
38obj-$(CONFIG_KMEMTRACE) += kmemtrace.o
39obj-$(CONFIG_WORKQUEUE_TRACER) += trace_workqueue.o
40obj-$(CONFIG_BLK_DEV_IO_TRACE) += blktrace.o
36 41
37libftrace-y := ftrace.o 42libftrace-y := ftrace.o
diff --git a/kernel/trace/blktrace.c b/kernel/trace/blktrace.c
new file mode 100644
index 000000000000..e82cb9e930cc
--- /dev/null
+++ b/kernel/trace/blktrace.c
@@ -0,0 +1,1538 @@
1/*
2 * Copyright (C) 2006 Jens Axboe <axboe@kernel.dk>
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * along with this program; if not, write to the Free Software
15 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
16 *
17 */
18#include <linux/kernel.h>
19#include <linux/blkdev.h>
20#include <linux/blktrace_api.h>
21#include <linux/percpu.h>
22#include <linux/init.h>
23#include <linux/mutex.h>
24#include <linux/debugfs.h>
25#include <linux/time.h>
26#include <trace/block.h>
27#include <linux/uaccess.h>
28#include "trace_output.h"
29
30static unsigned int blktrace_seq __read_mostly = 1;
31
32static struct trace_array *blk_tr;
33static int __read_mostly blk_tracer_enabled;
34
35/* Select an alternative, minimalistic output than the original one */
36#define TRACE_BLK_OPT_CLASSIC 0x1
37
38static struct tracer_opt blk_tracer_opts[] = {
39 /* Default disable the minimalistic output */
40 { TRACER_OPT(blk_classic, TRACE_BLK_OPT_CLASSIC) },
41 { }
42};
43
44static struct tracer_flags blk_tracer_flags = {
45 .val = 0,
46 .opts = blk_tracer_opts,
47};
48
49/* Global reference count of probes */
50static DEFINE_MUTEX(blk_probe_mutex);
51static atomic_t blk_probes_ref = ATOMIC_INIT(0);
52
53static int blk_register_tracepoints(void);
54static void blk_unregister_tracepoints(void);
55
56/*
57 * Send out a notify message.
58 */
59static void trace_note(struct blk_trace *bt, pid_t pid, int action,
60 const void *data, size_t len)
61{
62 struct blk_io_trace *t;
63
64 if (!bt->rchan)
65 return;
66
67 t = relay_reserve(bt->rchan, sizeof(*t) + len);
68 if (t) {
69 const int cpu = smp_processor_id();
70
71 t->magic = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE_VERSION;
72 t->time = ktime_to_ns(ktime_get());
73 t->device = bt->dev;
74 t->action = action;
75 t->pid = pid;
76 t->cpu = cpu;
77 t->pdu_len = len;
78 memcpy((void *) t + sizeof(*t), data, len);
79 }
80}
81
82/*
83 * Send out a notify for this process, if we haven't done so since a trace
84 * started
85 */
86static void trace_note_tsk(struct blk_trace *bt, struct task_struct *tsk)
87{
88 tsk->btrace_seq = blktrace_seq;
89 trace_note(bt, tsk->pid, BLK_TN_PROCESS, tsk->comm, sizeof(tsk->comm));
90}
91
92static void trace_note_time(struct blk_trace *bt)
93{
94 struct timespec now;
95 unsigned long flags;
96 u32 words[2];
97
98 getnstimeofday(&now);
99 words[0] = now.tv_sec;
100 words[1] = now.tv_nsec;
101
102 local_irq_save(flags);
103 trace_note(bt, 0, BLK_TN_TIMESTAMP, words, sizeof(words));
104 local_irq_restore(flags);
105}
106
107void __trace_note_message(struct blk_trace *bt, const char *fmt, ...)
108{
109 int n;
110 va_list args;
111 unsigned long flags;
112 char *buf;
113
114 if (blk_tr) {
115 va_start(args, fmt);
116 ftrace_vprintk(fmt, args);
117 va_end(args);
118 return;
119 }
120
121 if (!bt->msg_data)
122 return;
123
124 local_irq_save(flags);
125 buf = per_cpu_ptr(bt->msg_data, smp_processor_id());
126 va_start(args, fmt);
127 n = vscnprintf(buf, BLK_TN_MAX_MSG, fmt, args);
128 va_end(args);
129
130 trace_note(bt, 0, BLK_TN_MESSAGE, buf, n);
131 local_irq_restore(flags);
132}
133EXPORT_SYMBOL_GPL(__trace_note_message);
134
135static int act_log_check(struct blk_trace *bt, u32 what, sector_t sector,
136 pid_t pid)
137{
138 if (((bt->act_mask << BLK_TC_SHIFT) & what) == 0)
139 return 1;
140 if (sector < bt->start_lba || sector > bt->end_lba)
141 return 1;
142 if (bt->pid && pid != bt->pid)
143 return 1;
144
145 return 0;
146}
147
148/*
149 * Data direction bit lookup
150 */
151static u32 ddir_act[2] __read_mostly = { BLK_TC_ACT(BLK_TC_READ),
152 BLK_TC_ACT(BLK_TC_WRITE) };
153
154/* The ilog2() calls fall out because they're constant */
155#define MASK_TC_BIT(rw, __name) ((rw & (1 << BIO_RW_ ## __name)) << \
156 (ilog2(BLK_TC_ ## __name) + BLK_TC_SHIFT - BIO_RW_ ## __name))
157
158/*
159 * The worker for the various blk_add_trace*() types. Fills out a
160 * blk_io_trace structure and places it in a per-cpu subbuffer.
161 */
162static void __blk_add_trace(struct blk_trace *bt, sector_t sector, int bytes,
163 int rw, u32 what, int error, int pdu_len, void *pdu_data)
164{
165 struct task_struct *tsk = current;
166 struct ring_buffer_event *event = NULL;
167 struct blk_io_trace *t;
168 unsigned long flags = 0;
169 unsigned long *sequence;
170 pid_t pid;
171 int cpu, pc = 0;
172
173 if (unlikely(bt->trace_state != Blktrace_running ||
174 !blk_tracer_enabled))
175 return;
176
177 what |= ddir_act[rw & WRITE];
178 what |= MASK_TC_BIT(rw, BARRIER);
179 what |= MASK_TC_BIT(rw, SYNCIO);
180 what |= MASK_TC_BIT(rw, AHEAD);
181 what |= MASK_TC_BIT(rw, META);
182 what |= MASK_TC_BIT(rw, DISCARD);
183
184 pid = tsk->pid;
185 if (unlikely(act_log_check(bt, what, sector, pid)))
186 return;
187 cpu = raw_smp_processor_id();
188
189 if (blk_tr) {
190 tracing_record_cmdline(current);
191
192 pc = preempt_count();
193 event = trace_buffer_lock_reserve(blk_tr, TRACE_BLK,
194 sizeof(*t) + pdu_len,
195 0, pc);
196 if (!event)
197 return;
198 t = ring_buffer_event_data(event);
199 goto record_it;
200 }
201
202 /*
203 * A word about the locking here - we disable interrupts to reserve
204 * some space in the relay per-cpu buffer, to prevent an irq
205 * from coming in and stepping on our toes.
206 */
207 local_irq_save(flags);
208
209 if (unlikely(tsk->btrace_seq != blktrace_seq))
210 trace_note_tsk(bt, tsk);
211
212 t = relay_reserve(bt->rchan, sizeof(*t) + pdu_len);
213 if (t) {
214 sequence = per_cpu_ptr(bt->sequence, cpu);
215
216 t->magic = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE_VERSION;
217 t->sequence = ++(*sequence);
218 t->time = ktime_to_ns(ktime_get());
219record_it:
220 /*
221 * These two are not needed in ftrace as they are in the
222 * generic trace_entry, filled by tracing_generic_entry_update,
223 * but for the trace_event->bin() synthesizer benefit we do it
224 * here too.
225 */
226 t->cpu = cpu;
227 t->pid = pid;
228
229 t->sector = sector;
230 t->bytes = bytes;
231 t->action = what;
232 t->device = bt->dev;
233 t->error = error;
234 t->pdu_len = pdu_len;
235
236 if (pdu_len)
237 memcpy((void *) t + sizeof(*t), pdu_data, pdu_len);
238
239 if (blk_tr) {
240 trace_buffer_unlock_commit(blk_tr, event, 0, pc);
241 return;
242 }
243 }
244
245 local_irq_restore(flags);
246}
247
248static struct dentry *blk_tree_root;
249static DEFINE_MUTEX(blk_tree_mutex);
250
251static void blk_trace_cleanup(struct blk_trace *bt)
252{
253 debugfs_remove(bt->msg_file);
254 debugfs_remove(bt->dropped_file);
255 relay_close(bt->rchan);
256 free_percpu(bt->sequence);
257 free_percpu(bt->msg_data);
258 kfree(bt);
259 mutex_lock(&blk_probe_mutex);
260 if (atomic_dec_and_test(&blk_probes_ref))
261 blk_unregister_tracepoints();
262 mutex_unlock(&blk_probe_mutex);
263}
264
265int blk_trace_remove(struct request_queue *q)
266{
267 struct blk_trace *bt;
268
269 bt = xchg(&q->blk_trace, NULL);
270 if (!bt)
271 return -EINVAL;
272
273 if (bt->trace_state == Blktrace_setup ||
274 bt->trace_state == Blktrace_stopped)
275 blk_trace_cleanup(bt);
276
277 return 0;
278}
279EXPORT_SYMBOL_GPL(blk_trace_remove);
280
281static int blk_dropped_open(struct inode *inode, struct file *filp)
282{
283 filp->private_data = inode->i_private;
284
285 return 0;
286}
287
288static ssize_t blk_dropped_read(struct file *filp, char __user *buffer,
289 size_t count, loff_t *ppos)
290{
291 struct blk_trace *bt = filp->private_data;
292 char buf[16];
293
294 snprintf(buf, sizeof(buf), "%u\n", atomic_read(&bt->dropped));
295
296 return simple_read_from_buffer(buffer, count, ppos, buf, strlen(buf));
297}
298
299static const struct file_operations blk_dropped_fops = {
300 .owner = THIS_MODULE,
301 .open = blk_dropped_open,
302 .read = blk_dropped_read,
303};
304
305static int blk_msg_open(struct inode *inode, struct file *filp)
306{
307 filp->private_data = inode->i_private;
308
309 return 0;
310}
311
312static ssize_t blk_msg_write(struct file *filp, const char __user *buffer,
313 size_t count, loff_t *ppos)
314{
315 char *msg;
316 struct blk_trace *bt;
317
318 if (count > BLK_TN_MAX_MSG)
319 return -EINVAL;
320
321 msg = kmalloc(count, GFP_KERNEL);
322 if (msg == NULL)
323 return -ENOMEM;
324
325 if (copy_from_user(msg, buffer, count)) {
326 kfree(msg);
327 return -EFAULT;
328 }
329
330 bt = filp->private_data;
331 __trace_note_message(bt, "%s", msg);
332 kfree(msg);
333
334 return count;
335}
336
337static const struct file_operations blk_msg_fops = {
338 .owner = THIS_MODULE,
339 .open = blk_msg_open,
340 .write = blk_msg_write,
341};
342
343/*
344 * Keep track of how many times we encountered a full subbuffer, to aid
345 * the user space app in telling how many lost events there were.
346 */
347static int blk_subbuf_start_callback(struct rchan_buf *buf, void *subbuf,
348 void *prev_subbuf, size_t prev_padding)
349{
350 struct blk_trace *bt;
351
352 if (!relay_buf_full(buf))
353 return 1;
354
355 bt = buf->chan->private_data;
356 atomic_inc(&bt->dropped);
357 return 0;
358}
359
360static int blk_remove_buf_file_callback(struct dentry *dentry)
361{
362 struct dentry *parent = dentry->d_parent;
363 debugfs_remove(dentry);
364
365 /*
366 * this will fail for all but the last file, but that is ok. what we
367 * care about is the top level buts->name directory going away, when
368 * the last trace file is gone. Then we don't have to rmdir() that
369 * manually on trace stop, so it nicely solves the issue with
370 * force killing of running traces.
371 */
372
373 debugfs_remove(parent);
374 return 0;
375}
376
377static struct dentry *blk_create_buf_file_callback(const char *filename,
378 struct dentry *parent,
379 int mode,
380 struct rchan_buf *buf,
381 int *is_global)
382{
383 return debugfs_create_file(filename, mode, parent, buf,
384 &relay_file_operations);
385}
386
387static struct rchan_callbacks blk_relay_callbacks = {
388 .subbuf_start = blk_subbuf_start_callback,
389 .create_buf_file = blk_create_buf_file_callback,
390 .remove_buf_file = blk_remove_buf_file_callback,
391};
392
393/*
394 * Setup everything required to start tracing
395 */
396int do_blk_trace_setup(struct request_queue *q, char *name, dev_t dev,
397 struct blk_user_trace_setup *buts)
398{
399 struct blk_trace *old_bt, *bt = NULL;
400 struct dentry *dir = NULL;
401 int ret, i;
402
403 if (!buts->buf_size || !buts->buf_nr)
404 return -EINVAL;
405
406 strncpy(buts->name, name, BLKTRACE_BDEV_SIZE);
407 buts->name[BLKTRACE_BDEV_SIZE - 1] = '\0';
408
409 /*
410 * some device names have larger paths - convert the slashes
411 * to underscores for this to work as expected
412 */
413 for (i = 0; i < strlen(buts->name); i++)
414 if (buts->name[i] == '/')
415 buts->name[i] = '_';
416
417 ret = -ENOMEM;
418 bt = kzalloc(sizeof(*bt), GFP_KERNEL);
419 if (!bt)
420 goto err;
421
422 bt->sequence = alloc_percpu(unsigned long);
423 if (!bt->sequence)
424 goto err;
425
426 bt->msg_data = __alloc_percpu(BLK_TN_MAX_MSG);
427 if (!bt->msg_data)
428 goto err;
429
430 ret = -ENOENT;
431
432 if (!blk_tree_root) {
433 blk_tree_root = debugfs_create_dir("block", NULL);
434 if (!blk_tree_root)
435 return -ENOMEM;
436 }
437
438 dir = debugfs_create_dir(buts->name, blk_tree_root);
439
440 if (!dir)
441 goto err;
442
443 bt->dir = dir;
444 bt->dev = dev;
445 atomic_set(&bt->dropped, 0);
446
447 ret = -EIO;
448 bt->dropped_file = debugfs_create_file("dropped", 0444, dir, bt,
449 &blk_dropped_fops);
450 if (!bt->dropped_file)
451 goto err;
452
453 bt->msg_file = debugfs_create_file("msg", 0222, dir, bt, &blk_msg_fops);
454 if (!bt->msg_file)
455 goto err;
456
457 bt->rchan = relay_open("trace", dir, buts->buf_size,
458 buts->buf_nr, &blk_relay_callbacks, bt);
459 if (!bt->rchan)
460 goto err;
461
462 bt->act_mask = buts->act_mask;
463 if (!bt->act_mask)
464 bt->act_mask = (u16) -1;
465
466 bt->start_lba = buts->start_lba;
467 bt->end_lba = buts->end_lba;
468 if (!bt->end_lba)
469 bt->end_lba = -1ULL;
470
471 bt->pid = buts->pid;
472 bt->trace_state = Blktrace_setup;
473
474 mutex_lock(&blk_probe_mutex);
475 if (atomic_add_return(1, &blk_probes_ref) == 1) {
476 ret = blk_register_tracepoints();
477 if (ret)
478 goto probe_err;
479 }
480 mutex_unlock(&blk_probe_mutex);
481
482 ret = -EBUSY;
483 old_bt = xchg(&q->blk_trace, bt);
484 if (old_bt) {
485 (void) xchg(&q->blk_trace, old_bt);
486 goto err;
487 }
488
489 return 0;
490probe_err:
491 atomic_dec(&blk_probes_ref);
492 mutex_unlock(&blk_probe_mutex);
493err:
494 if (bt) {
495 if (bt->msg_file)
496 debugfs_remove(bt->msg_file);
497 if (bt->dropped_file)
498 debugfs_remove(bt->dropped_file);
499 free_percpu(bt->sequence);
500 free_percpu(bt->msg_data);
501 if (bt->rchan)
502 relay_close(bt->rchan);
503 kfree(bt);
504 }
505 return ret;
506}
507
508int blk_trace_setup(struct request_queue *q, char *name, dev_t dev,
509 char __user *arg)
510{
511 struct blk_user_trace_setup buts;
512 int ret;
513
514 ret = copy_from_user(&buts, arg, sizeof(buts));
515 if (ret)
516 return -EFAULT;
517
518 ret = do_blk_trace_setup(q, name, dev, &buts);
519 if (ret)
520 return ret;
521
522 if (copy_to_user(arg, &buts, sizeof(buts)))
523 return -EFAULT;
524
525 return 0;
526}
527EXPORT_SYMBOL_GPL(blk_trace_setup);
528
529int blk_trace_startstop(struct request_queue *q, int start)
530{
531 int ret;
532 struct blk_trace *bt = q->blk_trace;
533
534 if (bt == NULL)
535 return -EINVAL;
536
537 /*
538 * For starting a trace, we can transition from a setup or stopped
539 * trace. For stopping a trace, the state must be running
540 */
541 ret = -EINVAL;
542 if (start) {
543 if (bt->trace_state == Blktrace_setup ||
544 bt->trace_state == Blktrace_stopped) {
545 blktrace_seq++;
546 smp_mb();
547 bt->trace_state = Blktrace_running;
548
549 trace_note_time(bt);
550 ret = 0;
551 }
552 } else {
553 if (bt->trace_state == Blktrace_running) {
554 bt->trace_state = Blktrace_stopped;
555 relay_flush(bt->rchan);
556 ret = 0;
557 }
558 }
559
560 return ret;
561}
562EXPORT_SYMBOL_GPL(blk_trace_startstop);
563
564/**
565 * blk_trace_ioctl: - handle the ioctls associated with tracing
566 * @bdev: the block device
567 * @cmd: the ioctl cmd
568 * @arg: the argument data, if any
569 *
570 **/
571int blk_trace_ioctl(struct block_device *bdev, unsigned cmd, char __user *arg)
572{
573 struct request_queue *q;
574 int ret, start = 0;
575 char b[BDEVNAME_SIZE];
576
577 q = bdev_get_queue(bdev);
578 if (!q)
579 return -ENXIO;
580
581 mutex_lock(&bdev->bd_mutex);
582
583 switch (cmd) {
584 case BLKTRACESETUP:
585 bdevname(bdev, b);
586 ret = blk_trace_setup(q, b, bdev->bd_dev, arg);
587 break;
588 case BLKTRACESTART:
589 start = 1;
590 case BLKTRACESTOP:
591 ret = blk_trace_startstop(q, start);
592 break;
593 case BLKTRACETEARDOWN:
594 ret = blk_trace_remove(q);
595 break;
596 default:
597 ret = -ENOTTY;
598 break;
599 }
600
601 mutex_unlock(&bdev->bd_mutex);
602 return ret;
603}
604
605/**
606 * blk_trace_shutdown: - stop and cleanup trace structures
607 * @q: the request queue associated with the device
608 *
609 **/
610void blk_trace_shutdown(struct request_queue *q)
611{
612 if (q->blk_trace) {
613 blk_trace_startstop(q, 0);
614 blk_trace_remove(q);
615 }
616}
617
618/*
619 * blktrace probes
620 */
621
622/**
623 * blk_add_trace_rq - Add a trace for a request oriented action
624 * @q: queue the io is for
625 * @rq: the source request
626 * @what: the action
627 *
628 * Description:
629 * Records an action against a request. Will log the bio offset + size.
630 *
631 **/
632static void blk_add_trace_rq(struct request_queue *q, struct request *rq,
633 u32 what)
634{
635 struct blk_trace *bt = q->blk_trace;
636 int rw = rq->cmd_flags & 0x03;
637
638 if (likely(!bt))
639 return;
640
641 if (blk_discard_rq(rq))
642 rw |= (1 << BIO_RW_DISCARD);
643
644 if (blk_pc_request(rq)) {
645 what |= BLK_TC_ACT(BLK_TC_PC);
646 __blk_add_trace(bt, 0, rq->data_len, rw, what, rq->errors,
647 sizeof(rq->cmd), rq->cmd);
648 } else {
649 what |= BLK_TC_ACT(BLK_TC_FS);
650 __blk_add_trace(bt, rq->hard_sector, rq->hard_nr_sectors << 9,
651 rw, what, rq->errors, 0, NULL);
652 }
653}
654
655static void blk_add_trace_rq_abort(struct request_queue *q, struct request *rq)
656{
657 blk_add_trace_rq(q, rq, BLK_TA_ABORT);
658}
659
660static void blk_add_trace_rq_insert(struct request_queue *q, struct request *rq)
661{
662 blk_add_trace_rq(q, rq, BLK_TA_INSERT);
663}
664
665static void blk_add_trace_rq_issue(struct request_queue *q, struct request *rq)
666{
667 blk_add_trace_rq(q, rq, BLK_TA_ISSUE);
668}
669
670static void blk_add_trace_rq_requeue(struct request_queue *q,
671 struct request *rq)
672{
673 blk_add_trace_rq(q, rq, BLK_TA_REQUEUE);
674}
675
676static void blk_add_trace_rq_complete(struct request_queue *q,
677 struct request *rq)
678{
679 blk_add_trace_rq(q, rq, BLK_TA_COMPLETE);
680}
681
682/**
683 * blk_add_trace_bio - Add a trace for a bio oriented action
684 * @q: queue the io is for
685 * @bio: the source bio
686 * @what: the action
687 *
688 * Description:
689 * Records an action against a bio. Will log the bio offset + size.
690 *
691 **/
692static void blk_add_trace_bio(struct request_queue *q, struct bio *bio,
693 u32 what)
694{
695 struct blk_trace *bt = q->blk_trace;
696
697 if (likely(!bt))
698 return;
699
700 __blk_add_trace(bt, bio->bi_sector, bio->bi_size, bio->bi_rw, what,
701 !bio_flagged(bio, BIO_UPTODATE), 0, NULL);
702}
703
704static void blk_add_trace_bio_bounce(struct request_queue *q, struct bio *bio)
705{
706 blk_add_trace_bio(q, bio, BLK_TA_BOUNCE);
707}
708
709static void blk_add_trace_bio_complete(struct request_queue *q, struct bio *bio)
710{
711 blk_add_trace_bio(q, bio, BLK_TA_COMPLETE);
712}
713
714static void blk_add_trace_bio_backmerge(struct request_queue *q,
715 struct bio *bio)
716{
717 blk_add_trace_bio(q, bio, BLK_TA_BACKMERGE);
718}
719
720static void blk_add_trace_bio_frontmerge(struct request_queue *q,
721 struct bio *bio)
722{
723 blk_add_trace_bio(q, bio, BLK_TA_FRONTMERGE);
724}
725
726static void blk_add_trace_bio_queue(struct request_queue *q, struct bio *bio)
727{
728 blk_add_trace_bio(q, bio, BLK_TA_QUEUE);
729}
730
731static void blk_add_trace_getrq(struct request_queue *q,
732 struct bio *bio, int rw)
733{
734 if (bio)
735 blk_add_trace_bio(q, bio, BLK_TA_GETRQ);
736 else {
737 struct blk_trace *bt = q->blk_trace;
738
739 if (bt)
740 __blk_add_trace(bt, 0, 0, rw, BLK_TA_GETRQ, 0, 0, NULL);
741 }
742}
743
744
745static void blk_add_trace_sleeprq(struct request_queue *q,
746 struct bio *bio, int rw)
747{
748 if (bio)
749 blk_add_trace_bio(q, bio, BLK_TA_SLEEPRQ);
750 else {
751 struct blk_trace *bt = q->blk_trace;
752
753 if (bt)
754 __blk_add_trace(bt, 0, 0, rw, BLK_TA_SLEEPRQ,
755 0, 0, NULL);
756 }
757}
758
759static void blk_add_trace_plug(struct request_queue *q)
760{
761 struct blk_trace *bt = q->blk_trace;
762
763 if (bt)
764 __blk_add_trace(bt, 0, 0, 0, BLK_TA_PLUG, 0, 0, NULL);
765}
766
767static void blk_add_trace_unplug_io(struct request_queue *q)
768{
769 struct blk_trace *bt = q->blk_trace;
770
771 if (bt) {
772 unsigned int pdu = q->rq.count[READ] + q->rq.count[WRITE];
773 __be64 rpdu = cpu_to_be64(pdu);
774
775 __blk_add_trace(bt, 0, 0, 0, BLK_TA_UNPLUG_IO, 0,
776 sizeof(rpdu), &rpdu);
777 }
778}
779
780static void blk_add_trace_unplug_timer(struct request_queue *q)
781{
782 struct blk_trace *bt = q->blk_trace;
783
784 if (bt) {
785 unsigned int pdu = q->rq.count[READ] + q->rq.count[WRITE];
786 __be64 rpdu = cpu_to_be64(pdu);
787
788 __blk_add_trace(bt, 0, 0, 0, BLK_TA_UNPLUG_TIMER, 0,
789 sizeof(rpdu), &rpdu);
790 }
791}
792
793static void blk_add_trace_split(struct request_queue *q, struct bio *bio,
794 unsigned int pdu)
795{
796 struct blk_trace *bt = q->blk_trace;
797
798 if (bt) {
799 __be64 rpdu = cpu_to_be64(pdu);
800
801 __blk_add_trace(bt, bio->bi_sector, bio->bi_size, bio->bi_rw,
802 BLK_TA_SPLIT, !bio_flagged(bio, BIO_UPTODATE),
803 sizeof(rpdu), &rpdu);
804 }
805}
806
807/**
808 * blk_add_trace_remap - Add a trace for a remap operation
809 * @q: queue the io is for
810 * @bio: the source bio
811 * @dev: target device
812 * @from: source sector
813 * @to: target sector
814 *
815 * Description:
816 * Device mapper or raid target sometimes need to split a bio because
817 * it spans a stripe (or similar). Add a trace for that action.
818 *
819 **/
820static void blk_add_trace_remap(struct request_queue *q, struct bio *bio,
821 dev_t dev, sector_t from, sector_t to)
822{
823 struct blk_trace *bt = q->blk_trace;
824 struct blk_io_trace_remap r;
825
826 if (likely(!bt))
827 return;
828
829 r.device = cpu_to_be32(dev);
830 r.device_from = cpu_to_be32(bio->bi_bdev->bd_dev);
831 r.sector = cpu_to_be64(to);
832
833 __blk_add_trace(bt, from, bio->bi_size, bio->bi_rw, BLK_TA_REMAP,
834 !bio_flagged(bio, BIO_UPTODATE), sizeof(r), &r);
835}
836
837/**
838 * blk_add_driver_data - Add binary message with driver-specific data
839 * @q: queue the io is for
840 * @rq: io request
841 * @data: driver-specific data
842 * @len: length of driver-specific data
843 *
844 * Description:
845 * Some drivers might want to write driver-specific data per request.
846 *
847 **/
848void blk_add_driver_data(struct request_queue *q,
849 struct request *rq,
850 void *data, size_t len)
851{
852 struct blk_trace *bt = q->blk_trace;
853
854 if (likely(!bt))
855 return;
856
857 if (blk_pc_request(rq))
858 __blk_add_trace(bt, 0, rq->data_len, 0, BLK_TA_DRV_DATA,
859 rq->errors, len, data);
860 else
861 __blk_add_trace(bt, rq->hard_sector, rq->hard_nr_sectors << 9,
862 0, BLK_TA_DRV_DATA, rq->errors, len, data);
863}
864EXPORT_SYMBOL_GPL(blk_add_driver_data);
865
866static int blk_register_tracepoints(void)
867{
868 int ret;
869
870 ret = register_trace_block_rq_abort(blk_add_trace_rq_abort);
871 WARN_ON(ret);
872 ret = register_trace_block_rq_insert(blk_add_trace_rq_insert);
873 WARN_ON(ret);
874 ret = register_trace_block_rq_issue(blk_add_trace_rq_issue);
875 WARN_ON(ret);
876 ret = register_trace_block_rq_requeue(blk_add_trace_rq_requeue);
877 WARN_ON(ret);
878 ret = register_trace_block_rq_complete(blk_add_trace_rq_complete);
879 WARN_ON(ret);
880 ret = register_trace_block_bio_bounce(blk_add_trace_bio_bounce);
881 WARN_ON(ret);
882 ret = register_trace_block_bio_complete(blk_add_trace_bio_complete);
883 WARN_ON(ret);
884 ret = register_trace_block_bio_backmerge(blk_add_trace_bio_backmerge);
885 WARN_ON(ret);
886 ret = register_trace_block_bio_frontmerge(blk_add_trace_bio_frontmerge);
887 WARN_ON(ret);
888 ret = register_trace_block_bio_queue(blk_add_trace_bio_queue);
889 WARN_ON(ret);
890 ret = register_trace_block_getrq(blk_add_trace_getrq);
891 WARN_ON(ret);
892 ret = register_trace_block_sleeprq(blk_add_trace_sleeprq);
893 WARN_ON(ret);
894 ret = register_trace_block_plug(blk_add_trace_plug);
895 WARN_ON(ret);
896 ret = register_trace_block_unplug_timer(blk_add_trace_unplug_timer);
897 WARN_ON(ret);
898 ret = register_trace_block_unplug_io(blk_add_trace_unplug_io);
899 WARN_ON(ret);
900 ret = register_trace_block_split(blk_add_trace_split);
901 WARN_ON(ret);
902 ret = register_trace_block_remap(blk_add_trace_remap);
903 WARN_ON(ret);
904 return 0;
905}
906
907static void blk_unregister_tracepoints(void)
908{
909 unregister_trace_block_remap(blk_add_trace_remap);
910 unregister_trace_block_split(blk_add_trace_split);
911 unregister_trace_block_unplug_io(blk_add_trace_unplug_io);
912 unregister_trace_block_unplug_timer(blk_add_trace_unplug_timer);
913 unregister_trace_block_plug(blk_add_trace_plug);
914 unregister_trace_block_sleeprq(blk_add_trace_sleeprq);
915 unregister_trace_block_getrq(blk_add_trace_getrq);
916 unregister_trace_block_bio_queue(blk_add_trace_bio_queue);
917 unregister_trace_block_bio_frontmerge(blk_add_trace_bio_frontmerge);
918 unregister_trace_block_bio_backmerge(blk_add_trace_bio_backmerge);
919 unregister_trace_block_bio_complete(blk_add_trace_bio_complete);
920 unregister_trace_block_bio_bounce(blk_add_trace_bio_bounce);
921 unregister_trace_block_rq_complete(blk_add_trace_rq_complete);
922 unregister_trace_block_rq_requeue(blk_add_trace_rq_requeue);
923 unregister_trace_block_rq_issue(blk_add_trace_rq_issue);
924 unregister_trace_block_rq_insert(blk_add_trace_rq_insert);
925 unregister_trace_block_rq_abort(blk_add_trace_rq_abort);
926
927 tracepoint_synchronize_unregister();
928}
929
930/*
931 * struct blk_io_tracer formatting routines
932 */
933
934static void fill_rwbs(char *rwbs, const struct blk_io_trace *t)
935{
936 int i = 0;
937
938 if (t->action & BLK_TC_DISCARD)
939 rwbs[i++] = 'D';
940 else if (t->action & BLK_TC_WRITE)
941 rwbs[i++] = 'W';
942 else if (t->bytes)
943 rwbs[i++] = 'R';
944 else
945 rwbs[i++] = 'N';
946
947 if (t->action & BLK_TC_AHEAD)
948 rwbs[i++] = 'A';
949 if (t->action & BLK_TC_BARRIER)
950 rwbs[i++] = 'B';
951 if (t->action & BLK_TC_SYNC)
952 rwbs[i++] = 'S';
953 if (t->action & BLK_TC_META)
954 rwbs[i++] = 'M';
955
956 rwbs[i] = '\0';
957}
958
959static inline
960const struct blk_io_trace *te_blk_io_trace(const struct trace_entry *ent)
961{
962 return (const struct blk_io_trace *)ent;
963}
964
965static inline const void *pdu_start(const struct trace_entry *ent)
966{
967 return te_blk_io_trace(ent) + 1;
968}
969
970static inline u32 t_sec(const struct trace_entry *ent)
971{
972 return te_blk_io_trace(ent)->bytes >> 9;
973}
974
975static inline unsigned long long t_sector(const struct trace_entry *ent)
976{
977 return te_blk_io_trace(ent)->sector;
978}
979
980static inline __u16 t_error(const struct trace_entry *ent)
981{
982 return te_blk_io_trace(ent)->sector;
983}
984
985static __u64 get_pdu_int(const struct trace_entry *ent)
986{
987 const __u64 *val = pdu_start(ent);
988 return be64_to_cpu(*val);
989}
990
991static void get_pdu_remap(const struct trace_entry *ent,
992 struct blk_io_trace_remap *r)
993{
994 const struct blk_io_trace_remap *__r = pdu_start(ent);
995 __u64 sector = __r->sector;
996
997 r->device = be32_to_cpu(__r->device);
998 r->device_from = be32_to_cpu(__r->device_from);
999 r->sector = be64_to_cpu(sector);
1000}
1001
1002static int blk_log_action_iter(struct trace_iterator *iter, const char *act)
1003{
1004 char rwbs[6];
1005 unsigned long long ts = ns2usecs(iter->ts);
1006 unsigned long usec_rem = do_div(ts, USEC_PER_SEC);
1007 unsigned secs = (unsigned long)ts;
1008 const struct trace_entry *ent = iter->ent;
1009 const struct blk_io_trace *t = (const struct blk_io_trace *)ent;
1010
1011 fill_rwbs(rwbs, t);
1012
1013 return trace_seq_printf(&iter->seq,
1014 "%3d,%-3d %2d %5d.%06lu %5u %2s %3s ",
1015 MAJOR(t->device), MINOR(t->device), iter->cpu,
1016 secs, usec_rem, ent->pid, act, rwbs);
1017}
1018
1019static int blk_log_action_seq(struct trace_seq *s, const struct blk_io_trace *t,
1020 const char *act)
1021{
1022 char rwbs[6];
1023 fill_rwbs(rwbs, t);
1024 return trace_seq_printf(s, "%3d,%-3d %2s %3s ",
1025 MAJOR(t->device), MINOR(t->device), act, rwbs);
1026}
1027
1028static int blk_log_generic(struct trace_seq *s, const struct trace_entry *ent)
1029{
1030 const char *cmd = trace_find_cmdline(ent->pid);
1031
1032 if (t_sec(ent))
1033 return trace_seq_printf(s, "%llu + %u [%s]\n",
1034 t_sector(ent), t_sec(ent), cmd);
1035 return trace_seq_printf(s, "[%s]\n", cmd);
1036}
1037
1038static int blk_log_with_error(struct trace_seq *s,
1039 const struct trace_entry *ent)
1040{
1041 if (t_sec(ent))
1042 return trace_seq_printf(s, "%llu + %u [%d]\n", t_sector(ent),
1043 t_sec(ent), t_error(ent));
1044 return trace_seq_printf(s, "%llu [%d]\n", t_sector(ent), t_error(ent));
1045}
1046
1047static int blk_log_remap(struct trace_seq *s, const struct trace_entry *ent)
1048{
1049 struct blk_io_trace_remap r = { .device = 0, };
1050
1051 get_pdu_remap(ent, &r);
1052 return trace_seq_printf(s, "%llu + %u <- (%d,%d) %llu\n",
1053 t_sector(ent),
1054 t_sec(ent), MAJOR(r.device), MINOR(r.device),
1055 (unsigned long long)r.sector);
1056}
1057
1058static int blk_log_plug(struct trace_seq *s, const struct trace_entry *ent)
1059{
1060 return trace_seq_printf(s, "[%s]\n", trace_find_cmdline(ent->pid));
1061}
1062
1063static int blk_log_unplug(struct trace_seq *s, const struct trace_entry *ent)
1064{
1065 return trace_seq_printf(s, "[%s] %llu\n", trace_find_cmdline(ent->pid),
1066 get_pdu_int(ent));
1067}
1068
1069static int blk_log_split(struct trace_seq *s, const struct trace_entry *ent)
1070{
1071 return trace_seq_printf(s, "%llu / %llu [%s]\n", t_sector(ent),
1072 get_pdu_int(ent), trace_find_cmdline(ent->pid));
1073}
1074
1075/*
1076 * struct tracer operations
1077 */
1078
1079static void blk_tracer_print_header(struct seq_file *m)
1080{
1081 if (!(blk_tracer_flags.val & TRACE_BLK_OPT_CLASSIC))
1082 return;
1083 seq_puts(m, "# DEV CPU TIMESTAMP PID ACT FLG\n"
1084 "# | | | | | |\n");
1085}
1086
1087static void blk_tracer_start(struct trace_array *tr)
1088{
1089 mutex_lock(&blk_probe_mutex);
1090 if (atomic_add_return(1, &blk_probes_ref) == 1)
1091 if (blk_register_tracepoints())
1092 atomic_dec(&blk_probes_ref);
1093 mutex_unlock(&blk_probe_mutex);
1094 trace_flags &= ~TRACE_ITER_CONTEXT_INFO;
1095}
1096
1097static int blk_tracer_init(struct trace_array *tr)
1098{
1099 blk_tr = tr;
1100 blk_tracer_start(tr);
1101 mutex_lock(&blk_probe_mutex);
1102 blk_tracer_enabled++;
1103 mutex_unlock(&blk_probe_mutex);
1104 return 0;
1105}
1106
1107static void blk_tracer_stop(struct trace_array *tr)
1108{
1109 trace_flags |= TRACE_ITER_CONTEXT_INFO;
1110 mutex_lock(&blk_probe_mutex);
1111 if (atomic_dec_and_test(&blk_probes_ref))
1112 blk_unregister_tracepoints();
1113 mutex_unlock(&blk_probe_mutex);
1114}
1115
1116static void blk_tracer_reset(struct trace_array *tr)
1117{
1118 if (!atomic_read(&blk_probes_ref))
1119 return;
1120
1121 mutex_lock(&blk_probe_mutex);
1122 blk_tracer_enabled--;
1123 WARN_ON(blk_tracer_enabled < 0);
1124 mutex_unlock(&blk_probe_mutex);
1125
1126 blk_tracer_stop(tr);
1127}
1128
1129static struct {
1130 const char *act[2];
1131 int (*print)(struct trace_seq *s, const struct trace_entry *ent);
1132} what2act[] __read_mostly = {
1133 [__BLK_TA_QUEUE] = {{ "Q", "queue" }, blk_log_generic },
1134 [__BLK_TA_BACKMERGE] = {{ "M", "backmerge" }, blk_log_generic },
1135 [__BLK_TA_FRONTMERGE] = {{ "F", "frontmerge" }, blk_log_generic },
1136 [__BLK_TA_GETRQ] = {{ "G", "getrq" }, blk_log_generic },
1137 [__BLK_TA_SLEEPRQ] = {{ "S", "sleeprq" }, blk_log_generic },
1138 [__BLK_TA_REQUEUE] = {{ "R", "requeue" }, blk_log_with_error },
1139 [__BLK_TA_ISSUE] = {{ "D", "issue" }, blk_log_generic },
1140 [__BLK_TA_COMPLETE] = {{ "C", "complete" }, blk_log_with_error },
1141 [__BLK_TA_PLUG] = {{ "P", "plug" }, blk_log_plug },
1142 [__BLK_TA_UNPLUG_IO] = {{ "U", "unplug_io" }, blk_log_unplug },
1143 [__BLK_TA_UNPLUG_TIMER] = {{ "UT", "unplug_timer" }, blk_log_unplug },
1144 [__BLK_TA_INSERT] = {{ "I", "insert" }, blk_log_generic },
1145 [__BLK_TA_SPLIT] = {{ "X", "split" }, blk_log_split },
1146 [__BLK_TA_BOUNCE] = {{ "B", "bounce" }, blk_log_generic },
1147 [__BLK_TA_REMAP] = {{ "A", "remap" }, blk_log_remap },
1148};
1149
1150static enum print_line_t blk_trace_event_print(struct trace_iterator *iter,
1151 int flags)
1152{
1153 struct trace_seq *s = &iter->seq;
1154 const struct blk_io_trace *t = (struct blk_io_trace *)iter->ent;
1155 const u16 what = t->action & ((1 << BLK_TC_SHIFT) - 1);
1156 int ret;
1157
1158 if (!trace_print_context(iter))
1159 return TRACE_TYPE_PARTIAL_LINE;
1160
1161 if (unlikely(what == 0 || what > ARRAY_SIZE(what2act)))
1162 ret = trace_seq_printf(s, "Bad pc action %x\n", what);
1163 else {
1164 const bool long_act = !!(trace_flags & TRACE_ITER_VERBOSE);
1165 ret = blk_log_action_seq(s, t, what2act[what].act[long_act]);
1166 if (ret)
1167 ret = what2act[what].print(s, iter->ent);
1168 }
1169
1170 return ret ? TRACE_TYPE_HANDLED : TRACE_TYPE_PARTIAL_LINE;
1171}
1172
1173static int blk_trace_synthesize_old_trace(struct trace_iterator *iter)
1174{
1175 struct trace_seq *s = &iter->seq;
1176 struct blk_io_trace *t = (struct blk_io_trace *)iter->ent;
1177 const int offset = offsetof(struct blk_io_trace, sector);
1178 struct blk_io_trace old = {
1179 .magic = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE_VERSION,
1180 .time = ns2usecs(iter->ts),
1181 };
1182
1183 if (!trace_seq_putmem(s, &old, offset))
1184 return 0;
1185 return trace_seq_putmem(s, &t->sector,
1186 sizeof(old) - offset + t->pdu_len);
1187}
1188
1189static enum print_line_t
1190blk_trace_event_print_binary(struct trace_iterator *iter, int flags)
1191{
1192 return blk_trace_synthesize_old_trace(iter) ?
1193 TRACE_TYPE_HANDLED : TRACE_TYPE_PARTIAL_LINE;
1194}
1195
1196static enum print_line_t blk_tracer_print_line(struct trace_iterator *iter)
1197{
1198 const struct blk_io_trace *t;
1199 u16 what;
1200 int ret;
1201
1202 if (!(blk_tracer_flags.val & TRACE_BLK_OPT_CLASSIC))
1203 return TRACE_TYPE_UNHANDLED;
1204
1205 t = (const struct blk_io_trace *)iter->ent;
1206 what = t->action & ((1 << BLK_TC_SHIFT) - 1);
1207
1208 if (unlikely(what == 0 || what > ARRAY_SIZE(what2act)))
1209 ret = trace_seq_printf(&iter->seq, "Bad pc action %x\n", what);
1210 else {
1211 const bool long_act = !!(trace_flags & TRACE_ITER_VERBOSE);
1212 ret = blk_log_action_iter(iter, what2act[what].act[long_act]);
1213 if (ret)
1214 ret = what2act[what].print(&iter->seq, iter->ent);
1215 }
1216
1217 return ret ? TRACE_TYPE_HANDLED : TRACE_TYPE_PARTIAL_LINE;
1218}
1219
1220static struct tracer blk_tracer __read_mostly = {
1221 .name = "blk",
1222 .init = blk_tracer_init,
1223 .reset = blk_tracer_reset,
1224 .start = blk_tracer_start,
1225 .stop = blk_tracer_stop,
1226 .print_header = blk_tracer_print_header,
1227 .print_line = blk_tracer_print_line,
1228 .flags = &blk_tracer_flags,
1229};
1230
1231static struct trace_event trace_blk_event = {
1232 .type = TRACE_BLK,
1233 .trace = blk_trace_event_print,
1234 .latency_trace = blk_trace_event_print,
1235 .binary = blk_trace_event_print_binary,
1236};
1237
1238static int __init init_blk_tracer(void)
1239{
1240 if (!register_ftrace_event(&trace_blk_event)) {
1241 pr_warning("Warning: could not register block events\n");
1242 return 1;
1243 }
1244
1245 if (register_tracer(&blk_tracer) != 0) {
1246 pr_warning("Warning: could not register the block tracer\n");
1247 unregister_ftrace_event(&trace_blk_event);
1248 return 1;
1249 }
1250
1251 return 0;
1252}
1253
1254device_initcall(init_blk_tracer);
1255
1256static int blk_trace_remove_queue(struct request_queue *q)
1257{
1258 struct blk_trace *bt;
1259
1260 bt = xchg(&q->blk_trace, NULL);
1261 if (bt == NULL)
1262 return -EINVAL;
1263
1264 kfree(bt);
1265 return 0;
1266}
1267
1268/*
1269 * Setup everything required to start tracing
1270 */
1271static int blk_trace_setup_queue(struct request_queue *q, dev_t dev)
1272{
1273 struct blk_trace *old_bt, *bt = NULL;
1274 int ret;
1275
1276 ret = -ENOMEM;
1277 bt = kzalloc(sizeof(*bt), GFP_KERNEL);
1278 if (!bt)
1279 goto err;
1280
1281 bt->dev = dev;
1282 bt->act_mask = (u16)-1;
1283 bt->end_lba = -1ULL;
1284 bt->trace_state = Blktrace_running;
1285
1286 old_bt = xchg(&q->blk_trace, bt);
1287 if (old_bt != NULL) {
1288 (void)xchg(&q->blk_trace, old_bt);
1289 kfree(bt);
1290 ret = -EBUSY;
1291 }
1292 return 0;
1293err:
1294 return ret;
1295}
1296
1297/*
1298 * sysfs interface to enable and configure tracing
1299 */
1300
1301static ssize_t sysfs_blk_trace_enable_show(struct device *dev,
1302 struct device_attribute *attr,
1303 char *buf)
1304{
1305 struct hd_struct *p = dev_to_part(dev);
1306 struct block_device *bdev;
1307 ssize_t ret = -ENXIO;
1308
1309 lock_kernel();
1310 bdev = bdget(part_devt(p));
1311 if (bdev != NULL) {
1312 struct request_queue *q = bdev_get_queue(bdev);
1313
1314 if (q != NULL) {
1315 mutex_lock(&bdev->bd_mutex);
1316 ret = sprintf(buf, "%u\n", !!q->blk_trace);
1317 mutex_unlock(&bdev->bd_mutex);
1318 }
1319
1320 bdput(bdev);
1321 }
1322
1323 unlock_kernel();
1324 return ret;
1325}
1326
1327static ssize_t sysfs_blk_trace_enable_store(struct device *dev,
1328 struct device_attribute *attr,
1329 const char *buf, size_t count)
1330{
1331 struct block_device *bdev;
1332 struct request_queue *q;
1333 struct hd_struct *p;
1334 int value;
1335 ssize_t ret = -ENXIO;
1336
1337 if (count == 0 || sscanf(buf, "%d", &value) != 1)
1338 goto out;
1339
1340 lock_kernel();
1341 p = dev_to_part(dev);
1342 bdev = bdget(part_devt(p));
1343 if (bdev == NULL)
1344 goto out_unlock_kernel;
1345
1346 q = bdev_get_queue(bdev);
1347 if (q == NULL)
1348 goto out_bdput;
1349
1350 mutex_lock(&bdev->bd_mutex);
1351 if (value)
1352 ret = blk_trace_setup_queue(q, bdev->bd_dev);
1353 else
1354 ret = blk_trace_remove_queue(q);
1355 mutex_unlock(&bdev->bd_mutex);
1356
1357 if (ret == 0)
1358 ret = count;
1359out_bdput:
1360 bdput(bdev);
1361out_unlock_kernel:
1362 unlock_kernel();
1363out:
1364 return ret;
1365}
1366
1367static ssize_t sysfs_blk_trace_attr_show(struct device *dev,
1368 struct device_attribute *attr,
1369 char *buf);
1370static ssize_t sysfs_blk_trace_attr_store(struct device *dev,
1371 struct device_attribute *attr,
1372 const char *buf, size_t count);
1373#define BLK_TRACE_DEVICE_ATTR(_name) \
1374 DEVICE_ATTR(_name, S_IRUGO | S_IWUSR, \
1375 sysfs_blk_trace_attr_show, \
1376 sysfs_blk_trace_attr_store)
1377
1378static DEVICE_ATTR(enable, S_IRUGO | S_IWUSR,
1379 sysfs_blk_trace_enable_show, sysfs_blk_trace_enable_store);
1380static BLK_TRACE_DEVICE_ATTR(act_mask);
1381static BLK_TRACE_DEVICE_ATTR(pid);
1382static BLK_TRACE_DEVICE_ATTR(start_lba);
1383static BLK_TRACE_DEVICE_ATTR(end_lba);
1384
1385static struct attribute *blk_trace_attrs[] = {
1386 &dev_attr_enable.attr,
1387 &dev_attr_act_mask.attr,
1388 &dev_attr_pid.attr,
1389 &dev_attr_start_lba.attr,
1390 &dev_attr_end_lba.attr,
1391 NULL
1392};
1393
1394struct attribute_group blk_trace_attr_group = {
1395 .name = "trace",
1396 .attrs = blk_trace_attrs,
1397};
1398
1399static int blk_str2act_mask(const char *str)
1400{
1401 int mask = 0;
1402 char *copy = kstrdup(str, GFP_KERNEL), *s;
1403
1404 if (copy == NULL)
1405 return -ENOMEM;
1406
1407 s = strstrip(copy);
1408
1409 while (1) {
1410 char *sep = strchr(s, ',');
1411
1412 if (sep != NULL)
1413 *sep = '\0';
1414
1415 if (strcasecmp(s, "barrier") == 0)
1416 mask |= BLK_TC_BARRIER;
1417 else if (strcasecmp(s, "complete") == 0)
1418 mask |= BLK_TC_COMPLETE;
1419 else if (strcasecmp(s, "fs") == 0)
1420 mask |= BLK_TC_FS;
1421 else if (strcasecmp(s, "issue") == 0)
1422 mask |= BLK_TC_ISSUE;
1423 else if (strcasecmp(s, "pc") == 0)
1424 mask |= BLK_TC_PC;
1425 else if (strcasecmp(s, "queue") == 0)
1426 mask |= BLK_TC_QUEUE;
1427 else if (strcasecmp(s, "read") == 0)
1428 mask |= BLK_TC_READ;
1429 else if (strcasecmp(s, "requeue") == 0)
1430 mask |= BLK_TC_REQUEUE;
1431 else if (strcasecmp(s, "sync") == 0)
1432 mask |= BLK_TC_SYNC;
1433 else if (strcasecmp(s, "write") == 0)
1434 mask |= BLK_TC_WRITE;
1435
1436 if (sep == NULL)
1437 break;
1438
1439 s = sep + 1;
1440 }
1441 kfree(copy);
1442
1443 return mask;
1444}
1445
1446static ssize_t sysfs_blk_trace_attr_show(struct device *dev,
1447 struct device_attribute *attr,
1448 char *buf)
1449{
1450 struct hd_struct *p = dev_to_part(dev);
1451 struct request_queue *q;
1452 struct block_device *bdev;
1453 ssize_t ret = -ENXIO;
1454
1455 lock_kernel();
1456 bdev = bdget(part_devt(p));
1457 if (bdev == NULL)
1458 goto out_unlock_kernel;
1459
1460 q = bdev_get_queue(bdev);
1461 if (q == NULL)
1462 goto out_bdput;
1463 mutex_lock(&bdev->bd_mutex);
1464 if (q->blk_trace == NULL)
1465 ret = sprintf(buf, "disabled\n");
1466 else if (attr == &dev_attr_act_mask)
1467 ret = sprintf(buf, "%#x\n", q->blk_trace->act_mask);
1468 else if (attr == &dev_attr_pid)
1469 ret = sprintf(buf, "%u\n", q->blk_trace->pid);
1470 else if (attr == &dev_attr_start_lba)
1471 ret = sprintf(buf, "%llu\n", q->blk_trace->start_lba);
1472 else if (attr == &dev_attr_end_lba)
1473 ret = sprintf(buf, "%llu\n", q->blk_trace->end_lba);
1474 mutex_unlock(&bdev->bd_mutex);
1475out_bdput:
1476 bdput(bdev);
1477out_unlock_kernel:
1478 unlock_kernel();
1479 return ret;
1480}
1481
1482static ssize_t sysfs_blk_trace_attr_store(struct device *dev,
1483 struct device_attribute *attr,
1484 const char *buf, size_t count)
1485{
1486 struct block_device *bdev;
1487 struct request_queue *q;
1488 struct hd_struct *p;
1489 u64 value;
1490 ssize_t ret = -ENXIO;
1491
1492 if (count == 0)
1493 goto out;
1494
1495 if (attr == &dev_attr_act_mask) {
1496 if (sscanf(buf, "%llx", &value) != 1) {
1497 /* Assume it is a list of trace category names */
1498 value = blk_str2act_mask(buf);
1499 if (value < 0)
1500 goto out;
1501 }
1502 } else if (sscanf(buf, "%llu", &value) != 1)
1503 goto out;
1504
1505 lock_kernel();
1506 p = dev_to_part(dev);
1507 bdev = bdget(part_devt(p));
1508 if (bdev == NULL)
1509 goto out_unlock_kernel;
1510
1511 q = bdev_get_queue(bdev);
1512 if (q == NULL)
1513 goto out_bdput;
1514
1515 mutex_lock(&bdev->bd_mutex);
1516 ret = 0;
1517 if (q->blk_trace == NULL)
1518 ret = blk_trace_setup_queue(q, bdev->bd_dev);
1519
1520 if (ret == 0) {
1521 if (attr == &dev_attr_act_mask)
1522 q->blk_trace->act_mask = value;
1523 else if (attr == &dev_attr_pid)
1524 q->blk_trace->pid = value;
1525 else if (attr == &dev_attr_start_lba)
1526 q->blk_trace->start_lba = value;
1527 else if (attr == &dev_attr_end_lba)
1528 q->blk_trace->end_lba = value;
1529 ret = count;
1530 }
1531 mutex_unlock(&bdev->bd_mutex);
1532out_bdput:
1533 bdput(bdev);
1534out_unlock_kernel:
1535 unlock_kernel();
1536out:
1537 return ret;
1538}
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
index 11ad796ca049..5a3a06b21eee 100644
--- a/kernel/trace/ftrace.c
+++ b/kernel/trace/ftrace.c
@@ -27,6 +27,7 @@
27#include <linux/sysctl.h> 27#include <linux/sysctl.h>
28#include <linux/ctype.h> 28#include <linux/ctype.h>
29#include <linux/list.h> 29#include <linux/list.h>
30#include <linux/hash.h>
30 31
31#include <asm/ftrace.h> 32#include <asm/ftrace.h>
32 33
@@ -44,14 +45,14 @@
44 ftrace_kill(); \ 45 ftrace_kill(); \
45 } while (0) 46 } while (0)
46 47
48/* hash bits for specific function selection */
49#define FTRACE_HASH_BITS 7
50#define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)
51
47/* ftrace_enabled is a method to turn ftrace on or off */ 52/* ftrace_enabled is a method to turn ftrace on or off */
48int ftrace_enabled __read_mostly; 53int ftrace_enabled __read_mostly;
49static int last_ftrace_enabled; 54static int last_ftrace_enabled;
50 55
51/* set when tracing only a pid */
52struct pid *ftrace_pid_trace;
53static struct pid * const ftrace_swapper_pid = &init_struct_pid;
54
55/* Quick disabling of function tracer. */ 56/* Quick disabling of function tracer. */
56int function_trace_stop; 57int function_trace_stop;
57 58
@@ -61,9 +62,7 @@ int function_trace_stop;
61 */ 62 */
62static int ftrace_disabled __read_mostly; 63static int ftrace_disabled __read_mostly;
63 64
64static DEFINE_SPINLOCK(ftrace_lock); 65static DEFINE_MUTEX(ftrace_lock);
65static DEFINE_MUTEX(ftrace_sysctl_lock);
66static DEFINE_MUTEX(ftrace_start_lock);
67 66
68static struct ftrace_ops ftrace_list_end __read_mostly = 67static struct ftrace_ops ftrace_list_end __read_mostly =
69{ 68{
@@ -134,9 +133,6 @@ static void ftrace_test_stop_func(unsigned long ip, unsigned long parent_ip)
134 133
135static int __register_ftrace_function(struct ftrace_ops *ops) 134static int __register_ftrace_function(struct ftrace_ops *ops)
136{ 135{
137 /* should not be called from interrupt context */
138 spin_lock(&ftrace_lock);
139
140 ops->next = ftrace_list; 136 ops->next = ftrace_list;
141 /* 137 /*
142 * We are entering ops into the ftrace_list but another 138 * We are entering ops into the ftrace_list but another
@@ -172,18 +168,12 @@ static int __register_ftrace_function(struct ftrace_ops *ops)
172#endif 168#endif
173 } 169 }
174 170
175 spin_unlock(&ftrace_lock);
176
177 return 0; 171 return 0;
178} 172}
179 173
180static int __unregister_ftrace_function(struct ftrace_ops *ops) 174static int __unregister_ftrace_function(struct ftrace_ops *ops)
181{ 175{
182 struct ftrace_ops **p; 176 struct ftrace_ops **p;
183 int ret = 0;
184
185 /* should not be called from interrupt context */
186 spin_lock(&ftrace_lock);
187 177
188 /* 178 /*
189 * If we are removing the last function, then simply point 179 * If we are removing the last function, then simply point
@@ -192,17 +182,15 @@ static int __unregister_ftrace_function(struct ftrace_ops *ops)
192 if (ftrace_list == ops && ops->next == &ftrace_list_end) { 182 if (ftrace_list == ops && ops->next == &ftrace_list_end) {
193 ftrace_trace_function = ftrace_stub; 183 ftrace_trace_function = ftrace_stub;
194 ftrace_list = &ftrace_list_end; 184 ftrace_list = &ftrace_list_end;
195 goto out; 185 return 0;
196 } 186 }
197 187
198 for (p = &ftrace_list; *p != &ftrace_list_end; p = &(*p)->next) 188 for (p = &ftrace_list; *p != &ftrace_list_end; p = &(*p)->next)
199 if (*p == ops) 189 if (*p == ops)
200 break; 190 break;
201 191
202 if (*p != ops) { 192 if (*p != ops)
203 ret = -1; 193 return -1;
204 goto out;
205 }
206 194
207 *p = (*p)->next; 195 *p = (*p)->next;
208 196
@@ -223,18 +211,14 @@ static int __unregister_ftrace_function(struct ftrace_ops *ops)
223 } 211 }
224 } 212 }
225 213
226 out: 214 return 0;
227 spin_unlock(&ftrace_lock);
228
229 return ret;
230} 215}
231 216
232static void ftrace_update_pid_func(void) 217static void ftrace_update_pid_func(void)
233{ 218{
234 ftrace_func_t func; 219 ftrace_func_t func;
235 220
236 /* should not be called from interrupt context */ 221 mutex_lock(&ftrace_lock);
237 spin_lock(&ftrace_lock);
238 222
239 if (ftrace_trace_function == ftrace_stub) 223 if (ftrace_trace_function == ftrace_stub)
240 goto out; 224 goto out;
@@ -256,21 +240,30 @@ static void ftrace_update_pid_func(void)
256#endif 240#endif
257 241
258 out: 242 out:
259 spin_unlock(&ftrace_lock); 243 mutex_unlock(&ftrace_lock);
260} 244}
261 245
246/* set when tracing only a pid */
247struct pid *ftrace_pid_trace;
248static struct pid * const ftrace_swapper_pid = &init_struct_pid;
249
262#ifdef CONFIG_DYNAMIC_FTRACE 250#ifdef CONFIG_DYNAMIC_FTRACE
251
263#ifndef CONFIG_FTRACE_MCOUNT_RECORD 252#ifndef CONFIG_FTRACE_MCOUNT_RECORD
264# error Dynamic ftrace depends on MCOUNT_RECORD 253# error Dynamic ftrace depends on MCOUNT_RECORD
265#endif 254#endif
266 255
267/* 256static struct hlist_head ftrace_func_hash[FTRACE_FUNC_HASHSIZE] __read_mostly;
268 * Since MCOUNT_ADDR may point to mcount itself, we do not want 257
269 * to get it confused by reading a reference in the code as we 258struct ftrace_func_probe {
270 * are parsing on objcopy output of text. Use a variable for 259 struct hlist_node node;
271 * it instead. 260 struct ftrace_probe_ops *ops;
272 */ 261 unsigned long flags;
273static unsigned long mcount_addr = MCOUNT_ADDR; 262 unsigned long ip;
263 void *data;
264 struct rcu_head rcu;
265};
266
274 267
275enum { 268enum {
276 FTRACE_ENABLE_CALLS = (1 << 0), 269 FTRACE_ENABLE_CALLS = (1 << 0),
@@ -290,7 +283,7 @@ static DEFINE_MUTEX(ftrace_regex_lock);
290 283
291struct ftrace_page { 284struct ftrace_page {
292 struct ftrace_page *next; 285 struct ftrace_page *next;
293 unsigned long index; 286 int index;
294 struct dyn_ftrace records[]; 287 struct dyn_ftrace records[];
295}; 288};
296 289
@@ -305,6 +298,19 @@ static struct ftrace_page *ftrace_pages;
305 298
306static struct dyn_ftrace *ftrace_free_records; 299static struct dyn_ftrace *ftrace_free_records;
307 300
301/*
302 * This is a double for. Do not use 'break' to break out of the loop,
303 * you must use a goto.
304 */
305#define do_for_each_ftrace_rec(pg, rec) \
306 for (pg = ftrace_pages_start; pg; pg = pg->next) { \
307 int _____i; \
308 for (_____i = 0; _____i < pg->index; _____i++) { \
309 rec = &pg->records[_____i];
310
311#define while_for_each_ftrace_rec() \
312 } \
313 }
308 314
309#ifdef CONFIG_KPROBES 315#ifdef CONFIG_KPROBES
310 316
@@ -349,23 +355,16 @@ void ftrace_release(void *start, unsigned long size)
349 struct ftrace_page *pg; 355 struct ftrace_page *pg;
350 unsigned long s = (unsigned long)start; 356 unsigned long s = (unsigned long)start;
351 unsigned long e = s + size; 357 unsigned long e = s + size;
352 int i;
353 358
354 if (ftrace_disabled || !start) 359 if (ftrace_disabled || !start)
355 return; 360 return;
356 361
357 /* should not be called from interrupt context */ 362 mutex_lock(&ftrace_lock);
358 spin_lock(&ftrace_lock); 363 do_for_each_ftrace_rec(pg, rec) {
359 364 if ((rec->ip >= s) && (rec->ip < e))
360 for (pg = ftrace_pages_start; pg; pg = pg->next) { 365 ftrace_free_rec(rec);
361 for (i = 0; i < pg->index; i++) { 366 } while_for_each_ftrace_rec();
362 rec = &pg->records[i]; 367 mutex_unlock(&ftrace_lock);
363
364 if ((rec->ip >= s) && (rec->ip < e))
365 ftrace_free_rec(rec);
366 }
367 }
368 spin_unlock(&ftrace_lock);
369} 368}
370 369
371static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip) 370static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip)
@@ -461,10 +460,10 @@ static void ftrace_bug(int failed, unsigned long ip)
461static int 460static int
462__ftrace_replace_code(struct dyn_ftrace *rec, int enable) 461__ftrace_replace_code(struct dyn_ftrace *rec, int enable)
463{ 462{
464 unsigned long ip, fl;
465 unsigned long ftrace_addr; 463 unsigned long ftrace_addr;
464 unsigned long ip, fl;
466 465
467 ftrace_addr = (unsigned long)ftrace_caller; 466 ftrace_addr = (unsigned long)FTRACE_ADDR;
468 467
469 ip = rec->ip; 468 ip = rec->ip;
470 469
@@ -473,7 +472,7 @@ __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
473 * it is not enabled then do nothing. 472 * it is not enabled then do nothing.
474 * 473 *
475 * If this record is not to be traced and 474 * If this record is not to be traced and
476 * it is enabled then disabled it. 475 * it is enabled then disable it.
477 * 476 *
478 */ 477 */
479 if (rec->flags & FTRACE_FL_NOTRACE) { 478 if (rec->flags & FTRACE_FL_NOTRACE) {
@@ -493,7 +492,7 @@ __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
493 if (fl == (FTRACE_FL_FILTER | FTRACE_FL_ENABLED)) 492 if (fl == (FTRACE_FL_FILTER | FTRACE_FL_ENABLED))
494 return 0; 493 return 0;
495 494
496 /* Record is not filtered and is not enabled do nothing */ 495 /* Record is not filtered or enabled, do nothing */
497 if (!fl) 496 if (!fl)
498 return 0; 497 return 0;
499 498
@@ -515,7 +514,7 @@ __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
515 514
516 } else { 515 } else {
517 516
518 /* if record is not enabled do nothing */ 517 /* if record is not enabled, do nothing */
519 if (!(rec->flags & FTRACE_FL_ENABLED)) 518 if (!(rec->flags & FTRACE_FL_ENABLED))
520 return 0; 519 return 0;
521 520
@@ -531,44 +530,40 @@ __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
531 530
532static void ftrace_replace_code(int enable) 531static void ftrace_replace_code(int enable)
533{ 532{
534 int i, failed;
535 struct dyn_ftrace *rec; 533 struct dyn_ftrace *rec;
536 struct ftrace_page *pg; 534 struct ftrace_page *pg;
535 int failed;
537 536
538 for (pg = ftrace_pages_start; pg; pg = pg->next) { 537 do_for_each_ftrace_rec(pg, rec) {
539 for (i = 0; i < pg->index; i++) { 538 /*
540 rec = &pg->records[i]; 539 * Skip over free records and records that have
541 540 * failed.
542 /* 541 */
543 * Skip over free records and records that have 542 if (rec->flags & FTRACE_FL_FREE ||
544 * failed. 543 rec->flags & FTRACE_FL_FAILED)
545 */ 544 continue;
546 if (rec->flags & FTRACE_FL_FREE ||
547 rec->flags & FTRACE_FL_FAILED)
548 continue;
549 545
550 /* ignore updates to this record's mcount site */ 546 /* ignore updates to this record's mcount site */
551 if (get_kprobe((void *)rec->ip)) { 547 if (get_kprobe((void *)rec->ip)) {
552 freeze_record(rec); 548 freeze_record(rec);
553 continue; 549 continue;
554 } else { 550 } else {
555 unfreeze_record(rec); 551 unfreeze_record(rec);
556 } 552 }
557 553
558 failed = __ftrace_replace_code(rec, enable); 554 failed = __ftrace_replace_code(rec, enable);
559 if (failed && (rec->flags & FTRACE_FL_CONVERTED)) { 555 if (failed && (rec->flags & FTRACE_FL_CONVERTED)) {
560 rec->flags |= FTRACE_FL_FAILED; 556 rec->flags |= FTRACE_FL_FAILED;
561 if ((system_state == SYSTEM_BOOTING) || 557 if ((system_state == SYSTEM_BOOTING) ||
562 !core_kernel_text(rec->ip)) { 558 !core_kernel_text(rec->ip)) {
563 ftrace_free_rec(rec); 559 ftrace_free_rec(rec);
564 } else { 560 } else {
565 ftrace_bug(failed, rec->ip); 561 ftrace_bug(failed, rec->ip);
566 /* Stop processing */ 562 /* Stop processing */
567 return; 563 return;
568 } 564 }
569 }
570 } 565 }
571 } 566 } while_for_each_ftrace_rec();
572} 567}
573 568
574static int 569static int
@@ -579,7 +574,7 @@ ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
579 574
580 ip = rec->ip; 575 ip = rec->ip;
581 576
582 ret = ftrace_make_nop(mod, rec, mcount_addr); 577 ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
583 if (ret) { 578 if (ret) {
584 ftrace_bug(ret, ip); 579 ftrace_bug(ret, ip);
585 rec->flags |= FTRACE_FL_FAILED; 580 rec->flags |= FTRACE_FL_FAILED;
@@ -662,13 +657,10 @@ static void ftrace_startup(int command)
662 if (unlikely(ftrace_disabled)) 657 if (unlikely(ftrace_disabled))
663 return; 658 return;
664 659
665 mutex_lock(&ftrace_start_lock);
666 ftrace_start_up++; 660 ftrace_start_up++;
667 command |= FTRACE_ENABLE_CALLS; 661 command |= FTRACE_ENABLE_CALLS;
668 662
669 ftrace_startup_enable(command); 663 ftrace_startup_enable(command);
670
671 mutex_unlock(&ftrace_start_lock);
672} 664}
673 665
674static void ftrace_shutdown(int command) 666static void ftrace_shutdown(int command)
@@ -676,7 +668,6 @@ static void ftrace_shutdown(int command)
676 if (unlikely(ftrace_disabled)) 668 if (unlikely(ftrace_disabled))
677 return; 669 return;
678 670
679 mutex_lock(&ftrace_start_lock);
680 ftrace_start_up--; 671 ftrace_start_up--;
681 if (!ftrace_start_up) 672 if (!ftrace_start_up)
682 command |= FTRACE_DISABLE_CALLS; 673 command |= FTRACE_DISABLE_CALLS;
@@ -687,11 +678,9 @@ static void ftrace_shutdown(int command)
687 } 678 }
688 679
689 if (!command || !ftrace_enabled) 680 if (!command || !ftrace_enabled)
690 goto out; 681 return;
691 682
692 ftrace_run_update_code(command); 683 ftrace_run_update_code(command);
693 out:
694 mutex_unlock(&ftrace_start_lock);
695} 684}
696 685
697static void ftrace_startup_sysctl(void) 686static void ftrace_startup_sysctl(void)
@@ -701,7 +690,6 @@ static void ftrace_startup_sysctl(void)
701 if (unlikely(ftrace_disabled)) 690 if (unlikely(ftrace_disabled))
702 return; 691 return;
703 692
704 mutex_lock(&ftrace_start_lock);
705 /* Force update next time */ 693 /* Force update next time */
706 saved_ftrace_func = NULL; 694 saved_ftrace_func = NULL;
707 /* ftrace_start_up is true if we want ftrace running */ 695 /* ftrace_start_up is true if we want ftrace running */
@@ -709,7 +697,6 @@ static void ftrace_startup_sysctl(void)
709 command |= FTRACE_ENABLE_CALLS; 697 command |= FTRACE_ENABLE_CALLS;
710 698
711 ftrace_run_update_code(command); 699 ftrace_run_update_code(command);
712 mutex_unlock(&ftrace_start_lock);
713} 700}
714 701
715static void ftrace_shutdown_sysctl(void) 702static void ftrace_shutdown_sysctl(void)
@@ -719,13 +706,11 @@ static void ftrace_shutdown_sysctl(void)
719 if (unlikely(ftrace_disabled)) 706 if (unlikely(ftrace_disabled))
720 return; 707 return;
721 708
722 mutex_lock(&ftrace_start_lock);
723 /* ftrace_start_up is true if ftrace is running */ 709 /* ftrace_start_up is true if ftrace is running */
724 if (ftrace_start_up) 710 if (ftrace_start_up)
725 command |= FTRACE_DISABLE_CALLS; 711 command |= FTRACE_DISABLE_CALLS;
726 712
727 ftrace_run_update_code(command); 713 ftrace_run_update_code(command);
728 mutex_unlock(&ftrace_start_lock);
729} 714}
730 715
731static cycle_t ftrace_update_time; 716static cycle_t ftrace_update_time;
@@ -812,13 +797,16 @@ enum {
812 FTRACE_ITER_CONT = (1 << 1), 797 FTRACE_ITER_CONT = (1 << 1),
813 FTRACE_ITER_NOTRACE = (1 << 2), 798 FTRACE_ITER_NOTRACE = (1 << 2),
814 FTRACE_ITER_FAILURES = (1 << 3), 799 FTRACE_ITER_FAILURES = (1 << 3),
800 FTRACE_ITER_PRINTALL = (1 << 4),
801 FTRACE_ITER_HASH = (1 << 5),
815}; 802};
816 803
817#define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */ 804#define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
818 805
819struct ftrace_iterator { 806struct ftrace_iterator {
820 struct ftrace_page *pg; 807 struct ftrace_page *pg;
821 unsigned idx; 808 int hidx;
809 int idx;
822 unsigned flags; 810 unsigned flags;
823 unsigned char buffer[FTRACE_BUFF_MAX+1]; 811 unsigned char buffer[FTRACE_BUFF_MAX+1];
824 unsigned buffer_idx; 812 unsigned buffer_idx;
@@ -826,15 +814,89 @@ struct ftrace_iterator {
826}; 814};
827 815
828static void * 816static void *
817t_hash_next(struct seq_file *m, void *v, loff_t *pos)
818{
819 struct ftrace_iterator *iter = m->private;
820 struct hlist_node *hnd = v;
821 struct hlist_head *hhd;
822
823 WARN_ON(!(iter->flags & FTRACE_ITER_HASH));
824
825 (*pos)++;
826
827 retry:
828 if (iter->hidx >= FTRACE_FUNC_HASHSIZE)
829 return NULL;
830
831 hhd = &ftrace_func_hash[iter->hidx];
832
833 if (hlist_empty(hhd)) {
834 iter->hidx++;
835 hnd = NULL;
836 goto retry;
837 }
838
839 if (!hnd)
840 hnd = hhd->first;
841 else {
842 hnd = hnd->next;
843 if (!hnd) {
844 iter->hidx++;
845 goto retry;
846 }
847 }
848
849 return hnd;
850}
851
852static void *t_hash_start(struct seq_file *m, loff_t *pos)
853{
854 struct ftrace_iterator *iter = m->private;
855 void *p = NULL;
856
857 iter->flags |= FTRACE_ITER_HASH;
858
859 return t_hash_next(m, p, pos);
860}
861
862static int t_hash_show(struct seq_file *m, void *v)
863{
864 struct ftrace_func_probe *rec;
865 struct hlist_node *hnd = v;
866 char str[KSYM_SYMBOL_LEN];
867
868 rec = hlist_entry(hnd, struct ftrace_func_probe, node);
869
870 if (rec->ops->print)
871 return rec->ops->print(m, rec->ip, rec->ops, rec->data);
872
873 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
874 seq_printf(m, "%s:", str);
875
876 kallsyms_lookup((unsigned long)rec->ops->func, NULL, NULL, NULL, str);
877 seq_printf(m, "%s", str);
878
879 if (rec->data)
880 seq_printf(m, ":%p", rec->data);
881 seq_putc(m, '\n');
882
883 return 0;
884}
885
886static void *
829t_next(struct seq_file *m, void *v, loff_t *pos) 887t_next(struct seq_file *m, void *v, loff_t *pos)
830{ 888{
831 struct ftrace_iterator *iter = m->private; 889 struct ftrace_iterator *iter = m->private;
832 struct dyn_ftrace *rec = NULL; 890 struct dyn_ftrace *rec = NULL;
833 891
892 if (iter->flags & FTRACE_ITER_HASH)
893 return t_hash_next(m, v, pos);
894
834 (*pos)++; 895 (*pos)++;
835 896
836 /* should not be called from interrupt context */ 897 if (iter->flags & FTRACE_ITER_PRINTALL)
837 spin_lock(&ftrace_lock); 898 return NULL;
899
838 retry: 900 retry:
839 if (iter->idx >= iter->pg->index) { 901 if (iter->idx >= iter->pg->index) {
840 if (iter->pg->next) { 902 if (iter->pg->next) {
@@ -863,7 +925,6 @@ t_next(struct seq_file *m, void *v, loff_t *pos)
863 goto retry; 925 goto retry;
864 } 926 }
865 } 927 }
866 spin_unlock(&ftrace_lock);
867 928
868 return rec; 929 return rec;
869} 930}
@@ -873,6 +934,23 @@ static void *t_start(struct seq_file *m, loff_t *pos)
873 struct ftrace_iterator *iter = m->private; 934 struct ftrace_iterator *iter = m->private;
874 void *p = NULL; 935 void *p = NULL;
875 936
937 mutex_lock(&ftrace_lock);
938 /*
939 * For set_ftrace_filter reading, if we have the filter
940 * off, we can short cut and just print out that all
941 * functions are enabled.
942 */
943 if (iter->flags & FTRACE_ITER_FILTER && !ftrace_filtered) {
944 if (*pos > 0)
945 return t_hash_start(m, pos);
946 iter->flags |= FTRACE_ITER_PRINTALL;
947 (*pos)++;
948 return iter;
949 }
950
951 if (iter->flags & FTRACE_ITER_HASH)
952 return t_hash_start(m, pos);
953
876 if (*pos > 0) { 954 if (*pos > 0) {
877 if (iter->idx < 0) 955 if (iter->idx < 0)
878 return p; 956 return p;
@@ -882,18 +960,31 @@ static void *t_start(struct seq_file *m, loff_t *pos)
882 960
883 p = t_next(m, p, pos); 961 p = t_next(m, p, pos);
884 962
963 if (!p)
964 return t_hash_start(m, pos);
965
885 return p; 966 return p;
886} 967}
887 968
888static void t_stop(struct seq_file *m, void *p) 969static void t_stop(struct seq_file *m, void *p)
889{ 970{
971 mutex_unlock(&ftrace_lock);
890} 972}
891 973
892static int t_show(struct seq_file *m, void *v) 974static int t_show(struct seq_file *m, void *v)
893{ 975{
976 struct ftrace_iterator *iter = m->private;
894 struct dyn_ftrace *rec = v; 977 struct dyn_ftrace *rec = v;
895 char str[KSYM_SYMBOL_LEN]; 978 char str[KSYM_SYMBOL_LEN];
896 979
980 if (iter->flags & FTRACE_ITER_HASH)
981 return t_hash_show(m, v);
982
983 if (iter->flags & FTRACE_ITER_PRINTALL) {
984 seq_printf(m, "#### all functions enabled ####\n");
985 return 0;
986 }
987
897 if (!rec) 988 if (!rec)
898 return 0; 989 return 0;
899 990
@@ -972,23 +1063,16 @@ static void ftrace_filter_reset(int enable)
972 struct ftrace_page *pg; 1063 struct ftrace_page *pg;
973 struct dyn_ftrace *rec; 1064 struct dyn_ftrace *rec;
974 unsigned long type = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE; 1065 unsigned long type = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
975 unsigned i;
976 1066
977 /* should not be called from interrupt context */ 1067 mutex_lock(&ftrace_lock);
978 spin_lock(&ftrace_lock);
979 if (enable) 1068 if (enable)
980 ftrace_filtered = 0; 1069 ftrace_filtered = 0;
981 pg = ftrace_pages_start; 1070 do_for_each_ftrace_rec(pg, rec) {
982 while (pg) { 1071 if (rec->flags & FTRACE_FL_FAILED)
983 for (i = 0; i < pg->index; i++) { 1072 continue;
984 rec = &pg->records[i]; 1073 rec->flags &= ~type;
985 if (rec->flags & FTRACE_FL_FAILED) 1074 } while_for_each_ftrace_rec();
986 continue; 1075 mutex_unlock(&ftrace_lock);
987 rec->flags &= ~type;
988 }
989 pg = pg->next;
990 }
991 spin_unlock(&ftrace_lock);
992} 1076}
993 1077
994static int 1078static int
@@ -1069,86 +1153,536 @@ enum {
1069 MATCH_END_ONLY, 1153 MATCH_END_ONLY,
1070}; 1154};
1071 1155
1072static void 1156/*
1073ftrace_match(unsigned char *buff, int len, int enable) 1157 * (static function - no need for kernel doc)
1158 *
1159 * Pass in a buffer containing a glob and this function will
1160 * set search to point to the search part of the buffer and
1161 * return the type of search it is (see enum above).
1162 * This does modify buff.
1163 *
1164 * Returns enum type.
1165 * search returns the pointer to use for comparison.
1166 * not returns 1 if buff started with a '!'
1167 * 0 otherwise.
1168 */
1169static int
1170ftrace_setup_glob(char *buff, int len, char **search, int *not)
1074{ 1171{
1075 char str[KSYM_SYMBOL_LEN];
1076 char *search = NULL;
1077 struct ftrace_page *pg;
1078 struct dyn_ftrace *rec;
1079 int type = MATCH_FULL; 1172 int type = MATCH_FULL;
1080 unsigned long flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE; 1173 int i;
1081 unsigned i, match = 0, search_len = 0;
1082 int not = 0;
1083 1174
1084 if (buff[0] == '!') { 1175 if (buff[0] == '!') {
1085 not = 1; 1176 *not = 1;
1086 buff++; 1177 buff++;
1087 len--; 1178 len--;
1088 } 1179 } else
1180 *not = 0;
1181
1182 *search = buff;
1089 1183
1090 for (i = 0; i < len; i++) { 1184 for (i = 0; i < len; i++) {
1091 if (buff[i] == '*') { 1185 if (buff[i] == '*') {
1092 if (!i) { 1186 if (!i) {
1093 search = buff + i + 1; 1187 *search = buff + 1;
1094 type = MATCH_END_ONLY; 1188 type = MATCH_END_ONLY;
1095 search_len = len - (i + 1);
1096 } else { 1189 } else {
1097 if (type == MATCH_END_ONLY) { 1190 if (type == MATCH_END_ONLY)
1098 type = MATCH_MIDDLE_ONLY; 1191 type = MATCH_MIDDLE_ONLY;
1099 } else { 1192 else
1100 match = i;
1101 type = MATCH_FRONT_ONLY; 1193 type = MATCH_FRONT_ONLY;
1102 }
1103 buff[i] = 0; 1194 buff[i] = 0;
1104 break; 1195 break;
1105 } 1196 }
1106 } 1197 }
1107 } 1198 }
1108 1199
1109 /* should not be called from interrupt context */ 1200 return type;
1110 spin_lock(&ftrace_lock); 1201}
1111 if (enable) 1202
1112 ftrace_filtered = 1; 1203static int ftrace_match(char *str, char *regex, int len, int type)
1113 pg = ftrace_pages_start; 1204{
1114 while (pg) { 1205 int matched = 0;
1115 for (i = 0; i < pg->index; i++) { 1206 char *ptr;
1116 int matched = 0; 1207
1117 char *ptr; 1208 switch (type) {
1118 1209 case MATCH_FULL:
1119 rec = &pg->records[i]; 1210 if (strcmp(str, regex) == 0)
1120 if (rec->flags & FTRACE_FL_FAILED) 1211 matched = 1;
1212 break;
1213 case MATCH_FRONT_ONLY:
1214 if (strncmp(str, regex, len) == 0)
1215 matched = 1;
1216 break;
1217 case MATCH_MIDDLE_ONLY:
1218 if (strstr(str, regex))
1219 matched = 1;
1220 break;
1221 case MATCH_END_ONLY:
1222 ptr = strstr(str, regex);
1223 if (ptr && (ptr[len] == 0))
1224 matched = 1;
1225 break;
1226 }
1227
1228 return matched;
1229}
1230
1231static int
1232ftrace_match_record(struct dyn_ftrace *rec, char *regex, int len, int type)
1233{
1234 char str[KSYM_SYMBOL_LEN];
1235
1236 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
1237 return ftrace_match(str, regex, len, type);
1238}
1239
1240static void ftrace_match_records(char *buff, int len, int enable)
1241{
1242 unsigned int search_len;
1243 struct ftrace_page *pg;
1244 struct dyn_ftrace *rec;
1245 unsigned long flag;
1246 char *search;
1247 int type;
1248 int not;
1249
1250 flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1251 type = ftrace_setup_glob(buff, len, &search, &not);
1252
1253 search_len = strlen(search);
1254
1255 mutex_lock(&ftrace_lock);
1256 do_for_each_ftrace_rec(pg, rec) {
1257
1258 if (rec->flags & FTRACE_FL_FAILED)
1259 continue;
1260
1261 if (ftrace_match_record(rec, search, search_len, type)) {
1262 if (not)
1263 rec->flags &= ~flag;
1264 else
1265 rec->flags |= flag;
1266 }
1267 /*
1268 * Only enable filtering if we have a function that
1269 * is filtered on.
1270 */
1271 if (enable && (rec->flags & FTRACE_FL_FILTER))
1272 ftrace_filtered = 1;
1273 } while_for_each_ftrace_rec();
1274 mutex_unlock(&ftrace_lock);
1275}
1276
1277static int
1278ftrace_match_module_record(struct dyn_ftrace *rec, char *mod,
1279 char *regex, int len, int type)
1280{
1281 char str[KSYM_SYMBOL_LEN];
1282 char *modname;
1283
1284 kallsyms_lookup(rec->ip, NULL, NULL, &modname, str);
1285
1286 if (!modname || strcmp(modname, mod))
1287 return 0;
1288
1289 /* blank search means to match all funcs in the mod */
1290 if (len)
1291 return ftrace_match(str, regex, len, type);
1292 else
1293 return 1;
1294}
1295
1296static void ftrace_match_module_records(char *buff, char *mod, int enable)
1297{
1298 unsigned search_len = 0;
1299 struct ftrace_page *pg;
1300 struct dyn_ftrace *rec;
1301 int type = MATCH_FULL;
1302 char *search = buff;
1303 unsigned long flag;
1304 int not = 0;
1305
1306 flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1307
1308 /* blank or '*' mean the same */
1309 if (strcmp(buff, "*") == 0)
1310 buff[0] = 0;
1311
1312 /* handle the case of 'dont filter this module' */
1313 if (strcmp(buff, "!") == 0 || strcmp(buff, "!*") == 0) {
1314 buff[0] = 0;
1315 not = 1;
1316 }
1317
1318 if (strlen(buff)) {
1319 type = ftrace_setup_glob(buff, strlen(buff), &search, &not);
1320 search_len = strlen(search);
1321 }
1322
1323 mutex_lock(&ftrace_lock);
1324 do_for_each_ftrace_rec(pg, rec) {
1325
1326 if (rec->flags & FTRACE_FL_FAILED)
1327 continue;
1328
1329 if (ftrace_match_module_record(rec, mod,
1330 search, search_len, type)) {
1331 if (not)
1332 rec->flags &= ~flag;
1333 else
1334 rec->flags |= flag;
1335 }
1336 if (enable && (rec->flags & FTRACE_FL_FILTER))
1337 ftrace_filtered = 1;
1338
1339 } while_for_each_ftrace_rec();
1340 mutex_unlock(&ftrace_lock);
1341}
1342
1343/*
1344 * We register the module command as a template to show others how
1345 * to register the a command as well.
1346 */
1347
1348static int
1349ftrace_mod_callback(char *func, char *cmd, char *param, int enable)
1350{
1351 char *mod;
1352
1353 /*
1354 * cmd == 'mod' because we only registered this func
1355 * for the 'mod' ftrace_func_command.
1356 * But if you register one func with multiple commands,
1357 * you can tell which command was used by the cmd
1358 * parameter.
1359 */
1360
1361 /* we must have a module name */
1362 if (!param)
1363 return -EINVAL;
1364
1365 mod = strsep(&param, ":");
1366 if (!strlen(mod))
1367 return -EINVAL;
1368
1369 ftrace_match_module_records(func, mod, enable);
1370 return 0;
1371}
1372
1373static struct ftrace_func_command ftrace_mod_cmd = {
1374 .name = "mod",
1375 .func = ftrace_mod_callback,
1376};
1377
1378static int __init ftrace_mod_cmd_init(void)
1379{
1380 return register_ftrace_command(&ftrace_mod_cmd);
1381}
1382device_initcall(ftrace_mod_cmd_init);
1383
1384static void
1385function_trace_probe_call(unsigned long ip, unsigned long parent_ip)
1386{
1387 struct ftrace_func_probe *entry;
1388 struct hlist_head *hhd;
1389 struct hlist_node *n;
1390 unsigned long key;
1391 int resched;
1392
1393 key = hash_long(ip, FTRACE_HASH_BITS);
1394
1395 hhd = &ftrace_func_hash[key];
1396
1397 if (hlist_empty(hhd))
1398 return;
1399
1400 /*
1401 * Disable preemption for these calls to prevent a RCU grace
1402 * period. This syncs the hash iteration and freeing of items
1403 * on the hash. rcu_read_lock is too dangerous here.
1404 */
1405 resched = ftrace_preempt_disable();
1406 hlist_for_each_entry_rcu(entry, n, hhd, node) {
1407 if (entry->ip == ip)
1408 entry->ops->func(ip, parent_ip, &entry->data);
1409 }
1410 ftrace_preempt_enable(resched);
1411}
1412
1413static struct ftrace_ops trace_probe_ops __read_mostly =
1414{
1415 .func = function_trace_probe_call,
1416};
1417
1418static int ftrace_probe_registered;
1419
1420static void __enable_ftrace_function_probe(void)
1421{
1422 int i;
1423
1424 if (ftrace_probe_registered)
1425 return;
1426
1427 for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
1428 struct hlist_head *hhd = &ftrace_func_hash[i];
1429 if (hhd->first)
1430 break;
1431 }
1432 /* Nothing registered? */
1433 if (i == FTRACE_FUNC_HASHSIZE)
1434 return;
1435
1436 __register_ftrace_function(&trace_probe_ops);
1437 ftrace_startup(0);
1438 ftrace_probe_registered = 1;
1439}
1440
1441static void __disable_ftrace_function_probe(void)
1442{
1443 int i;
1444
1445 if (!ftrace_probe_registered)
1446 return;
1447
1448 for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
1449 struct hlist_head *hhd = &ftrace_func_hash[i];
1450 if (hhd->first)
1451 return;
1452 }
1453
1454 /* no more funcs left */
1455 __unregister_ftrace_function(&trace_probe_ops);
1456 ftrace_shutdown(0);
1457 ftrace_probe_registered = 0;
1458}
1459
1460
1461static void ftrace_free_entry_rcu(struct rcu_head *rhp)
1462{
1463 struct ftrace_func_probe *entry =
1464 container_of(rhp, struct ftrace_func_probe, rcu);
1465
1466 if (entry->ops->free)
1467 entry->ops->free(&entry->data);
1468 kfree(entry);
1469}
1470
1471
1472int
1473register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
1474 void *data)
1475{
1476 struct ftrace_func_probe *entry;
1477 struct ftrace_page *pg;
1478 struct dyn_ftrace *rec;
1479 int type, len, not;
1480 unsigned long key;
1481 int count = 0;
1482 char *search;
1483
1484 type = ftrace_setup_glob(glob, strlen(glob), &search, &not);
1485 len = strlen(search);
1486
1487 /* we do not support '!' for function probes */
1488 if (WARN_ON(not))
1489 return -EINVAL;
1490
1491 mutex_lock(&ftrace_lock);
1492 do_for_each_ftrace_rec(pg, rec) {
1493
1494 if (rec->flags & FTRACE_FL_FAILED)
1495 continue;
1496
1497 if (!ftrace_match_record(rec, search, len, type))
1498 continue;
1499
1500 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
1501 if (!entry) {
1502 /* If we did not process any, then return error */
1503 if (!count)
1504 count = -ENOMEM;
1505 goto out_unlock;
1506 }
1507
1508 count++;
1509
1510 entry->data = data;
1511
1512 /*
1513 * The caller might want to do something special
1514 * for each function we find. We call the callback
1515 * to give the caller an opportunity to do so.
1516 */
1517 if (ops->callback) {
1518 if (ops->callback(rec->ip, &entry->data) < 0) {
1519 /* caller does not like this func */
1520 kfree(entry);
1121 continue; 1521 continue;
1122 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
1123 switch (type) {
1124 case MATCH_FULL:
1125 if (strcmp(str, buff) == 0)
1126 matched = 1;
1127 break;
1128 case MATCH_FRONT_ONLY:
1129 if (memcmp(str, buff, match) == 0)
1130 matched = 1;
1131 break;
1132 case MATCH_MIDDLE_ONLY:
1133 if (strstr(str, search))
1134 matched = 1;
1135 break;
1136 case MATCH_END_ONLY:
1137 ptr = strstr(str, search);
1138 if (ptr && (ptr[search_len] == 0))
1139 matched = 1;
1140 break;
1141 } 1522 }
1142 if (matched) { 1523 }
1143 if (not) 1524
1144 rec->flags &= ~flag; 1525 entry->ops = ops;
1145 else 1526 entry->ip = rec->ip;
1146 rec->flags |= flag; 1527
1528 key = hash_long(entry->ip, FTRACE_HASH_BITS);
1529 hlist_add_head_rcu(&entry->node, &ftrace_func_hash[key]);
1530
1531 } while_for_each_ftrace_rec();
1532 __enable_ftrace_function_probe();
1533
1534 out_unlock:
1535 mutex_unlock(&ftrace_lock);
1536
1537 return count;
1538}
1539
1540enum {
1541 PROBE_TEST_FUNC = 1,
1542 PROBE_TEST_DATA = 2
1543};
1544
1545static void
1546__unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
1547 void *data, int flags)
1548{
1549 struct ftrace_func_probe *entry;
1550 struct hlist_node *n, *tmp;
1551 char str[KSYM_SYMBOL_LEN];
1552 int type = MATCH_FULL;
1553 int i, len = 0;
1554 char *search;
1555
1556 if (glob && (strcmp(glob, "*") || !strlen(glob)))
1557 glob = NULL;
1558 else {
1559 int not;
1560
1561 type = ftrace_setup_glob(glob, strlen(glob), &search, &not);
1562 len = strlen(search);
1563
1564 /* we do not support '!' for function probes */
1565 if (WARN_ON(not))
1566 return;
1567 }
1568
1569 mutex_lock(&ftrace_lock);
1570 for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
1571 struct hlist_head *hhd = &ftrace_func_hash[i];
1572
1573 hlist_for_each_entry_safe(entry, n, tmp, hhd, node) {
1574
1575 /* break up if statements for readability */
1576 if ((flags & PROBE_TEST_FUNC) && entry->ops != ops)
1577 continue;
1578
1579 if ((flags & PROBE_TEST_DATA) && entry->data != data)
1580 continue;
1581
1582 /* do this last, since it is the most expensive */
1583 if (glob) {
1584 kallsyms_lookup(entry->ip, NULL, NULL,
1585 NULL, str);
1586 if (!ftrace_match(str, glob, len, type))
1587 continue;
1147 } 1588 }
1589
1590 hlist_del(&entry->node);
1591 call_rcu(&entry->rcu, ftrace_free_entry_rcu);
1592 }
1593 }
1594 __disable_ftrace_function_probe();
1595 mutex_unlock(&ftrace_lock);
1596}
1597
1598void
1599unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
1600 void *data)
1601{
1602 __unregister_ftrace_function_probe(glob, ops, data,
1603 PROBE_TEST_FUNC | PROBE_TEST_DATA);
1604}
1605
1606void
1607unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops)
1608{
1609 __unregister_ftrace_function_probe(glob, ops, NULL, PROBE_TEST_FUNC);
1610}
1611
1612void unregister_ftrace_function_probe_all(char *glob)
1613{
1614 __unregister_ftrace_function_probe(glob, NULL, NULL, 0);
1615}
1616
1617static LIST_HEAD(ftrace_commands);
1618static DEFINE_MUTEX(ftrace_cmd_mutex);
1619
1620int register_ftrace_command(struct ftrace_func_command *cmd)
1621{
1622 struct ftrace_func_command *p;
1623 int ret = 0;
1624
1625 mutex_lock(&ftrace_cmd_mutex);
1626 list_for_each_entry(p, &ftrace_commands, list) {
1627 if (strcmp(cmd->name, p->name) == 0) {
1628 ret = -EBUSY;
1629 goto out_unlock;
1630 }
1631 }
1632 list_add(&cmd->list, &ftrace_commands);
1633 out_unlock:
1634 mutex_unlock(&ftrace_cmd_mutex);
1635
1636 return ret;
1637}
1638
1639int unregister_ftrace_command(struct ftrace_func_command *cmd)
1640{
1641 struct ftrace_func_command *p, *n;
1642 int ret = -ENODEV;
1643
1644 mutex_lock(&ftrace_cmd_mutex);
1645 list_for_each_entry_safe(p, n, &ftrace_commands, list) {
1646 if (strcmp(cmd->name, p->name) == 0) {
1647 ret = 0;
1648 list_del_init(&p->list);
1649 goto out_unlock;
1148 } 1650 }
1149 pg = pg->next;
1150 } 1651 }
1151 spin_unlock(&ftrace_lock); 1652 out_unlock:
1653 mutex_unlock(&ftrace_cmd_mutex);
1654
1655 return ret;
1656}
1657
1658static int ftrace_process_regex(char *buff, int len, int enable)
1659{
1660 char *func, *command, *next = buff;
1661 struct ftrace_func_command *p;
1662 int ret = -EINVAL;
1663
1664 func = strsep(&next, ":");
1665
1666 if (!next) {
1667 ftrace_match_records(func, len, enable);
1668 return 0;
1669 }
1670
1671 /* command found */
1672
1673 command = strsep(&next, ":");
1674
1675 mutex_lock(&ftrace_cmd_mutex);
1676 list_for_each_entry(p, &ftrace_commands, list) {
1677 if (strcmp(p->name, command) == 0) {
1678 ret = p->func(func, command, next, enable);
1679 goto out_unlock;
1680 }
1681 }
1682 out_unlock:
1683 mutex_unlock(&ftrace_cmd_mutex);
1684
1685 return ret;
1152} 1686}
1153 1687
1154static ssize_t 1688static ssize_t
@@ -1218,7 +1752,10 @@ ftrace_regex_write(struct file *file, const char __user *ubuf,
1218 if (isspace(ch)) { 1752 if (isspace(ch)) {
1219 iter->filtered++; 1753 iter->filtered++;
1220 iter->buffer[iter->buffer_idx] = 0; 1754 iter->buffer[iter->buffer_idx] = 0;
1221 ftrace_match(iter->buffer, iter->buffer_idx, enable); 1755 ret = ftrace_process_regex(iter->buffer,
1756 iter->buffer_idx, enable);
1757 if (ret)
1758 goto out;
1222 iter->buffer_idx = 0; 1759 iter->buffer_idx = 0;
1223 } else 1760 } else
1224 iter->flags |= FTRACE_ITER_CONT; 1761 iter->flags |= FTRACE_ITER_CONT;
@@ -1257,7 +1794,7 @@ ftrace_set_regex(unsigned char *buf, int len, int reset, int enable)
1257 if (reset) 1794 if (reset)
1258 ftrace_filter_reset(enable); 1795 ftrace_filter_reset(enable);
1259 if (buf) 1796 if (buf)
1260 ftrace_match(buf, len, enable); 1797 ftrace_match_records(buf, len, enable);
1261 mutex_unlock(&ftrace_regex_lock); 1798 mutex_unlock(&ftrace_regex_lock);
1262} 1799}
1263 1800
@@ -1307,15 +1844,13 @@ ftrace_regex_release(struct inode *inode, struct file *file, int enable)
1307 if (iter->buffer_idx) { 1844 if (iter->buffer_idx) {
1308 iter->filtered++; 1845 iter->filtered++;
1309 iter->buffer[iter->buffer_idx] = 0; 1846 iter->buffer[iter->buffer_idx] = 0;
1310 ftrace_match(iter->buffer, iter->buffer_idx, enable); 1847 ftrace_match_records(iter->buffer, iter->buffer_idx, enable);
1311 } 1848 }
1312 1849
1313 mutex_lock(&ftrace_sysctl_lock); 1850 mutex_lock(&ftrace_lock);
1314 mutex_lock(&ftrace_start_lock);
1315 if (ftrace_start_up && ftrace_enabled) 1851 if (ftrace_start_up && ftrace_enabled)
1316 ftrace_run_update_code(FTRACE_ENABLE_CALLS); 1852 ftrace_run_update_code(FTRACE_ENABLE_CALLS);
1317 mutex_unlock(&ftrace_start_lock); 1853 mutex_unlock(&ftrace_lock);
1318 mutex_unlock(&ftrace_sysctl_lock);
1319 1854
1320 kfree(iter); 1855 kfree(iter);
1321 mutex_unlock(&ftrace_regex_lock); 1856 mutex_unlock(&ftrace_regex_lock);
@@ -1391,6 +1926,10 @@ static void *g_start(struct seq_file *m, loff_t *pos)
1391 1926
1392 mutex_lock(&graph_lock); 1927 mutex_lock(&graph_lock);
1393 1928
1929 /* Nothing, tell g_show to print all functions are enabled */
1930 if (!ftrace_graph_count && !*pos)
1931 return (void *)1;
1932
1394 p = g_next(m, p, pos); 1933 p = g_next(m, p, pos);
1395 1934
1396 return p; 1935 return p;
@@ -1409,6 +1948,11 @@ static int g_show(struct seq_file *m, void *v)
1409 if (!ptr) 1948 if (!ptr)
1410 return 0; 1949 return 0;
1411 1950
1951 if (ptr == (unsigned long *)1) {
1952 seq_printf(m, "#### all functions enabled ####\n");
1953 return 0;
1954 }
1955
1412 kallsyms_lookup(*ptr, NULL, NULL, NULL, str); 1956 kallsyms_lookup(*ptr, NULL, NULL, NULL, str);
1413 1957
1414 seq_printf(m, "%s\n", str); 1958 seq_printf(m, "%s\n", str);
@@ -1462,42 +2006,52 @@ ftrace_graph_read(struct file *file, char __user *ubuf,
1462} 2006}
1463 2007
1464static int 2008static int
1465ftrace_set_func(unsigned long *array, int idx, char *buffer) 2009ftrace_set_func(unsigned long *array, int *idx, char *buffer)
1466{ 2010{
1467 char str[KSYM_SYMBOL_LEN];
1468 struct dyn_ftrace *rec; 2011 struct dyn_ftrace *rec;
1469 struct ftrace_page *pg; 2012 struct ftrace_page *pg;
2013 int search_len;
1470 int found = 0; 2014 int found = 0;
1471 int i, j; 2015 int type, not;
2016 char *search;
2017 bool exists;
2018 int i;
1472 2019
1473 if (ftrace_disabled) 2020 if (ftrace_disabled)
1474 return -ENODEV; 2021 return -ENODEV;
1475 2022
1476 /* should not be called from interrupt context */ 2023 /* decode regex */
1477 spin_lock(&ftrace_lock); 2024 type = ftrace_setup_glob(buffer, strlen(buffer), &search, &not);
2025 if (not)
2026 return -EINVAL;
1478 2027
1479 for (pg = ftrace_pages_start; pg; pg = pg->next) { 2028 search_len = strlen(search);
1480 for (i = 0; i < pg->index; i++) {
1481 rec = &pg->records[i];
1482 2029
1483 if (rec->flags & (FTRACE_FL_FAILED | FTRACE_FL_FREE)) 2030 mutex_lock(&ftrace_lock);
1484 continue; 2031 do_for_each_ftrace_rec(pg, rec) {
2032
2033 if (*idx >= FTRACE_GRAPH_MAX_FUNCS)
2034 break;
1485 2035
1486 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str); 2036 if (rec->flags & (FTRACE_FL_FAILED | FTRACE_FL_FREE))
1487 if (strcmp(str, buffer) == 0) { 2037 continue;
2038
2039 if (ftrace_match_record(rec, search, search_len, type)) {
2040 /* ensure it is not already in the array */
2041 exists = false;
2042 for (i = 0; i < *idx; i++)
2043 if (array[i] == rec->ip) {
2044 exists = true;
2045 break;
2046 }
2047 if (!exists) {
2048 array[(*idx)++] = rec->ip;
1488 found = 1; 2049 found = 1;
1489 for (j = 0; j < idx; j++)
1490 if (array[j] == rec->ip) {
1491 found = 0;
1492 break;
1493 }
1494 if (found)
1495 array[idx] = rec->ip;
1496 break;
1497 } 2050 }
1498 } 2051 }
1499 } 2052 } while_for_each_ftrace_rec();
1500 spin_unlock(&ftrace_lock); 2053
2054 mutex_unlock(&ftrace_lock);
1501 2055
1502 return found ? 0 : -EINVAL; 2056 return found ? 0 : -EINVAL;
1503} 2057}
@@ -1565,13 +2119,11 @@ ftrace_graph_write(struct file *file, const char __user *ubuf,
1565 } 2119 }
1566 buffer[index] = 0; 2120 buffer[index] = 0;
1567 2121
1568 /* we allow only one at a time */ 2122 /* we allow only one expression at a time */
1569 ret = ftrace_set_func(array, ftrace_graph_count, buffer); 2123 ret = ftrace_set_func(array, &ftrace_graph_count, buffer);
1570 if (ret) 2124 if (ret)
1571 goto out; 2125 goto out;
1572 2126
1573 ftrace_graph_count++;
1574
1575 file->f_pos += read; 2127 file->f_pos += read;
1576 2128
1577 ret = read; 2129 ret = read;
@@ -1635,7 +2187,7 @@ static int ftrace_convert_nops(struct module *mod,
1635 unsigned long addr; 2187 unsigned long addr;
1636 unsigned long flags; 2188 unsigned long flags;
1637 2189
1638 mutex_lock(&ftrace_start_lock); 2190 mutex_lock(&ftrace_lock);
1639 p = start; 2191 p = start;
1640 while (p < end) { 2192 while (p < end) {
1641 addr = ftrace_call_adjust(*p++); 2193 addr = ftrace_call_adjust(*p++);
@@ -1654,7 +2206,7 @@ static int ftrace_convert_nops(struct module *mod,
1654 local_irq_save(flags); 2206 local_irq_save(flags);
1655 ftrace_update_code(mod); 2207 ftrace_update_code(mod);
1656 local_irq_restore(flags); 2208 local_irq_restore(flags);
1657 mutex_unlock(&ftrace_start_lock); 2209 mutex_unlock(&ftrace_lock);
1658 2210
1659 return 0; 2211 return 0;
1660} 2212}
@@ -1827,7 +2379,7 @@ ftrace_pid_write(struct file *filp, const char __user *ubuf,
1827 if (ret < 0) 2379 if (ret < 0)
1828 return ret; 2380 return ret;
1829 2381
1830 mutex_lock(&ftrace_start_lock); 2382 mutex_lock(&ftrace_lock);
1831 if (val < 0) { 2383 if (val < 0) {
1832 /* disable pid tracing */ 2384 /* disable pid tracing */
1833 if (!ftrace_pid_trace) 2385 if (!ftrace_pid_trace)
@@ -1866,7 +2418,7 @@ ftrace_pid_write(struct file *filp, const char __user *ubuf,
1866 ftrace_startup_enable(0); 2418 ftrace_startup_enable(0);
1867 2419
1868 out: 2420 out:
1869 mutex_unlock(&ftrace_start_lock); 2421 mutex_unlock(&ftrace_lock);
1870 2422
1871 return cnt; 2423 return cnt;
1872} 2424}
@@ -1894,7 +2446,6 @@ static __init int ftrace_init_debugfs(void)
1894 "'set_ftrace_pid' entry\n"); 2446 "'set_ftrace_pid' entry\n");
1895 return 0; 2447 return 0;
1896} 2448}
1897
1898fs_initcall(ftrace_init_debugfs); 2449fs_initcall(ftrace_init_debugfs);
1899 2450
1900/** 2451/**
@@ -1929,17 +2480,17 @@ int register_ftrace_function(struct ftrace_ops *ops)
1929 if (unlikely(ftrace_disabled)) 2480 if (unlikely(ftrace_disabled))
1930 return -1; 2481 return -1;
1931 2482
1932 mutex_lock(&ftrace_sysctl_lock); 2483 mutex_lock(&ftrace_lock);
1933 2484
1934 ret = __register_ftrace_function(ops); 2485 ret = __register_ftrace_function(ops);
1935 ftrace_startup(0); 2486 ftrace_startup(0);
1936 2487
1937 mutex_unlock(&ftrace_sysctl_lock); 2488 mutex_unlock(&ftrace_lock);
1938 return ret; 2489 return ret;
1939} 2490}
1940 2491
1941/** 2492/**
1942 * unregister_ftrace_function - unresgister a function for profiling. 2493 * unregister_ftrace_function - unregister a function for profiling.
1943 * @ops - ops structure that holds the function to unregister 2494 * @ops - ops structure that holds the function to unregister
1944 * 2495 *
1945 * Unregister a function that was added to be called by ftrace profiling. 2496 * Unregister a function that was added to be called by ftrace profiling.
@@ -1948,10 +2499,10 @@ int unregister_ftrace_function(struct ftrace_ops *ops)
1948{ 2499{
1949 int ret; 2500 int ret;
1950 2501
1951 mutex_lock(&ftrace_sysctl_lock); 2502 mutex_lock(&ftrace_lock);
1952 ret = __unregister_ftrace_function(ops); 2503 ret = __unregister_ftrace_function(ops);
1953 ftrace_shutdown(0); 2504 ftrace_shutdown(0);
1954 mutex_unlock(&ftrace_sysctl_lock); 2505 mutex_unlock(&ftrace_lock);
1955 2506
1956 return ret; 2507 return ret;
1957} 2508}
@@ -1966,7 +2517,7 @@ ftrace_enable_sysctl(struct ctl_table *table, int write,
1966 if (unlikely(ftrace_disabled)) 2517 if (unlikely(ftrace_disabled))
1967 return -ENODEV; 2518 return -ENODEV;
1968 2519
1969 mutex_lock(&ftrace_sysctl_lock); 2520 mutex_lock(&ftrace_lock);
1970 2521
1971 ret = proc_dointvec(table, write, file, buffer, lenp, ppos); 2522 ret = proc_dointvec(table, write, file, buffer, lenp, ppos);
1972 2523
@@ -1995,7 +2546,7 @@ ftrace_enable_sysctl(struct ctl_table *table, int write,
1995 } 2546 }
1996 2547
1997 out: 2548 out:
1998 mutex_unlock(&ftrace_sysctl_lock); 2549 mutex_unlock(&ftrace_lock);
1999 return ret; 2550 return ret;
2000} 2551}
2001 2552
@@ -2111,7 +2662,7 @@ int register_ftrace_graph(trace_func_graph_ret_t retfunc,
2111{ 2662{
2112 int ret = 0; 2663 int ret = 0;
2113 2664
2114 mutex_lock(&ftrace_sysctl_lock); 2665 mutex_lock(&ftrace_lock);
2115 2666
2116 ftrace_suspend_notifier.notifier_call = ftrace_suspend_notifier_call; 2667 ftrace_suspend_notifier.notifier_call = ftrace_suspend_notifier_call;
2117 register_pm_notifier(&ftrace_suspend_notifier); 2668 register_pm_notifier(&ftrace_suspend_notifier);
@@ -2129,13 +2680,13 @@ int register_ftrace_graph(trace_func_graph_ret_t retfunc,
2129 ftrace_startup(FTRACE_START_FUNC_RET); 2680 ftrace_startup(FTRACE_START_FUNC_RET);
2130 2681
2131out: 2682out:
2132 mutex_unlock(&ftrace_sysctl_lock); 2683 mutex_unlock(&ftrace_lock);
2133 return ret; 2684 return ret;
2134} 2685}
2135 2686
2136void unregister_ftrace_graph(void) 2687void unregister_ftrace_graph(void)
2137{ 2688{
2138 mutex_lock(&ftrace_sysctl_lock); 2689 mutex_lock(&ftrace_lock);
2139 2690
2140 atomic_dec(&ftrace_graph_active); 2691 atomic_dec(&ftrace_graph_active);
2141 ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub; 2692 ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
@@ -2143,7 +2694,7 @@ void unregister_ftrace_graph(void)
2143 ftrace_shutdown(FTRACE_STOP_FUNC_RET); 2694 ftrace_shutdown(FTRACE_STOP_FUNC_RET);
2144 unregister_pm_notifier(&ftrace_suspend_notifier); 2695 unregister_pm_notifier(&ftrace_suspend_notifier);
2145 2696
2146 mutex_unlock(&ftrace_sysctl_lock); 2697 mutex_unlock(&ftrace_lock);
2147} 2698}
2148 2699
2149/* Allocate a return stack for newly created task */ 2700/* Allocate a return stack for newly created task */
diff --git a/kernel/trace/kmemtrace.c b/kernel/trace/kmemtrace.c
new file mode 100644
index 000000000000..ae201b3eda89
--- /dev/null
+++ b/kernel/trace/kmemtrace.c
@@ -0,0 +1,339 @@
1/*
2 * Memory allocator tracing
3 *
4 * Copyright (C) 2008 Eduard - Gabriel Munteanu
5 * Copyright (C) 2008 Pekka Enberg <penberg@cs.helsinki.fi>
6 * Copyright (C) 2008 Frederic Weisbecker <fweisbec@gmail.com>
7 */
8
9#include <linux/dcache.h>
10#include <linux/debugfs.h>
11#include <linux/fs.h>
12#include <linux/seq_file.h>
13#include <trace/kmemtrace.h>
14
15#include "trace.h"
16#include "trace_output.h"
17
18/* Select an alternative, minimalistic output than the original one */
19#define TRACE_KMEM_OPT_MINIMAL 0x1
20
21static struct tracer_opt kmem_opts[] = {
22 /* Default disable the minimalistic output */
23 { TRACER_OPT(kmem_minimalistic, TRACE_KMEM_OPT_MINIMAL) },
24 { }
25};
26
27static struct tracer_flags kmem_tracer_flags = {
28 .val = 0,
29 .opts = kmem_opts
30};
31
32
33static bool kmem_tracing_enabled __read_mostly;
34static struct trace_array *kmemtrace_array;
35
36static int kmem_trace_init(struct trace_array *tr)
37{
38 int cpu;
39 kmemtrace_array = tr;
40
41 for_each_cpu_mask(cpu, cpu_possible_map)
42 tracing_reset(tr, cpu);
43
44 kmem_tracing_enabled = true;
45
46 return 0;
47}
48
49static void kmem_trace_reset(struct trace_array *tr)
50{
51 kmem_tracing_enabled = false;
52}
53
54static void kmemtrace_headers(struct seq_file *s)
55{
56 /* Don't need headers for the original kmemtrace output */
57 if (!(kmem_tracer_flags.val & TRACE_KMEM_OPT_MINIMAL))
58 return;
59
60 seq_printf(s, "#\n");
61 seq_printf(s, "# ALLOC TYPE REQ GIVEN FLAGS "
62 " POINTER NODE CALLER\n");
63 seq_printf(s, "# FREE | | | | "
64 " | | | |\n");
65 seq_printf(s, "# |\n\n");
66}
67
68/*
69 * The two following functions give the original output from kmemtrace,
70 * or something close to....perhaps they need some missing things
71 */
72static enum print_line_t
73kmemtrace_print_alloc_original(struct trace_iterator *iter,
74 struct kmemtrace_alloc_entry *entry)
75{
76 struct trace_seq *s = &iter->seq;
77 int ret;
78
79 /* Taken from the old linux/kmemtrace.h */
80 ret = trace_seq_printf(s, "type_id %d call_site %lu ptr %lu "
81 "bytes_req %lu bytes_alloc %lu gfp_flags %lu node %d\n",
82 entry->type_id, entry->call_site, (unsigned long) entry->ptr,
83 (unsigned long) entry->bytes_req, (unsigned long) entry->bytes_alloc,
84 (unsigned long) entry->gfp_flags, entry->node);
85
86 if (!ret)
87 return TRACE_TYPE_PARTIAL_LINE;
88
89 return TRACE_TYPE_HANDLED;
90}
91
92static enum print_line_t
93kmemtrace_print_free_original(struct trace_iterator *iter,
94 struct kmemtrace_free_entry *entry)
95{
96 struct trace_seq *s = &iter->seq;
97 int ret;
98
99 /* Taken from the old linux/kmemtrace.h */
100 ret = trace_seq_printf(s, "type_id %d call_site %lu ptr %lu\n",
101 entry->type_id, entry->call_site, (unsigned long) entry->ptr);
102
103 if (!ret)
104 return TRACE_TYPE_PARTIAL_LINE;
105
106 return TRACE_TYPE_HANDLED;
107}
108
109
110/* The two other following provide a more minimalistic output */
111static enum print_line_t
112kmemtrace_print_alloc_compress(struct trace_iterator *iter,
113 struct kmemtrace_alloc_entry *entry)
114{
115 struct trace_seq *s = &iter->seq;
116 int ret;
117
118 /* Alloc entry */
119 ret = trace_seq_printf(s, " + ");
120 if (!ret)
121 return TRACE_TYPE_PARTIAL_LINE;
122
123 /* Type */
124 switch (entry->type_id) {
125 case KMEMTRACE_TYPE_KMALLOC:
126 ret = trace_seq_printf(s, "K ");
127 break;
128 case KMEMTRACE_TYPE_CACHE:
129 ret = trace_seq_printf(s, "C ");
130 break;
131 case KMEMTRACE_TYPE_PAGES:
132 ret = trace_seq_printf(s, "P ");
133 break;
134 default:
135 ret = trace_seq_printf(s, "? ");
136 }
137
138 if (!ret)
139 return TRACE_TYPE_PARTIAL_LINE;
140
141 /* Requested */
142 ret = trace_seq_printf(s, "%4zu ", entry->bytes_req);
143 if (!ret)
144 return TRACE_TYPE_PARTIAL_LINE;
145
146 /* Allocated */
147 ret = trace_seq_printf(s, "%4zu ", entry->bytes_alloc);
148 if (!ret)
149 return TRACE_TYPE_PARTIAL_LINE;
150
151 /* Flags
152 * TODO: would be better to see the name of the GFP flag names
153 */
154 ret = trace_seq_printf(s, "%08x ", entry->gfp_flags);
155 if (!ret)
156 return TRACE_TYPE_PARTIAL_LINE;
157
158 /* Pointer to allocated */
159 ret = trace_seq_printf(s, "0x%tx ", (ptrdiff_t)entry->ptr);
160 if (!ret)
161 return TRACE_TYPE_PARTIAL_LINE;
162
163 /* Node */
164 ret = trace_seq_printf(s, "%4d ", entry->node);
165 if (!ret)
166 return TRACE_TYPE_PARTIAL_LINE;
167
168 /* Call site */
169 ret = seq_print_ip_sym(s, entry->call_site, 0);
170 if (!ret)
171 return TRACE_TYPE_PARTIAL_LINE;
172
173 if (!trace_seq_printf(s, "\n"))
174 return TRACE_TYPE_PARTIAL_LINE;
175
176 return TRACE_TYPE_HANDLED;
177}
178
179static enum print_line_t
180kmemtrace_print_free_compress(struct trace_iterator *iter,
181 struct kmemtrace_free_entry *entry)
182{
183 struct trace_seq *s = &iter->seq;
184 int ret;
185
186 /* Free entry */
187 ret = trace_seq_printf(s, " - ");
188 if (!ret)
189 return TRACE_TYPE_PARTIAL_LINE;
190
191 /* Type */
192 switch (entry->type_id) {
193 case KMEMTRACE_TYPE_KMALLOC:
194 ret = trace_seq_printf(s, "K ");
195 break;
196 case KMEMTRACE_TYPE_CACHE:
197 ret = trace_seq_printf(s, "C ");
198 break;
199 case KMEMTRACE_TYPE_PAGES:
200 ret = trace_seq_printf(s, "P ");
201 break;
202 default:
203 ret = trace_seq_printf(s, "? ");
204 }
205
206 if (!ret)
207 return TRACE_TYPE_PARTIAL_LINE;
208
209 /* Skip requested/allocated/flags */
210 ret = trace_seq_printf(s, " ");
211 if (!ret)
212 return TRACE_TYPE_PARTIAL_LINE;
213
214 /* Pointer to allocated */
215 ret = trace_seq_printf(s, "0x%tx ", (ptrdiff_t)entry->ptr);
216 if (!ret)
217 return TRACE_TYPE_PARTIAL_LINE;
218
219 /* Skip node */
220 ret = trace_seq_printf(s, " ");
221 if (!ret)
222 return TRACE_TYPE_PARTIAL_LINE;
223
224 /* Call site */
225 ret = seq_print_ip_sym(s, entry->call_site, 0);
226 if (!ret)
227 return TRACE_TYPE_PARTIAL_LINE;
228
229 if (!trace_seq_printf(s, "\n"))
230 return TRACE_TYPE_PARTIAL_LINE;
231
232 return TRACE_TYPE_HANDLED;
233}
234
235static enum print_line_t kmemtrace_print_line(struct trace_iterator *iter)
236{
237 struct trace_entry *entry = iter->ent;
238
239 switch (entry->type) {
240 case TRACE_KMEM_ALLOC: {
241 struct kmemtrace_alloc_entry *field;
242 trace_assign_type(field, entry);
243 if (kmem_tracer_flags.val & TRACE_KMEM_OPT_MINIMAL)
244 return kmemtrace_print_alloc_compress(iter, field);
245 else
246 return kmemtrace_print_alloc_original(iter, field);
247 }
248
249 case TRACE_KMEM_FREE: {
250 struct kmemtrace_free_entry *field;
251 trace_assign_type(field, entry);
252 if (kmem_tracer_flags.val & TRACE_KMEM_OPT_MINIMAL)
253 return kmemtrace_print_free_compress(iter, field);
254 else
255 return kmemtrace_print_free_original(iter, field);
256 }
257
258 default:
259 return TRACE_TYPE_UNHANDLED;
260 }
261}
262
263/* Trace allocations */
264void kmemtrace_mark_alloc_node(enum kmemtrace_type_id type_id,
265 unsigned long call_site,
266 const void *ptr,
267 size_t bytes_req,
268 size_t bytes_alloc,
269 gfp_t gfp_flags,
270 int node)
271{
272 struct ring_buffer_event *event;
273 struct kmemtrace_alloc_entry *entry;
274 struct trace_array *tr = kmemtrace_array;
275
276 if (!kmem_tracing_enabled)
277 return;
278
279 event = trace_buffer_lock_reserve(tr, TRACE_KMEM_ALLOC,
280 sizeof(*entry), 0, 0);
281 if (!event)
282 return;
283 entry = ring_buffer_event_data(event);
284
285 entry->call_site = call_site;
286 entry->ptr = ptr;
287 entry->bytes_req = bytes_req;
288 entry->bytes_alloc = bytes_alloc;
289 entry->gfp_flags = gfp_flags;
290 entry->node = node;
291
292 trace_buffer_unlock_commit(tr, event, 0, 0);
293}
294EXPORT_SYMBOL(kmemtrace_mark_alloc_node);
295
296void kmemtrace_mark_free(enum kmemtrace_type_id type_id,
297 unsigned long call_site,
298 const void *ptr)
299{
300 struct ring_buffer_event *event;
301 struct kmemtrace_free_entry *entry;
302 struct trace_array *tr = kmemtrace_array;
303
304 if (!kmem_tracing_enabled)
305 return;
306
307 event = trace_buffer_lock_reserve(tr, TRACE_KMEM_FREE,
308 sizeof(*entry), 0, 0);
309 if (!event)
310 return;
311 entry = ring_buffer_event_data(event);
312 entry->type_id = type_id;
313 entry->call_site = call_site;
314 entry->ptr = ptr;
315
316 trace_buffer_unlock_commit(tr, event, 0, 0);
317}
318EXPORT_SYMBOL(kmemtrace_mark_free);
319
320static struct tracer kmem_tracer __read_mostly = {
321 .name = "kmemtrace",
322 .init = kmem_trace_init,
323 .reset = kmem_trace_reset,
324 .print_line = kmemtrace_print_line,
325 .print_header = kmemtrace_headers,
326 .flags = &kmem_tracer_flags
327};
328
329void kmemtrace_init(void)
330{
331 /* earliest opportunity to start kmem tracing */
332}
333
334static int __init init_kmem_tracer(void)
335{
336 return register_tracer(&kmem_tracer);
337}
338
339device_initcall(init_kmem_tracer);
diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c
index bd38c5cfd8ad..8f19f1aa42b0 100644
--- a/kernel/trace/ring_buffer.c
+++ b/kernel/trace/ring_buffer.c
@@ -4,9 +4,11 @@
4 * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com> 4 * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com>
5 */ 5 */
6#include <linux/ring_buffer.h> 6#include <linux/ring_buffer.h>
7#include <linux/ftrace_irq.h>
7#include <linux/spinlock.h> 8#include <linux/spinlock.h>
8#include <linux/debugfs.h> 9#include <linux/debugfs.h>
9#include <linux/uaccess.h> 10#include <linux/uaccess.h>
11#include <linux/hardirq.h>
10#include <linux/module.h> 12#include <linux/module.h>
11#include <linux/percpu.h> 13#include <linux/percpu.h>
12#include <linux/mutex.h> 14#include <linux/mutex.h>
@@ -57,7 +59,7 @@ enum {
57 RB_BUFFERS_DISABLED = 1 << RB_BUFFERS_DISABLED_BIT, 59 RB_BUFFERS_DISABLED = 1 << RB_BUFFERS_DISABLED_BIT,
58}; 60};
59 61
60static long ring_buffer_flags __read_mostly = RB_BUFFERS_ON; 62static unsigned long ring_buffer_flags __read_mostly = RB_BUFFERS_ON;
61 63
62/** 64/**
63 * tracing_on - enable all tracing buffers 65 * tracing_on - enable all tracing buffers
@@ -89,13 +91,22 @@ EXPORT_SYMBOL_GPL(tracing_off);
89 * tracing_off_permanent - permanently disable ring buffers 91 * tracing_off_permanent - permanently disable ring buffers
90 * 92 *
91 * This function, once called, will disable all ring buffers 93 * This function, once called, will disable all ring buffers
92 * permanenty. 94 * permanently.
93 */ 95 */
94void tracing_off_permanent(void) 96void tracing_off_permanent(void)
95{ 97{
96 set_bit(RB_BUFFERS_DISABLED_BIT, &ring_buffer_flags); 98 set_bit(RB_BUFFERS_DISABLED_BIT, &ring_buffer_flags);
97} 99}
98 100
101/**
102 * tracing_is_on - show state of ring buffers enabled
103 */
104int tracing_is_on(void)
105{
106 return ring_buffer_flags == RB_BUFFERS_ON;
107}
108EXPORT_SYMBOL_GPL(tracing_is_on);
109
99#include "trace.h" 110#include "trace.h"
100 111
101/* Up this if you want to test the TIME_EXTENTS and normalization */ 112/* Up this if you want to test the TIME_EXTENTS and normalization */
@@ -123,8 +134,7 @@ void ring_buffer_normalize_time_stamp(int cpu, u64 *ts)
123EXPORT_SYMBOL_GPL(ring_buffer_normalize_time_stamp); 134EXPORT_SYMBOL_GPL(ring_buffer_normalize_time_stamp);
124 135
125#define RB_EVNT_HDR_SIZE (sizeof(struct ring_buffer_event)) 136#define RB_EVNT_HDR_SIZE (sizeof(struct ring_buffer_event))
126#define RB_ALIGNMENT_SHIFT 2 137#define RB_ALIGNMENT 4U
127#define RB_ALIGNMENT (1 << RB_ALIGNMENT_SHIFT)
128#define RB_MAX_SMALL_DATA 28 138#define RB_MAX_SMALL_DATA 28
129 139
130enum { 140enum {
@@ -133,7 +143,7 @@ enum {
133}; 143};
134 144
135/* inline for ring buffer fast paths */ 145/* inline for ring buffer fast paths */
136static inline unsigned 146static unsigned
137rb_event_length(struct ring_buffer_event *event) 147rb_event_length(struct ring_buffer_event *event)
138{ 148{
139 unsigned length; 149 unsigned length;
@@ -151,7 +161,7 @@ rb_event_length(struct ring_buffer_event *event)
151 161
152 case RINGBUF_TYPE_DATA: 162 case RINGBUF_TYPE_DATA:
153 if (event->len) 163 if (event->len)
154 length = event->len << RB_ALIGNMENT_SHIFT; 164 length = event->len * RB_ALIGNMENT;
155 else 165 else
156 length = event->array[0]; 166 length = event->array[0];
157 return length + RB_EVNT_HDR_SIZE; 167 return length + RB_EVNT_HDR_SIZE;
@@ -179,7 +189,7 @@ unsigned ring_buffer_event_length(struct ring_buffer_event *event)
179EXPORT_SYMBOL_GPL(ring_buffer_event_length); 189EXPORT_SYMBOL_GPL(ring_buffer_event_length);
180 190
181/* inline for ring buffer fast paths */ 191/* inline for ring buffer fast paths */
182static inline void * 192static void *
183rb_event_data(struct ring_buffer_event *event) 193rb_event_data(struct ring_buffer_event *event)
184{ 194{
185 BUG_ON(event->type != RINGBUF_TYPE_DATA); 195 BUG_ON(event->type != RINGBUF_TYPE_DATA);
@@ -209,7 +219,7 @@ EXPORT_SYMBOL_GPL(ring_buffer_event_data);
209 219
210struct buffer_data_page { 220struct buffer_data_page {
211 u64 time_stamp; /* page time stamp */ 221 u64 time_stamp; /* page time stamp */
212 local_t commit; /* write commited index */ 222 local_t commit; /* write committed index */
213 unsigned char data[]; /* data of buffer page */ 223 unsigned char data[]; /* data of buffer page */
214}; 224};
215 225
@@ -229,10 +239,9 @@ static void rb_init_page(struct buffer_data_page *bpage)
229 * Also stolen from mm/slob.c. Thanks to Mathieu Desnoyers for pointing 239 * Also stolen from mm/slob.c. Thanks to Mathieu Desnoyers for pointing
230 * this issue out. 240 * this issue out.
231 */ 241 */
232static inline void free_buffer_page(struct buffer_page *bpage) 242static void free_buffer_page(struct buffer_page *bpage)
233{ 243{
234 if (bpage->page) 244 free_page((unsigned long)bpage->page);
235 free_page((unsigned long)bpage->page);
236 kfree(bpage); 245 kfree(bpage);
237} 246}
238 247
@@ -260,7 +269,7 @@ struct ring_buffer_per_cpu {
260 struct list_head pages; 269 struct list_head pages;
261 struct buffer_page *head_page; /* read from head */ 270 struct buffer_page *head_page; /* read from head */
262 struct buffer_page *tail_page; /* write to tail */ 271 struct buffer_page *tail_page; /* write to tail */
263 struct buffer_page *commit_page; /* commited pages */ 272 struct buffer_page *commit_page; /* committed pages */
264 struct buffer_page *reader_page; 273 struct buffer_page *reader_page;
265 unsigned long overrun; 274 unsigned long overrun;
266 unsigned long entries; 275 unsigned long entries;
@@ -273,8 +282,8 @@ struct ring_buffer {
273 unsigned pages; 282 unsigned pages;
274 unsigned flags; 283 unsigned flags;
275 int cpus; 284 int cpus;
276 cpumask_var_t cpumask;
277 atomic_t record_disabled; 285 atomic_t record_disabled;
286 cpumask_var_t cpumask;
278 287
279 struct mutex mutex; 288 struct mutex mutex;
280 289
@@ -303,7 +312,7 @@ struct ring_buffer_iter {
303 * check_pages - integrity check of buffer pages 312 * check_pages - integrity check of buffer pages
304 * @cpu_buffer: CPU buffer with pages to test 313 * @cpu_buffer: CPU buffer with pages to test
305 * 314 *
306 * As a safty measure we check to make sure the data pages have not 315 * As a safety measure we check to make sure the data pages have not
307 * been corrupted. 316 * been corrupted.
308 */ 317 */
309static int rb_check_pages(struct ring_buffer_per_cpu *cpu_buffer) 318static int rb_check_pages(struct ring_buffer_per_cpu *cpu_buffer)
@@ -811,7 +820,7 @@ rb_event_index(struct ring_buffer_event *event)
811 return (addr & ~PAGE_MASK) - (PAGE_SIZE - BUF_PAGE_SIZE); 820 return (addr & ~PAGE_MASK) - (PAGE_SIZE - BUF_PAGE_SIZE);
812} 821}
813 822
814static inline int 823static int
815rb_is_commit(struct ring_buffer_per_cpu *cpu_buffer, 824rb_is_commit(struct ring_buffer_per_cpu *cpu_buffer,
816 struct ring_buffer_event *event) 825 struct ring_buffer_event *event)
817{ 826{
@@ -825,7 +834,7 @@ rb_is_commit(struct ring_buffer_per_cpu *cpu_buffer,
825 rb_commit_index(cpu_buffer) == index; 834 rb_commit_index(cpu_buffer) == index;
826} 835}
827 836
828static inline void 837static void
829rb_set_commit_event(struct ring_buffer_per_cpu *cpu_buffer, 838rb_set_commit_event(struct ring_buffer_per_cpu *cpu_buffer,
830 struct ring_buffer_event *event) 839 struct ring_buffer_event *event)
831{ 840{
@@ -850,7 +859,7 @@ rb_set_commit_event(struct ring_buffer_per_cpu *cpu_buffer,
850 local_set(&cpu_buffer->commit_page->page->commit, index); 859 local_set(&cpu_buffer->commit_page->page->commit, index);
851} 860}
852 861
853static inline void 862static void
854rb_set_commit_to_write(struct ring_buffer_per_cpu *cpu_buffer) 863rb_set_commit_to_write(struct ring_buffer_per_cpu *cpu_buffer)
855{ 864{
856 /* 865 /*
@@ -896,7 +905,7 @@ static void rb_reset_reader_page(struct ring_buffer_per_cpu *cpu_buffer)
896 cpu_buffer->reader_page->read = 0; 905 cpu_buffer->reader_page->read = 0;
897} 906}
898 907
899static inline void rb_inc_iter(struct ring_buffer_iter *iter) 908static void rb_inc_iter(struct ring_buffer_iter *iter)
900{ 909{
901 struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer; 910 struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
902 911
@@ -926,7 +935,7 @@ static inline void rb_inc_iter(struct ring_buffer_iter *iter)
926 * and with this, we can determine what to place into the 935 * and with this, we can determine what to place into the
927 * data field. 936 * data field.
928 */ 937 */
929static inline void 938static void
930rb_update_event(struct ring_buffer_event *event, 939rb_update_event(struct ring_buffer_event *event,
931 unsigned type, unsigned length) 940 unsigned type, unsigned length)
932{ 941{
@@ -938,15 +947,11 @@ rb_update_event(struct ring_buffer_event *event,
938 break; 947 break;
939 948
940 case RINGBUF_TYPE_TIME_EXTEND: 949 case RINGBUF_TYPE_TIME_EXTEND:
941 event->len = 950 event->len = DIV_ROUND_UP(RB_LEN_TIME_EXTEND, RB_ALIGNMENT);
942 (RB_LEN_TIME_EXTEND + (RB_ALIGNMENT-1))
943 >> RB_ALIGNMENT_SHIFT;
944 break; 951 break;
945 952
946 case RINGBUF_TYPE_TIME_STAMP: 953 case RINGBUF_TYPE_TIME_STAMP:
947 event->len = 954 event->len = DIV_ROUND_UP(RB_LEN_TIME_STAMP, RB_ALIGNMENT);
948 (RB_LEN_TIME_STAMP + (RB_ALIGNMENT-1))
949 >> RB_ALIGNMENT_SHIFT;
950 break; 955 break;
951 956
952 case RINGBUF_TYPE_DATA: 957 case RINGBUF_TYPE_DATA:
@@ -955,16 +960,14 @@ rb_update_event(struct ring_buffer_event *event,
955 event->len = 0; 960 event->len = 0;
956 event->array[0] = length; 961 event->array[0] = length;
957 } else 962 } else
958 event->len = 963 event->len = DIV_ROUND_UP(length, RB_ALIGNMENT);
959 (length + (RB_ALIGNMENT-1))
960 >> RB_ALIGNMENT_SHIFT;
961 break; 964 break;
962 default: 965 default:
963 BUG(); 966 BUG();
964 } 967 }
965} 968}
966 969
967static inline unsigned rb_calculate_event_length(unsigned length) 970static unsigned rb_calculate_event_length(unsigned length)
968{ 971{
969 struct ring_buffer_event event; /* Used only for sizeof array */ 972 struct ring_buffer_event event; /* Used only for sizeof array */
970 973
@@ -990,6 +993,7 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer,
990 struct ring_buffer *buffer = cpu_buffer->buffer; 993 struct ring_buffer *buffer = cpu_buffer->buffer;
991 struct ring_buffer_event *event; 994 struct ring_buffer_event *event;
992 unsigned long flags; 995 unsigned long flags;
996 bool lock_taken = false;
993 997
994 commit_page = cpu_buffer->commit_page; 998 commit_page = cpu_buffer->commit_page;
995 /* we just need to protect against interrupts */ 999 /* we just need to protect against interrupts */
@@ -1003,7 +1007,30 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer,
1003 struct buffer_page *next_page = tail_page; 1007 struct buffer_page *next_page = tail_page;
1004 1008
1005 local_irq_save(flags); 1009 local_irq_save(flags);
1006 __raw_spin_lock(&cpu_buffer->lock); 1010 /*
1011 * Since the write to the buffer is still not
1012 * fully lockless, we must be careful with NMIs.
1013 * The locks in the writers are taken when a write
1014 * crosses to a new page. The locks protect against
1015 * races with the readers (this will soon be fixed
1016 * with a lockless solution).
1017 *
1018 * Because we can not protect against NMIs, and we
1019 * want to keep traces reentrant, we need to manage
1020 * what happens when we are in an NMI.
1021 *
1022 * NMIs can happen after we take the lock.
1023 * If we are in an NMI, only take the lock
1024 * if it is not already taken. Otherwise
1025 * simply fail.
1026 */
1027 if (unlikely(in_nmi())) {
1028 if (!__raw_spin_trylock(&cpu_buffer->lock))
1029 goto out_reset;
1030 } else
1031 __raw_spin_lock(&cpu_buffer->lock);
1032
1033 lock_taken = true;
1007 1034
1008 rb_inc_page(cpu_buffer, &next_page); 1035 rb_inc_page(cpu_buffer, &next_page);
1009 1036
@@ -1012,7 +1039,7 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer,
1012 1039
1013 /* we grabbed the lock before incrementing */ 1040 /* we grabbed the lock before incrementing */
1014 if (RB_WARN_ON(cpu_buffer, next_page == reader_page)) 1041 if (RB_WARN_ON(cpu_buffer, next_page == reader_page))
1015 goto out_unlock; 1042 goto out_reset;
1016 1043
1017 /* 1044 /*
1018 * If for some reason, we had an interrupt storm that made 1045 * If for some reason, we had an interrupt storm that made
@@ -1021,12 +1048,12 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer,
1021 */ 1048 */
1022 if (unlikely(next_page == commit_page)) { 1049 if (unlikely(next_page == commit_page)) {
1023 WARN_ON_ONCE(1); 1050 WARN_ON_ONCE(1);
1024 goto out_unlock; 1051 goto out_reset;
1025 } 1052 }
1026 1053
1027 if (next_page == head_page) { 1054 if (next_page == head_page) {
1028 if (!(buffer->flags & RB_FL_OVERWRITE)) 1055 if (!(buffer->flags & RB_FL_OVERWRITE))
1029 goto out_unlock; 1056 goto out_reset;
1030 1057
1031 /* tail_page has not moved yet? */ 1058 /* tail_page has not moved yet? */
1032 if (tail_page == cpu_buffer->tail_page) { 1059 if (tail_page == cpu_buffer->tail_page) {
@@ -1100,12 +1127,13 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer,
1100 1127
1101 return event; 1128 return event;
1102 1129
1103 out_unlock: 1130 out_reset:
1104 /* reset write */ 1131 /* reset write */
1105 if (tail <= BUF_PAGE_SIZE) 1132 if (tail <= BUF_PAGE_SIZE)
1106 local_set(&tail_page->write, tail); 1133 local_set(&tail_page->write, tail);
1107 1134
1108 __raw_spin_unlock(&cpu_buffer->lock); 1135 if (likely(lock_taken))
1136 __raw_spin_unlock(&cpu_buffer->lock);
1109 local_irq_restore(flags); 1137 local_irq_restore(flags);
1110 return NULL; 1138 return NULL;
1111} 1139}
@@ -1265,7 +1293,6 @@ static DEFINE_PER_CPU(int, rb_need_resched);
1265 * ring_buffer_lock_reserve - reserve a part of the buffer 1293 * ring_buffer_lock_reserve - reserve a part of the buffer
1266 * @buffer: the ring buffer to reserve from 1294 * @buffer: the ring buffer to reserve from
1267 * @length: the length of the data to reserve (excluding event header) 1295 * @length: the length of the data to reserve (excluding event header)
1268 * @flags: a pointer to save the interrupt flags
1269 * 1296 *
1270 * Returns a reseverd event on the ring buffer to copy directly to. 1297 * Returns a reseverd event on the ring buffer to copy directly to.
1271 * The user of this interface will need to get the body to write into 1298 * The user of this interface will need to get the body to write into
@@ -1278,9 +1305,7 @@ static DEFINE_PER_CPU(int, rb_need_resched);
1278 * If NULL is returned, then nothing has been allocated or locked. 1305 * If NULL is returned, then nothing has been allocated or locked.
1279 */ 1306 */
1280struct ring_buffer_event * 1307struct ring_buffer_event *
1281ring_buffer_lock_reserve(struct ring_buffer *buffer, 1308ring_buffer_lock_reserve(struct ring_buffer *buffer, unsigned long length)
1282 unsigned long length,
1283 unsigned long *flags)
1284{ 1309{
1285 struct ring_buffer_per_cpu *cpu_buffer; 1310 struct ring_buffer_per_cpu *cpu_buffer;
1286 struct ring_buffer_event *event; 1311 struct ring_buffer_event *event;
@@ -1347,15 +1372,13 @@ static void rb_commit(struct ring_buffer_per_cpu *cpu_buffer,
1347 * ring_buffer_unlock_commit - commit a reserved 1372 * ring_buffer_unlock_commit - commit a reserved
1348 * @buffer: The buffer to commit to 1373 * @buffer: The buffer to commit to
1349 * @event: The event pointer to commit. 1374 * @event: The event pointer to commit.
1350 * @flags: the interrupt flags received from ring_buffer_lock_reserve.
1351 * 1375 *
1352 * This commits the data to the ring buffer, and releases any locks held. 1376 * This commits the data to the ring buffer, and releases any locks held.
1353 * 1377 *
1354 * Must be paired with ring_buffer_lock_reserve. 1378 * Must be paired with ring_buffer_lock_reserve.
1355 */ 1379 */
1356int ring_buffer_unlock_commit(struct ring_buffer *buffer, 1380int ring_buffer_unlock_commit(struct ring_buffer *buffer,
1357 struct ring_buffer_event *event, 1381 struct ring_buffer_event *event)
1358 unsigned long flags)
1359{ 1382{
1360 struct ring_buffer_per_cpu *cpu_buffer; 1383 struct ring_buffer_per_cpu *cpu_buffer;
1361 int cpu = raw_smp_processor_id(); 1384 int cpu = raw_smp_processor_id();
@@ -1438,7 +1461,7 @@ int ring_buffer_write(struct ring_buffer *buffer,
1438} 1461}
1439EXPORT_SYMBOL_GPL(ring_buffer_write); 1462EXPORT_SYMBOL_GPL(ring_buffer_write);
1440 1463
1441static inline int rb_per_cpu_empty(struct ring_buffer_per_cpu *cpu_buffer) 1464static int rb_per_cpu_empty(struct ring_buffer_per_cpu *cpu_buffer)
1442{ 1465{
1443 struct buffer_page *reader = cpu_buffer->reader_page; 1466 struct buffer_page *reader = cpu_buffer->reader_page;
1444 struct buffer_page *head = cpu_buffer->head_page; 1467 struct buffer_page *head = cpu_buffer->head_page;
@@ -2277,9 +2300,24 @@ int ring_buffer_swap_cpu(struct ring_buffer *buffer_a,
2277 if (buffer_a->pages != buffer_b->pages) 2300 if (buffer_a->pages != buffer_b->pages)
2278 return -EINVAL; 2301 return -EINVAL;
2279 2302
2303 if (ring_buffer_flags != RB_BUFFERS_ON)
2304 return -EAGAIN;
2305
2306 if (atomic_read(&buffer_a->record_disabled))
2307 return -EAGAIN;
2308
2309 if (atomic_read(&buffer_b->record_disabled))
2310 return -EAGAIN;
2311
2280 cpu_buffer_a = buffer_a->buffers[cpu]; 2312 cpu_buffer_a = buffer_a->buffers[cpu];
2281 cpu_buffer_b = buffer_b->buffers[cpu]; 2313 cpu_buffer_b = buffer_b->buffers[cpu];
2282 2314
2315 if (atomic_read(&cpu_buffer_a->record_disabled))
2316 return -EAGAIN;
2317
2318 if (atomic_read(&cpu_buffer_b->record_disabled))
2319 return -EAGAIN;
2320
2283 /* 2321 /*
2284 * We can't do a synchronize_sched here because this 2322 * We can't do a synchronize_sched here because this
2285 * function can be called in atomic context. 2323 * function can be called in atomic context.
@@ -2303,13 +2341,14 @@ int ring_buffer_swap_cpu(struct ring_buffer *buffer_a,
2303EXPORT_SYMBOL_GPL(ring_buffer_swap_cpu); 2341EXPORT_SYMBOL_GPL(ring_buffer_swap_cpu);
2304 2342
2305static void rb_remove_entries(struct ring_buffer_per_cpu *cpu_buffer, 2343static void rb_remove_entries(struct ring_buffer_per_cpu *cpu_buffer,
2306 struct buffer_data_page *bpage) 2344 struct buffer_data_page *bpage,
2345 unsigned int offset)
2307{ 2346{
2308 struct ring_buffer_event *event; 2347 struct ring_buffer_event *event;
2309 unsigned long head; 2348 unsigned long head;
2310 2349
2311 __raw_spin_lock(&cpu_buffer->lock); 2350 __raw_spin_lock(&cpu_buffer->lock);
2312 for (head = 0; head < local_read(&bpage->commit); 2351 for (head = offset; head < local_read(&bpage->commit);
2313 head += rb_event_length(event)) { 2352 head += rb_event_length(event)) {
2314 2353
2315 event = __rb_data_page_index(bpage, head); 2354 event = __rb_data_page_index(bpage, head);
@@ -2377,12 +2416,12 @@ void ring_buffer_free_read_page(struct ring_buffer *buffer, void *data)
2377 * to swap with a page in the ring buffer. 2416 * to swap with a page in the ring buffer.
2378 * 2417 *
2379 * for example: 2418 * for example:
2380 * rpage = ring_buffer_alloc_page(buffer); 2419 * rpage = ring_buffer_alloc_read_page(buffer);
2381 * if (!rpage) 2420 * if (!rpage)
2382 * return error; 2421 * return error;
2383 * ret = ring_buffer_read_page(buffer, &rpage, cpu, 0); 2422 * ret = ring_buffer_read_page(buffer, &rpage, cpu, 0);
2384 * if (ret) 2423 * if (ret >= 0)
2385 * process_page(rpage); 2424 * process_page(rpage, ret);
2386 * 2425 *
2387 * When @full is set, the function will not return true unless 2426 * When @full is set, the function will not return true unless
2388 * the writer is off the reader page. 2427 * the writer is off the reader page.
@@ -2393,8 +2432,8 @@ void ring_buffer_free_read_page(struct ring_buffer *buffer, void *data)
2393 * responsible for that. 2432 * responsible for that.
2394 * 2433 *
2395 * Returns: 2434 * Returns:
2396 * 1 if data has been transferred 2435 * >=0 if data has been transferred, returns the offset of consumed data.
2397 * 0 if no data has been transferred. 2436 * <0 if no data has been transferred.
2398 */ 2437 */
2399int ring_buffer_read_page(struct ring_buffer *buffer, 2438int ring_buffer_read_page(struct ring_buffer *buffer,
2400 void **data_page, int cpu, int full) 2439 void **data_page, int cpu, int full)
@@ -2403,7 +2442,8 @@ int ring_buffer_read_page(struct ring_buffer *buffer,
2403 struct ring_buffer_event *event; 2442 struct ring_buffer_event *event;
2404 struct buffer_data_page *bpage; 2443 struct buffer_data_page *bpage;
2405 unsigned long flags; 2444 unsigned long flags;
2406 int ret = 0; 2445 unsigned int read;
2446 int ret = -1;
2407 2447
2408 if (!data_page) 2448 if (!data_page)
2409 return 0; 2449 return 0;
@@ -2425,25 +2465,29 @@ int ring_buffer_read_page(struct ring_buffer *buffer,
2425 /* check for data */ 2465 /* check for data */
2426 if (!local_read(&cpu_buffer->reader_page->page->commit)) 2466 if (!local_read(&cpu_buffer->reader_page->page->commit))
2427 goto out; 2467 goto out;
2468
2469 read = cpu_buffer->reader_page->read;
2428 /* 2470 /*
2429 * If the writer is already off of the read page, then simply 2471 * If the writer is already off of the read page, then simply
2430 * switch the read page with the given page. Otherwise 2472 * switch the read page with the given page. Otherwise
2431 * we need to copy the data from the reader to the writer. 2473 * we need to copy the data from the reader to the writer.
2432 */ 2474 */
2433 if (cpu_buffer->reader_page == cpu_buffer->commit_page) { 2475 if (cpu_buffer->reader_page == cpu_buffer->commit_page) {
2434 unsigned int read = cpu_buffer->reader_page->read; 2476 unsigned int commit = rb_page_commit(cpu_buffer->reader_page);
2477 struct buffer_data_page *rpage = cpu_buffer->reader_page->page;
2435 2478
2436 if (full) 2479 if (full)
2437 goto out; 2480 goto out;
2438 /* The writer is still on the reader page, we must copy */ 2481 /* The writer is still on the reader page, we must copy */
2439 bpage = cpu_buffer->reader_page->page; 2482 memcpy(bpage->data + read, rpage->data + read, commit - read);
2440 memcpy(bpage->data,
2441 cpu_buffer->reader_page->page->data + read,
2442 local_read(&bpage->commit) - read);
2443 2483
2444 /* consume what was read */ 2484 /* consume what was read */
2445 cpu_buffer->reader_page += read; 2485 cpu_buffer->reader_page->read = commit;
2446 2486
2487 /* update bpage */
2488 local_set(&bpage->commit, commit);
2489 if (!read)
2490 bpage->time_stamp = rpage->time_stamp;
2447 } else { 2491 } else {
2448 /* swap the pages */ 2492 /* swap the pages */
2449 rb_init_page(bpage); 2493 rb_init_page(bpage);
@@ -2452,10 +2496,10 @@ int ring_buffer_read_page(struct ring_buffer *buffer,
2452 cpu_buffer->reader_page->read = 0; 2496 cpu_buffer->reader_page->read = 0;
2453 *data_page = bpage; 2497 *data_page = bpage;
2454 } 2498 }
2455 ret = 1; 2499 ret = read;
2456 2500
2457 /* update the entry counter */ 2501 /* update the entry counter */
2458 rb_remove_entries(cpu_buffer, bpage); 2502 rb_remove_entries(cpu_buffer, bpage, read);
2459 out: 2503 out:
2460 spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); 2504 spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
2461 2505
@@ -2466,7 +2510,7 @@ static ssize_t
2466rb_simple_read(struct file *filp, char __user *ubuf, 2510rb_simple_read(struct file *filp, char __user *ubuf,
2467 size_t cnt, loff_t *ppos) 2511 size_t cnt, loff_t *ppos)
2468{ 2512{
2469 long *p = filp->private_data; 2513 unsigned long *p = filp->private_data;
2470 char buf[64]; 2514 char buf[64];
2471 int r; 2515 int r;
2472 2516
@@ -2482,9 +2526,9 @@ static ssize_t
2482rb_simple_write(struct file *filp, const char __user *ubuf, 2526rb_simple_write(struct file *filp, const char __user *ubuf,
2483 size_t cnt, loff_t *ppos) 2527 size_t cnt, loff_t *ppos)
2484{ 2528{
2485 long *p = filp->private_data; 2529 unsigned long *p = filp->private_data;
2486 char buf[64]; 2530 char buf[64];
2487 long val; 2531 unsigned long val;
2488 int ret; 2532 int ret;
2489 2533
2490 if (cnt >= sizeof(buf)) 2534 if (cnt >= sizeof(buf))
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index 17bb88d86ac2..e1f3b99a2e52 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -31,12 +31,14 @@
31#include <linux/fs.h> 31#include <linux/fs.h>
32#include <linux/kprobes.h> 32#include <linux/kprobes.h>
33#include <linux/writeback.h> 33#include <linux/writeback.h>
34#include <linux/splice.h>
34 35
35#include <linux/stacktrace.h> 36#include <linux/stacktrace.h>
36#include <linux/ring_buffer.h> 37#include <linux/ring_buffer.h>
37#include <linux/irqflags.h> 38#include <linux/irqflags.h>
38 39
39#include "trace.h" 40#include "trace.h"
41#include "trace_output.h"
40 42
41#define TRACE_BUFFER_FLAGS (RB_FL_OVERWRITE) 43#define TRACE_BUFFER_FLAGS (RB_FL_OVERWRITE)
42 44
@@ -52,6 +54,11 @@ unsigned long __read_mostly tracing_thresh;
52 */ 54 */
53static bool __read_mostly tracing_selftest_running; 55static bool __read_mostly tracing_selftest_running;
54 56
57/*
58 * If a tracer is running, we do not want to run SELFTEST.
59 */
60static bool __read_mostly tracing_selftest_disabled;
61
55/* For tracers that don't implement custom flags */ 62/* For tracers that don't implement custom flags */
56static struct tracer_opt dummy_tracer_opt[] = { 63static struct tracer_opt dummy_tracer_opt[] = {
57 { } 64 { }
@@ -73,7 +80,7 @@ static int dummy_set_flag(u32 old_flags, u32 bit, int set)
73 * of the tracer is successful. But that is the only place that sets 80 * of the tracer is successful. But that is the only place that sets
74 * this back to zero. 81 * this back to zero.
75 */ 82 */
76int tracing_disabled = 1; 83static int tracing_disabled = 1;
77 84
78static DEFINE_PER_CPU(local_t, ftrace_cpu_disabled); 85static DEFINE_PER_CPU(local_t, ftrace_cpu_disabled);
79 86
@@ -109,14 +116,19 @@ static cpumask_var_t __read_mostly tracing_buffer_mask;
109 */ 116 */
110int ftrace_dump_on_oops; 117int ftrace_dump_on_oops;
111 118
112static int tracing_set_tracer(char *buf); 119static int tracing_set_tracer(const char *buf);
120
121#define BOOTUP_TRACER_SIZE 100
122static char bootup_tracer_buf[BOOTUP_TRACER_SIZE] __initdata;
123static char *default_bootup_tracer;
113 124
114static int __init set_ftrace(char *str) 125static int __init set_ftrace(char *str)
115{ 126{
116 tracing_set_tracer(str); 127 strncpy(bootup_tracer_buf, str, BOOTUP_TRACER_SIZE);
128 default_bootup_tracer = bootup_tracer_buf;
117 return 1; 129 return 1;
118} 130}
119__setup("ftrace", set_ftrace); 131__setup("ftrace=", set_ftrace);
120 132
121static int __init set_ftrace_dump_on_oops(char *str) 133static int __init set_ftrace_dump_on_oops(char *str)
122{ 134{
@@ -186,9 +198,6 @@ int tracing_is_enabled(void)
186 return tracer_enabled; 198 return tracer_enabled;
187} 199}
188 200
189/* function tracing enabled */
190int ftrace_function_enabled;
191
192/* 201/*
193 * trace_buf_size is the size in bytes that is allocated 202 * trace_buf_size is the size in bytes that is allocated
194 * for a buffer. Note, the number of bytes is always rounded 203 * for a buffer. Note, the number of bytes is always rounded
@@ -229,7 +238,7 @@ static DECLARE_WAIT_QUEUE_HEAD(trace_wait);
229 238
230/* trace_flags holds trace_options default values */ 239/* trace_flags holds trace_options default values */
231unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK | 240unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
232 TRACE_ITER_ANNOTATE; 241 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO;
233 242
234/** 243/**
235 * trace_wake_up - wake up tasks waiting for trace input 244 * trace_wake_up - wake up tasks waiting for trace input
@@ -287,6 +296,7 @@ static const char *trace_options[] = {
287 "userstacktrace", 296 "userstacktrace",
288 "sym-userobj", 297 "sym-userobj",
289 "printk-msg-only", 298 "printk-msg-only",
299 "context-info",
290 NULL 300 NULL
291}; 301};
292 302
@@ -326,133 +336,7 @@ __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
326 data->rt_priority = tsk->rt_priority; 336 data->rt_priority = tsk->rt_priority;
327 337
328 /* record this tasks comm */ 338 /* record this tasks comm */
329 tracing_record_cmdline(current); 339 tracing_record_cmdline(tsk);
330}
331
332/**
333 * trace_seq_printf - sequence printing of trace information
334 * @s: trace sequence descriptor
335 * @fmt: printf format string
336 *
337 * The tracer may use either sequence operations or its own
338 * copy to user routines. To simplify formating of a trace
339 * trace_seq_printf is used to store strings into a special
340 * buffer (@s). Then the output may be either used by
341 * the sequencer or pulled into another buffer.
342 */
343int
344trace_seq_printf(struct trace_seq *s, const char *fmt, ...)
345{
346 int len = (PAGE_SIZE - 1) - s->len;
347 va_list ap;
348 int ret;
349
350 if (!len)
351 return 0;
352
353 va_start(ap, fmt);
354 ret = vsnprintf(s->buffer + s->len, len, fmt, ap);
355 va_end(ap);
356
357 /* If we can't write it all, don't bother writing anything */
358 if (ret >= len)
359 return 0;
360
361 s->len += ret;
362
363 return len;
364}
365
366/**
367 * trace_seq_puts - trace sequence printing of simple string
368 * @s: trace sequence descriptor
369 * @str: simple string to record
370 *
371 * The tracer may use either the sequence operations or its own
372 * copy to user routines. This function records a simple string
373 * into a special buffer (@s) for later retrieval by a sequencer
374 * or other mechanism.
375 */
376static int
377trace_seq_puts(struct trace_seq *s, const char *str)
378{
379 int len = strlen(str);
380
381 if (len > ((PAGE_SIZE - 1) - s->len))
382 return 0;
383
384 memcpy(s->buffer + s->len, str, len);
385 s->len += len;
386
387 return len;
388}
389
390static int
391trace_seq_putc(struct trace_seq *s, unsigned char c)
392{
393 if (s->len >= (PAGE_SIZE - 1))
394 return 0;
395
396 s->buffer[s->len++] = c;
397
398 return 1;
399}
400
401static int
402trace_seq_putmem(struct trace_seq *s, void *mem, size_t len)
403{
404 if (len > ((PAGE_SIZE - 1) - s->len))
405 return 0;
406
407 memcpy(s->buffer + s->len, mem, len);
408 s->len += len;
409
410 return len;
411}
412
413#define MAX_MEMHEX_BYTES 8
414#define HEX_CHARS (MAX_MEMHEX_BYTES*2 + 1)
415
416static int
417trace_seq_putmem_hex(struct trace_seq *s, void *mem, size_t len)
418{
419 unsigned char hex[HEX_CHARS];
420 unsigned char *data = mem;
421 int i, j;
422
423#ifdef __BIG_ENDIAN
424 for (i = 0, j = 0; i < len; i++) {
425#else
426 for (i = len-1, j = 0; i >= 0; i--) {
427#endif
428 hex[j++] = hex_asc_hi(data[i]);
429 hex[j++] = hex_asc_lo(data[i]);
430 }
431 hex[j++] = ' ';
432
433 return trace_seq_putmem(s, hex, j);
434}
435
436static int
437trace_seq_path(struct trace_seq *s, struct path *path)
438{
439 unsigned char *p;
440
441 if (s->len >= (PAGE_SIZE - 1))
442 return 0;
443 p = d_path(path, s->buffer + s->len, PAGE_SIZE - s->len);
444 if (!IS_ERR(p)) {
445 p = mangle_path(s->buffer + s->len, p, "\n");
446 if (p) {
447 s->len = p - s->buffer;
448 return 1;
449 }
450 } else {
451 s->buffer[s->len++] = '?';
452 return 1;
453 }
454
455 return 0;
456} 340}
457 341
458static void 342static void
@@ -481,6 +365,25 @@ ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
481 return cnt; 365 return cnt;
482} 366}
483 367
368ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
369{
370 int len;
371 void *ret;
372
373 if (s->len <= s->readpos)
374 return -EBUSY;
375
376 len = s->len - s->readpos;
377 if (cnt > len)
378 cnt = len;
379 ret = memcpy(buf, s->buffer + s->readpos, cnt);
380 if (!ret)
381 return -EFAULT;
382
383 s->readpos += len;
384 return cnt;
385}
386
484static void 387static void
485trace_print_seq(struct seq_file *m, struct trace_seq *s) 388trace_print_seq(struct seq_file *m, struct trace_seq *s)
486{ 389{
@@ -543,7 +446,7 @@ update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
543 446
544 ftrace_enable_cpu(); 447 ftrace_enable_cpu();
545 448
546 WARN_ON_ONCE(ret); 449 WARN_ON_ONCE(ret && ret != -EAGAIN);
547 450
548 __update_max_tr(tr, tsk, cpu); 451 __update_max_tr(tr, tsk, cpu);
549 __raw_spin_unlock(&ftrace_max_lock); 452 __raw_spin_unlock(&ftrace_max_lock);
@@ -556,6 +459,8 @@ update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
556 * Register a new plugin tracer. 459 * Register a new plugin tracer.
557 */ 460 */
558int register_tracer(struct tracer *type) 461int register_tracer(struct tracer *type)
462__releases(kernel_lock)
463__acquires(kernel_lock)
559{ 464{
560 struct tracer *t; 465 struct tracer *t;
561 int len; 466 int len;
@@ -594,9 +499,12 @@ int register_tracer(struct tracer *type)
594 else 499 else
595 if (!type->flags->opts) 500 if (!type->flags->opts)
596 type->flags->opts = dummy_tracer_opt; 501 type->flags->opts = dummy_tracer_opt;
502 if (!type->wait_pipe)
503 type->wait_pipe = default_wait_pipe;
504
597 505
598#ifdef CONFIG_FTRACE_STARTUP_TEST 506#ifdef CONFIG_FTRACE_STARTUP_TEST
599 if (type->selftest) { 507 if (type->selftest && !tracing_selftest_disabled) {
600 struct tracer *saved_tracer = current_trace; 508 struct tracer *saved_tracer = current_trace;
601 struct trace_array *tr = &global_trace; 509 struct trace_array *tr = &global_trace;
602 int i; 510 int i;
@@ -638,8 +546,26 @@ int register_tracer(struct tracer *type)
638 out: 546 out:
639 tracing_selftest_running = false; 547 tracing_selftest_running = false;
640 mutex_unlock(&trace_types_lock); 548 mutex_unlock(&trace_types_lock);
641 lock_kernel();
642 549
550 if (ret || !default_bootup_tracer)
551 goto out_unlock;
552
553 if (strncmp(default_bootup_tracer, type->name, BOOTUP_TRACER_SIZE))
554 goto out_unlock;
555
556 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
557 /* Do we want this tracer to start on bootup? */
558 tracing_set_tracer(type->name);
559 default_bootup_tracer = NULL;
560 /* disable other selftests, since this will break it. */
561 tracing_selftest_disabled = 1;
562#ifdef CONFIG_FTRACE_STARTUP_TEST
563 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
564 type->name);
565#endif
566
567 out_unlock:
568 lock_kernel();
643 return ret; 569 return ret;
644} 570}
645 571
@@ -658,6 +584,15 @@ void unregister_tracer(struct tracer *type)
658 584
659 found: 585 found:
660 *t = (*t)->next; 586 *t = (*t)->next;
587
588 if (type == current_trace && tracer_enabled) {
589 tracer_enabled = 0;
590 tracing_stop();
591 if (current_trace->stop)
592 current_trace->stop(&global_trace);
593 current_trace = &nop_trace;
594 }
595
661 if (strlen(type->name) != max_tracer_type_len) 596 if (strlen(type->name) != max_tracer_type_len)
662 goto out; 597 goto out;
663 598
@@ -696,7 +631,7 @@ static int cmdline_idx;
696static DEFINE_SPINLOCK(trace_cmdline_lock); 631static DEFINE_SPINLOCK(trace_cmdline_lock);
697 632
698/* temporary disable recording */ 633/* temporary disable recording */
699atomic_t trace_record_cmdline_disabled __read_mostly; 634static atomic_t trace_record_cmdline_disabled __read_mostly;
700 635
701static void trace_init_cmdlines(void) 636static void trace_init_cmdlines(void)
702{ 637{
@@ -738,13 +673,12 @@ void tracing_start(void)
738 return; 673 return;
739 674
740 spin_lock_irqsave(&tracing_start_lock, flags); 675 spin_lock_irqsave(&tracing_start_lock, flags);
741 if (--trace_stop_count) 676 if (--trace_stop_count) {
742 goto out; 677 if (trace_stop_count < 0) {
743 678 /* Someone screwed up their debugging */
744 if (trace_stop_count < 0) { 679 WARN_ON_ONCE(1);
745 /* Someone screwed up their debugging */ 680 trace_stop_count = 0;
746 WARN_ON_ONCE(1); 681 }
747 trace_stop_count = 0;
748 goto out; 682 goto out;
749 } 683 }
750 684
@@ -876,78 +810,100 @@ tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
876 (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0); 810 (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
877} 811}
878 812
813struct ring_buffer_event *trace_buffer_lock_reserve(struct trace_array *tr,
814 unsigned char type,
815 unsigned long len,
816 unsigned long flags, int pc)
817{
818 struct ring_buffer_event *event;
819
820 event = ring_buffer_lock_reserve(tr->buffer, len);
821 if (event != NULL) {
822 struct trace_entry *ent = ring_buffer_event_data(event);
823
824 tracing_generic_entry_update(ent, flags, pc);
825 ent->type = type;
826 }
827
828 return event;
829}
830static void ftrace_trace_stack(struct trace_array *tr,
831 unsigned long flags, int skip, int pc);
832static void ftrace_trace_userstack(struct trace_array *tr,
833 unsigned long flags, int pc);
834
835void trace_buffer_unlock_commit(struct trace_array *tr,
836 struct ring_buffer_event *event,
837 unsigned long flags, int pc)
838{
839 ring_buffer_unlock_commit(tr->buffer, event);
840
841 ftrace_trace_stack(tr, flags, 6, pc);
842 ftrace_trace_userstack(tr, flags, pc);
843 trace_wake_up();
844}
845
879void 846void
880trace_function(struct trace_array *tr, struct trace_array_cpu *data, 847trace_function(struct trace_array *tr,
881 unsigned long ip, unsigned long parent_ip, unsigned long flags, 848 unsigned long ip, unsigned long parent_ip, unsigned long flags,
882 int pc) 849 int pc)
883{ 850{
884 struct ring_buffer_event *event; 851 struct ring_buffer_event *event;
885 struct ftrace_entry *entry; 852 struct ftrace_entry *entry;
886 unsigned long irq_flags;
887 853
888 /* If we are reading the ring buffer, don't trace */ 854 /* If we are reading the ring buffer, don't trace */
889 if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled)))) 855 if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled))))
890 return; 856 return;
891 857
892 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry), 858 event = trace_buffer_lock_reserve(tr, TRACE_FN, sizeof(*entry),
893 &irq_flags); 859 flags, pc);
894 if (!event) 860 if (!event)
895 return; 861 return;
896 entry = ring_buffer_event_data(event); 862 entry = ring_buffer_event_data(event);
897 tracing_generic_entry_update(&entry->ent, flags, pc);
898 entry->ent.type = TRACE_FN;
899 entry->ip = ip; 863 entry->ip = ip;
900 entry->parent_ip = parent_ip; 864 entry->parent_ip = parent_ip;
901 ring_buffer_unlock_commit(tr->buffer, event, irq_flags); 865 ring_buffer_unlock_commit(tr->buffer, event);
902} 866}
903 867
904#ifdef CONFIG_FUNCTION_GRAPH_TRACER 868#ifdef CONFIG_FUNCTION_GRAPH_TRACER
905static void __trace_graph_entry(struct trace_array *tr, 869static void __trace_graph_entry(struct trace_array *tr,
906 struct trace_array_cpu *data,
907 struct ftrace_graph_ent *trace, 870 struct ftrace_graph_ent *trace,
908 unsigned long flags, 871 unsigned long flags,
909 int pc) 872 int pc)
910{ 873{
911 struct ring_buffer_event *event; 874 struct ring_buffer_event *event;
912 struct ftrace_graph_ent_entry *entry; 875 struct ftrace_graph_ent_entry *entry;
913 unsigned long irq_flags;
914 876
915 if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled)))) 877 if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled))))
916 return; 878 return;
917 879
918 event = ring_buffer_lock_reserve(global_trace.buffer, sizeof(*entry), 880 event = trace_buffer_lock_reserve(&global_trace, TRACE_GRAPH_ENT,
919 &irq_flags); 881 sizeof(*entry), flags, pc);
920 if (!event) 882 if (!event)
921 return; 883 return;
922 entry = ring_buffer_event_data(event); 884 entry = ring_buffer_event_data(event);
923 tracing_generic_entry_update(&entry->ent, flags, pc);
924 entry->ent.type = TRACE_GRAPH_ENT;
925 entry->graph_ent = *trace; 885 entry->graph_ent = *trace;
926 ring_buffer_unlock_commit(global_trace.buffer, event, irq_flags); 886 ring_buffer_unlock_commit(global_trace.buffer, event);
927} 887}
928 888
929static void __trace_graph_return(struct trace_array *tr, 889static void __trace_graph_return(struct trace_array *tr,
930 struct trace_array_cpu *data,
931 struct ftrace_graph_ret *trace, 890 struct ftrace_graph_ret *trace,
932 unsigned long flags, 891 unsigned long flags,
933 int pc) 892 int pc)
934{ 893{
935 struct ring_buffer_event *event; 894 struct ring_buffer_event *event;
936 struct ftrace_graph_ret_entry *entry; 895 struct ftrace_graph_ret_entry *entry;
937 unsigned long irq_flags;
938 896
939 if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled)))) 897 if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled))))
940 return; 898 return;
941 899
942 event = ring_buffer_lock_reserve(global_trace.buffer, sizeof(*entry), 900 event = trace_buffer_lock_reserve(&global_trace, TRACE_GRAPH_RET,
943 &irq_flags); 901 sizeof(*entry), flags, pc);
944 if (!event) 902 if (!event)
945 return; 903 return;
946 entry = ring_buffer_event_data(event); 904 entry = ring_buffer_event_data(event);
947 tracing_generic_entry_update(&entry->ent, flags, pc);
948 entry->ent.type = TRACE_GRAPH_RET;
949 entry->ret = *trace; 905 entry->ret = *trace;
950 ring_buffer_unlock_commit(global_trace.buffer, event, irq_flags); 906 ring_buffer_unlock_commit(global_trace.buffer, event);
951} 907}
952#endif 908#endif
953 909
@@ -957,31 +913,23 @@ ftrace(struct trace_array *tr, struct trace_array_cpu *data,
957 int pc) 913 int pc)
958{ 914{
959 if (likely(!atomic_read(&data->disabled))) 915 if (likely(!atomic_read(&data->disabled)))
960 trace_function(tr, data, ip, parent_ip, flags, pc); 916 trace_function(tr, ip, parent_ip, flags, pc);
961} 917}
962 918
963static void ftrace_trace_stack(struct trace_array *tr, 919static void __ftrace_trace_stack(struct trace_array *tr,
964 struct trace_array_cpu *data, 920 unsigned long flags,
965 unsigned long flags, 921 int skip, int pc)
966 int skip, int pc)
967{ 922{
968#ifdef CONFIG_STACKTRACE 923#ifdef CONFIG_STACKTRACE
969 struct ring_buffer_event *event; 924 struct ring_buffer_event *event;
970 struct stack_entry *entry; 925 struct stack_entry *entry;
971 struct stack_trace trace; 926 struct stack_trace trace;
972 unsigned long irq_flags;
973 927
974 if (!(trace_flags & TRACE_ITER_STACKTRACE)) 928 event = trace_buffer_lock_reserve(tr, TRACE_STACK,
975 return; 929 sizeof(*entry), flags, pc);
976
977 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
978 &irq_flags);
979 if (!event) 930 if (!event)
980 return; 931 return;
981 entry = ring_buffer_event_data(event); 932 entry = ring_buffer_event_data(event);
982 tracing_generic_entry_update(&entry->ent, flags, pc);
983 entry->ent.type = TRACE_STACK;
984
985 memset(&entry->caller, 0, sizeof(entry->caller)); 933 memset(&entry->caller, 0, sizeof(entry->caller));
986 934
987 trace.nr_entries = 0; 935 trace.nr_entries = 0;
@@ -990,38 +938,43 @@ static void ftrace_trace_stack(struct trace_array *tr,
990 trace.entries = entry->caller; 938 trace.entries = entry->caller;
991 939
992 save_stack_trace(&trace); 940 save_stack_trace(&trace);
993 ring_buffer_unlock_commit(tr->buffer, event, irq_flags); 941 ring_buffer_unlock_commit(tr->buffer, event);
994#endif 942#endif
995} 943}
996 944
945static void ftrace_trace_stack(struct trace_array *tr,
946 unsigned long flags,
947 int skip, int pc)
948{
949 if (!(trace_flags & TRACE_ITER_STACKTRACE))
950 return;
951
952 __ftrace_trace_stack(tr, flags, skip, pc);
953}
954
997void __trace_stack(struct trace_array *tr, 955void __trace_stack(struct trace_array *tr,
998 struct trace_array_cpu *data,
999 unsigned long flags, 956 unsigned long flags,
1000 int skip) 957 int skip, int pc)
1001{ 958{
1002 ftrace_trace_stack(tr, data, flags, skip, preempt_count()); 959 __ftrace_trace_stack(tr, flags, skip, pc);
1003} 960}
1004 961
1005static void ftrace_trace_userstack(struct trace_array *tr, 962static void ftrace_trace_userstack(struct trace_array *tr,
1006 struct trace_array_cpu *data, 963 unsigned long flags, int pc)
1007 unsigned long flags, int pc)
1008{ 964{
1009#ifdef CONFIG_STACKTRACE 965#ifdef CONFIG_STACKTRACE
1010 struct ring_buffer_event *event; 966 struct ring_buffer_event *event;
1011 struct userstack_entry *entry; 967 struct userstack_entry *entry;
1012 struct stack_trace trace; 968 struct stack_trace trace;
1013 unsigned long irq_flags;
1014 969
1015 if (!(trace_flags & TRACE_ITER_USERSTACKTRACE)) 970 if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1016 return; 971 return;
1017 972
1018 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry), 973 event = trace_buffer_lock_reserve(tr, TRACE_USER_STACK,
1019 &irq_flags); 974 sizeof(*entry), flags, pc);
1020 if (!event) 975 if (!event)
1021 return; 976 return;
1022 entry = ring_buffer_event_data(event); 977 entry = ring_buffer_event_data(event);
1023 tracing_generic_entry_update(&entry->ent, flags, pc);
1024 entry->ent.type = TRACE_USER_STACK;
1025 978
1026 memset(&entry->caller, 0, sizeof(entry->caller)); 979 memset(&entry->caller, 0, sizeof(entry->caller));
1027 980
@@ -1031,70 +984,58 @@ static void ftrace_trace_userstack(struct trace_array *tr,
1031 trace.entries = entry->caller; 984 trace.entries = entry->caller;
1032 985
1033 save_stack_trace_user(&trace); 986 save_stack_trace_user(&trace);
1034 ring_buffer_unlock_commit(tr->buffer, event, irq_flags); 987 ring_buffer_unlock_commit(tr->buffer, event);
1035#endif 988#endif
1036} 989}
1037 990
1038void __trace_userstack(struct trace_array *tr, 991#ifdef UNUSED
1039 struct trace_array_cpu *data, 992static void __trace_userstack(struct trace_array *tr, unsigned long flags)
1040 unsigned long flags)
1041{ 993{
1042 ftrace_trace_userstack(tr, data, flags, preempt_count()); 994 ftrace_trace_userstack(tr, flags, preempt_count());
1043} 995}
996#endif /* UNUSED */
1044 997
1045static void 998static void
1046ftrace_trace_special(void *__tr, void *__data, 999ftrace_trace_special(void *__tr,
1047 unsigned long arg1, unsigned long arg2, unsigned long arg3, 1000 unsigned long arg1, unsigned long arg2, unsigned long arg3,
1048 int pc) 1001 int pc)
1049{ 1002{
1050 struct ring_buffer_event *event; 1003 struct ring_buffer_event *event;
1051 struct trace_array_cpu *data = __data;
1052 struct trace_array *tr = __tr; 1004 struct trace_array *tr = __tr;
1053 struct special_entry *entry; 1005 struct special_entry *entry;
1054 unsigned long irq_flags;
1055 1006
1056 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry), 1007 event = trace_buffer_lock_reserve(tr, TRACE_SPECIAL,
1057 &irq_flags); 1008 sizeof(*entry), 0, pc);
1058 if (!event) 1009 if (!event)
1059 return; 1010 return;
1060 entry = ring_buffer_event_data(event); 1011 entry = ring_buffer_event_data(event);
1061 tracing_generic_entry_update(&entry->ent, 0, pc);
1062 entry->ent.type = TRACE_SPECIAL;
1063 entry->arg1 = arg1; 1012 entry->arg1 = arg1;
1064 entry->arg2 = arg2; 1013 entry->arg2 = arg2;
1065 entry->arg3 = arg3; 1014 entry->arg3 = arg3;
1066 ring_buffer_unlock_commit(tr->buffer, event, irq_flags); 1015 trace_buffer_unlock_commit(tr, event, 0, pc);
1067 ftrace_trace_stack(tr, data, irq_flags, 4, pc);
1068 ftrace_trace_userstack(tr, data, irq_flags, pc);
1069
1070 trace_wake_up();
1071} 1016}
1072 1017
1073void 1018void
1074__trace_special(void *__tr, void *__data, 1019__trace_special(void *__tr, void *__data,
1075 unsigned long arg1, unsigned long arg2, unsigned long arg3) 1020 unsigned long arg1, unsigned long arg2, unsigned long arg3)
1076{ 1021{
1077 ftrace_trace_special(__tr, __data, arg1, arg2, arg3, preempt_count()); 1022 ftrace_trace_special(__tr, arg1, arg2, arg3, preempt_count());
1078} 1023}
1079 1024
1080void 1025void
1081tracing_sched_switch_trace(struct trace_array *tr, 1026tracing_sched_switch_trace(struct trace_array *tr,
1082 struct trace_array_cpu *data,
1083 struct task_struct *prev, 1027 struct task_struct *prev,
1084 struct task_struct *next, 1028 struct task_struct *next,
1085 unsigned long flags, int pc) 1029 unsigned long flags, int pc)
1086{ 1030{
1087 struct ring_buffer_event *event; 1031 struct ring_buffer_event *event;
1088 struct ctx_switch_entry *entry; 1032 struct ctx_switch_entry *entry;
1089 unsigned long irq_flags;
1090 1033
1091 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry), 1034 event = trace_buffer_lock_reserve(tr, TRACE_CTX,
1092 &irq_flags); 1035 sizeof(*entry), flags, pc);
1093 if (!event) 1036 if (!event)
1094 return; 1037 return;
1095 entry = ring_buffer_event_data(event); 1038 entry = ring_buffer_event_data(event);
1096 tracing_generic_entry_update(&entry->ent, flags, pc);
1097 entry->ent.type = TRACE_CTX;
1098 entry->prev_pid = prev->pid; 1039 entry->prev_pid = prev->pid;
1099 entry->prev_prio = prev->prio; 1040 entry->prev_prio = prev->prio;
1100 entry->prev_state = prev->state; 1041 entry->prev_state = prev->state;
@@ -1102,29 +1043,23 @@ tracing_sched_switch_trace(struct trace_array *tr,
1102 entry->next_prio = next->prio; 1043 entry->next_prio = next->prio;
1103 entry->next_state = next->state; 1044 entry->next_state = next->state;
1104 entry->next_cpu = task_cpu(next); 1045 entry->next_cpu = task_cpu(next);
1105 ring_buffer_unlock_commit(tr->buffer, event, irq_flags); 1046 trace_buffer_unlock_commit(tr, event, flags, pc);
1106 ftrace_trace_stack(tr, data, flags, 5, pc);
1107 ftrace_trace_userstack(tr, data, flags, pc);
1108} 1047}
1109 1048
1110void 1049void
1111tracing_sched_wakeup_trace(struct trace_array *tr, 1050tracing_sched_wakeup_trace(struct trace_array *tr,
1112 struct trace_array_cpu *data,
1113 struct task_struct *wakee, 1051 struct task_struct *wakee,
1114 struct task_struct *curr, 1052 struct task_struct *curr,
1115 unsigned long flags, int pc) 1053 unsigned long flags, int pc)
1116{ 1054{
1117 struct ring_buffer_event *event; 1055 struct ring_buffer_event *event;
1118 struct ctx_switch_entry *entry; 1056 struct ctx_switch_entry *entry;
1119 unsigned long irq_flags;
1120 1057
1121 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry), 1058 event = trace_buffer_lock_reserve(tr, TRACE_WAKE,
1122 &irq_flags); 1059 sizeof(*entry), flags, pc);
1123 if (!event) 1060 if (!event)
1124 return; 1061 return;
1125 entry = ring_buffer_event_data(event); 1062 entry = ring_buffer_event_data(event);
1126 tracing_generic_entry_update(&entry->ent, flags, pc);
1127 entry->ent.type = TRACE_WAKE;
1128 entry->prev_pid = curr->pid; 1063 entry->prev_pid = curr->pid;
1129 entry->prev_prio = curr->prio; 1064 entry->prev_prio = curr->prio;
1130 entry->prev_state = curr->state; 1065 entry->prev_state = curr->state;
@@ -1132,11 +1067,10 @@ tracing_sched_wakeup_trace(struct trace_array *tr,
1132 entry->next_prio = wakee->prio; 1067 entry->next_prio = wakee->prio;
1133 entry->next_state = wakee->state; 1068 entry->next_state = wakee->state;
1134 entry->next_cpu = task_cpu(wakee); 1069 entry->next_cpu = task_cpu(wakee);
1135 ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
1136 ftrace_trace_stack(tr, data, flags, 6, pc);
1137 ftrace_trace_userstack(tr, data, flags, pc);
1138 1070
1139 trace_wake_up(); 1071 ring_buffer_unlock_commit(tr->buffer, event);
1072 ftrace_trace_stack(tr, flags, 6, pc);
1073 ftrace_trace_userstack(tr, flags, pc);
1140} 1074}
1141 1075
1142void 1076void
@@ -1157,66 +1091,7 @@ ftrace_special(unsigned long arg1, unsigned long arg2, unsigned long arg3)
1157 data = tr->data[cpu]; 1091 data = tr->data[cpu];
1158 1092
1159 if (likely(atomic_inc_return(&data->disabled) == 1)) 1093 if (likely(atomic_inc_return(&data->disabled) == 1))
1160 ftrace_trace_special(tr, data, arg1, arg2, arg3, pc); 1094 ftrace_trace_special(tr, arg1, arg2, arg3, pc);
1161
1162 atomic_dec(&data->disabled);
1163 local_irq_restore(flags);
1164}
1165
1166#ifdef CONFIG_FUNCTION_TRACER
1167static void
1168function_trace_call_preempt_only(unsigned long ip, unsigned long parent_ip)
1169{
1170 struct trace_array *tr = &global_trace;
1171 struct trace_array_cpu *data;
1172 unsigned long flags;
1173 long disabled;
1174 int cpu, resched;
1175 int pc;
1176
1177 if (unlikely(!ftrace_function_enabled))
1178 return;
1179
1180 pc = preempt_count();
1181 resched = ftrace_preempt_disable();
1182 local_save_flags(flags);
1183 cpu = raw_smp_processor_id();
1184 data = tr->data[cpu];
1185 disabled = atomic_inc_return(&data->disabled);
1186
1187 if (likely(disabled == 1))
1188 trace_function(tr, data, ip, parent_ip, flags, pc);
1189
1190 atomic_dec(&data->disabled);
1191 ftrace_preempt_enable(resched);
1192}
1193
1194static void
1195function_trace_call(unsigned long ip, unsigned long parent_ip)
1196{
1197 struct trace_array *tr = &global_trace;
1198 struct trace_array_cpu *data;
1199 unsigned long flags;
1200 long disabled;
1201 int cpu;
1202 int pc;
1203
1204 if (unlikely(!ftrace_function_enabled))
1205 return;
1206
1207 /*
1208 * Need to use raw, since this must be called before the
1209 * recursive protection is performed.
1210 */
1211 local_irq_save(flags);
1212 cpu = raw_smp_processor_id();
1213 data = tr->data[cpu];
1214 disabled = atomic_inc_return(&data->disabled);
1215
1216 if (likely(disabled == 1)) {
1217 pc = preempt_count();
1218 trace_function(tr, data, ip, parent_ip, flags, pc);
1219 }
1220 1095
1221 atomic_dec(&data->disabled); 1096 atomic_dec(&data->disabled);
1222 local_irq_restore(flags); 1097 local_irq_restore(flags);
@@ -1244,7 +1119,7 @@ int trace_graph_entry(struct ftrace_graph_ent *trace)
1244 disabled = atomic_inc_return(&data->disabled); 1119 disabled = atomic_inc_return(&data->disabled);
1245 if (likely(disabled == 1)) { 1120 if (likely(disabled == 1)) {
1246 pc = preempt_count(); 1121 pc = preempt_count();
1247 __trace_graph_entry(tr, data, trace, flags, pc); 1122 __trace_graph_entry(tr, trace, flags, pc);
1248 } 1123 }
1249 /* Only do the atomic if it is not already set */ 1124 /* Only do the atomic if it is not already set */
1250 if (!test_tsk_trace_graph(current)) 1125 if (!test_tsk_trace_graph(current))
@@ -1270,7 +1145,7 @@ void trace_graph_return(struct ftrace_graph_ret *trace)
1270 disabled = atomic_inc_return(&data->disabled); 1145 disabled = atomic_inc_return(&data->disabled);
1271 if (likely(disabled == 1)) { 1146 if (likely(disabled == 1)) {
1272 pc = preempt_count(); 1147 pc = preempt_count();
1273 __trace_graph_return(tr, data, trace, flags, pc); 1148 __trace_graph_return(tr, trace, flags, pc);
1274 } 1149 }
1275 if (!trace->depth) 1150 if (!trace->depth)
1276 clear_tsk_trace_graph(current); 1151 clear_tsk_trace_graph(current);
@@ -1279,31 +1154,6 @@ void trace_graph_return(struct ftrace_graph_ret *trace)
1279} 1154}
1280#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 1155#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
1281 1156
1282static struct ftrace_ops trace_ops __read_mostly =
1283{
1284 .func = function_trace_call,
1285};
1286
1287void tracing_start_function_trace(void)
1288{
1289 ftrace_function_enabled = 0;
1290
1291 if (trace_flags & TRACE_ITER_PREEMPTONLY)
1292 trace_ops.func = function_trace_call_preempt_only;
1293 else
1294 trace_ops.func = function_trace_call;
1295
1296 register_ftrace_function(&trace_ops);
1297 ftrace_function_enabled = 1;
1298}
1299
1300void tracing_stop_function_trace(void)
1301{
1302 ftrace_function_enabled = 0;
1303 unregister_ftrace_function(&trace_ops);
1304}
1305#endif
1306
1307enum trace_file_type { 1157enum trace_file_type {
1308 TRACE_FILE_LAT_FMT = 1, 1158 TRACE_FILE_LAT_FMT = 1,
1309 TRACE_FILE_ANNOTATE = 2, 1159 TRACE_FILE_ANNOTATE = 2,
@@ -1376,8 +1226,8 @@ __find_next_entry(struct trace_iterator *iter, int *ent_cpu, u64 *ent_ts)
1376} 1226}
1377 1227
1378/* Find the next real entry, without updating the iterator itself */ 1228/* Find the next real entry, without updating the iterator itself */
1379static struct trace_entry * 1229struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
1380find_next_entry(struct trace_iterator *iter, int *ent_cpu, u64 *ent_ts) 1230 int *ent_cpu, u64 *ent_ts)
1381{ 1231{
1382 return __find_next_entry(iter, ent_cpu, ent_ts); 1232 return __find_next_entry(iter, ent_cpu, ent_ts);
1383} 1233}
@@ -1472,154 +1322,6 @@ static void s_stop(struct seq_file *m, void *p)
1472 mutex_unlock(&trace_types_lock); 1322 mutex_unlock(&trace_types_lock);
1473} 1323}
1474 1324
1475#ifdef CONFIG_KRETPROBES
1476static inline const char *kretprobed(const char *name)
1477{
1478 static const char tramp_name[] = "kretprobe_trampoline";
1479 int size = sizeof(tramp_name);
1480
1481 if (strncmp(tramp_name, name, size) == 0)
1482 return "[unknown/kretprobe'd]";
1483 return name;
1484}
1485#else
1486static inline const char *kretprobed(const char *name)
1487{
1488 return name;
1489}
1490#endif /* CONFIG_KRETPROBES */
1491
1492static int
1493seq_print_sym_short(struct trace_seq *s, const char *fmt, unsigned long address)
1494{
1495#ifdef CONFIG_KALLSYMS
1496 char str[KSYM_SYMBOL_LEN];
1497 const char *name;
1498
1499 kallsyms_lookup(address, NULL, NULL, NULL, str);
1500
1501 name = kretprobed(str);
1502
1503 return trace_seq_printf(s, fmt, name);
1504#endif
1505 return 1;
1506}
1507
1508static int
1509seq_print_sym_offset(struct trace_seq *s, const char *fmt,
1510 unsigned long address)
1511{
1512#ifdef CONFIG_KALLSYMS
1513 char str[KSYM_SYMBOL_LEN];
1514 const char *name;
1515
1516 sprint_symbol(str, address);
1517 name = kretprobed(str);
1518
1519 return trace_seq_printf(s, fmt, name);
1520#endif
1521 return 1;
1522}
1523
1524#ifndef CONFIG_64BIT
1525# define IP_FMT "%08lx"
1526#else
1527# define IP_FMT "%016lx"
1528#endif
1529
1530int
1531seq_print_ip_sym(struct trace_seq *s, unsigned long ip, unsigned long sym_flags)
1532{
1533 int ret;
1534
1535 if (!ip)
1536 return trace_seq_printf(s, "0");
1537
1538 if (sym_flags & TRACE_ITER_SYM_OFFSET)
1539 ret = seq_print_sym_offset(s, "%s", ip);
1540 else
1541 ret = seq_print_sym_short(s, "%s", ip);
1542
1543 if (!ret)
1544 return 0;
1545
1546 if (sym_flags & TRACE_ITER_SYM_ADDR)
1547 ret = trace_seq_printf(s, " <" IP_FMT ">", ip);
1548 return ret;
1549}
1550
1551static inline int seq_print_user_ip(struct trace_seq *s, struct mm_struct *mm,
1552 unsigned long ip, unsigned long sym_flags)
1553{
1554 struct file *file = NULL;
1555 unsigned long vmstart = 0;
1556 int ret = 1;
1557
1558 if (mm) {
1559 const struct vm_area_struct *vma;
1560
1561 down_read(&mm->mmap_sem);
1562 vma = find_vma(mm, ip);
1563 if (vma) {
1564 file = vma->vm_file;
1565 vmstart = vma->vm_start;
1566 }
1567 if (file) {
1568 ret = trace_seq_path(s, &file->f_path);
1569 if (ret)
1570 ret = trace_seq_printf(s, "[+0x%lx]", ip - vmstart);
1571 }
1572 up_read(&mm->mmap_sem);
1573 }
1574 if (ret && ((sym_flags & TRACE_ITER_SYM_ADDR) || !file))
1575 ret = trace_seq_printf(s, " <" IP_FMT ">", ip);
1576 return ret;
1577}
1578
1579static int
1580seq_print_userip_objs(const struct userstack_entry *entry, struct trace_seq *s,
1581 unsigned long sym_flags)
1582{
1583 struct mm_struct *mm = NULL;
1584 int ret = 1;
1585 unsigned int i;
1586
1587 if (trace_flags & TRACE_ITER_SYM_USEROBJ) {
1588 struct task_struct *task;
1589 /*
1590 * we do the lookup on the thread group leader,
1591 * since individual threads might have already quit!
1592 */
1593 rcu_read_lock();
1594 task = find_task_by_vpid(entry->ent.tgid);
1595 if (task)
1596 mm = get_task_mm(task);
1597 rcu_read_unlock();
1598 }
1599
1600 for (i = 0; i < FTRACE_STACK_ENTRIES; i++) {
1601 unsigned long ip = entry->caller[i];
1602
1603 if (ip == ULONG_MAX || !ret)
1604 break;
1605 if (i && ret)
1606 ret = trace_seq_puts(s, " <- ");
1607 if (!ip) {
1608 if (ret)
1609 ret = trace_seq_puts(s, "??");
1610 continue;
1611 }
1612 if (!ret)
1613 break;
1614 if (ret)
1615 ret = seq_print_user_ip(s, mm, ip, sym_flags);
1616 }
1617
1618 if (mm)
1619 mmput(mm);
1620 return ret;
1621}
1622
1623static void print_lat_help_header(struct seq_file *m) 1325static void print_lat_help_header(struct seq_file *m)
1624{ 1326{
1625 seq_puts(m, "# _------=> CPU# \n"); 1327 seq_puts(m, "# _------=> CPU# \n");
@@ -1704,103 +1406,6 @@ print_trace_header(struct seq_file *m, struct trace_iterator *iter)
1704 seq_puts(m, "\n"); 1406 seq_puts(m, "\n");
1705} 1407}
1706 1408
1707static void
1708lat_print_generic(struct trace_seq *s, struct trace_entry *entry, int cpu)
1709{
1710 int hardirq, softirq;
1711 char *comm;
1712
1713 comm = trace_find_cmdline(entry->pid);
1714
1715 trace_seq_printf(s, "%8.8s-%-5d ", comm, entry->pid);
1716 trace_seq_printf(s, "%3d", cpu);
1717 trace_seq_printf(s, "%c%c",
1718 (entry->flags & TRACE_FLAG_IRQS_OFF) ? 'd' :
1719 (entry->flags & TRACE_FLAG_IRQS_NOSUPPORT) ? 'X' : '.',
1720 ((entry->flags & TRACE_FLAG_NEED_RESCHED) ? 'N' : '.'));
1721
1722 hardirq = entry->flags & TRACE_FLAG_HARDIRQ;
1723 softirq = entry->flags & TRACE_FLAG_SOFTIRQ;
1724 if (hardirq && softirq) {
1725 trace_seq_putc(s, 'H');
1726 } else {
1727 if (hardirq) {
1728 trace_seq_putc(s, 'h');
1729 } else {
1730 if (softirq)
1731 trace_seq_putc(s, 's');
1732 else
1733 trace_seq_putc(s, '.');
1734 }
1735 }
1736
1737 if (entry->preempt_count)
1738 trace_seq_printf(s, "%x", entry->preempt_count);
1739 else
1740 trace_seq_puts(s, ".");
1741}
1742
1743unsigned long preempt_mark_thresh = 100;
1744
1745static void
1746lat_print_timestamp(struct trace_seq *s, u64 abs_usecs,
1747 unsigned long rel_usecs)
1748{
1749 trace_seq_printf(s, " %4lldus", abs_usecs);
1750 if (rel_usecs > preempt_mark_thresh)
1751 trace_seq_puts(s, "!: ");
1752 else if (rel_usecs > 1)
1753 trace_seq_puts(s, "+: ");
1754 else
1755 trace_seq_puts(s, " : ");
1756}
1757
1758static const char state_to_char[] = TASK_STATE_TO_CHAR_STR;
1759
1760static int task_state_char(unsigned long state)
1761{
1762 int bit = state ? __ffs(state) + 1 : 0;
1763
1764 return bit < sizeof(state_to_char) - 1 ? state_to_char[bit] : '?';
1765}
1766
1767/*
1768 * The message is supposed to contain an ending newline.
1769 * If the printing stops prematurely, try to add a newline of our own.
1770 */
1771void trace_seq_print_cont(struct trace_seq *s, struct trace_iterator *iter)
1772{
1773 struct trace_entry *ent;
1774 struct trace_field_cont *cont;
1775 bool ok = true;
1776
1777 ent = peek_next_entry(iter, iter->cpu, NULL);
1778 if (!ent || ent->type != TRACE_CONT) {
1779 trace_seq_putc(s, '\n');
1780 return;
1781 }
1782
1783 do {
1784 cont = (struct trace_field_cont *)ent;
1785 if (ok)
1786 ok = (trace_seq_printf(s, "%s", cont->buf) > 0);
1787
1788 ftrace_disable_cpu();
1789
1790 if (iter->buffer_iter[iter->cpu])
1791 ring_buffer_read(iter->buffer_iter[iter->cpu], NULL);
1792 else
1793 ring_buffer_consume(iter->tr->buffer, iter->cpu, NULL);
1794
1795 ftrace_enable_cpu();
1796
1797 ent = peek_next_entry(iter, iter->cpu, NULL);
1798 } while (ent && ent->type == TRACE_CONT);
1799
1800 if (!ok)
1801 trace_seq_putc(s, '\n');
1802}
1803
1804static void test_cpu_buff_start(struct trace_iterator *iter) 1409static void test_cpu_buff_start(struct trace_iterator *iter)
1805{ 1410{
1806 struct trace_seq *s = &iter->seq; 1411 struct trace_seq *s = &iter->seq;
@@ -1818,138 +1423,31 @@ static void test_cpu_buff_start(struct trace_iterator *iter)
1818 trace_seq_printf(s, "##### CPU %u buffer started ####\n", iter->cpu); 1423 trace_seq_printf(s, "##### CPU %u buffer started ####\n", iter->cpu);
1819} 1424}
1820 1425
1821static enum print_line_t 1426static enum print_line_t print_lat_fmt(struct trace_iterator *iter)
1822print_lat_fmt(struct trace_iterator *iter, unsigned int trace_idx, int cpu)
1823{ 1427{
1824 struct trace_seq *s = &iter->seq; 1428 struct trace_seq *s = &iter->seq;
1825 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK); 1429 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1826 struct trace_entry *next_entry; 1430 struct trace_event *event;
1827 unsigned long verbose = (trace_flags & TRACE_ITER_VERBOSE);
1828 struct trace_entry *entry = iter->ent; 1431 struct trace_entry *entry = iter->ent;
1829 unsigned long abs_usecs;
1830 unsigned long rel_usecs;
1831 u64 next_ts;
1832 char *comm;
1833 int S, T;
1834 int i;
1835
1836 if (entry->type == TRACE_CONT)
1837 return TRACE_TYPE_HANDLED;
1838 1432
1839 test_cpu_buff_start(iter); 1433 test_cpu_buff_start(iter);
1840 1434
1841 next_entry = find_next_entry(iter, NULL, &next_ts); 1435 event = ftrace_find_event(entry->type);
1842 if (!next_entry)
1843 next_ts = iter->ts;
1844 rel_usecs = ns2usecs(next_ts - iter->ts);
1845 abs_usecs = ns2usecs(iter->ts - iter->tr->time_start);
1846
1847 if (verbose) {
1848 comm = trace_find_cmdline(entry->pid);
1849 trace_seq_printf(s, "%16s %5d %3d %d %08x %08x [%08lx]"
1850 " %ld.%03ldms (+%ld.%03ldms): ",
1851 comm,
1852 entry->pid, cpu, entry->flags,
1853 entry->preempt_count, trace_idx,
1854 ns2usecs(iter->ts),
1855 abs_usecs/1000,
1856 abs_usecs % 1000, rel_usecs/1000,
1857 rel_usecs % 1000);
1858 } else {
1859 lat_print_generic(s, entry, cpu);
1860 lat_print_timestamp(s, abs_usecs, rel_usecs);
1861 }
1862 switch (entry->type) {
1863 case TRACE_FN: {
1864 struct ftrace_entry *field;
1865
1866 trace_assign_type(field, entry);
1867
1868 seq_print_ip_sym(s, field->ip, sym_flags);
1869 trace_seq_puts(s, " (");
1870 seq_print_ip_sym(s, field->parent_ip, sym_flags);
1871 trace_seq_puts(s, ")\n");
1872 break;
1873 }
1874 case TRACE_CTX:
1875 case TRACE_WAKE: {
1876 struct ctx_switch_entry *field;
1877
1878 trace_assign_type(field, entry);
1879
1880 T = task_state_char(field->next_state);
1881 S = task_state_char(field->prev_state);
1882 comm = trace_find_cmdline(field->next_pid);
1883 trace_seq_printf(s, " %5d:%3d:%c %s [%03d] %5d:%3d:%c %s\n",
1884 field->prev_pid,
1885 field->prev_prio,
1886 S, entry->type == TRACE_CTX ? "==>" : " +",
1887 field->next_cpu,
1888 field->next_pid,
1889 field->next_prio,
1890 T, comm);
1891 break;
1892 }
1893 case TRACE_SPECIAL: {
1894 struct special_entry *field;
1895
1896 trace_assign_type(field, entry);
1897
1898 trace_seq_printf(s, "# %ld %ld %ld\n",
1899 field->arg1,
1900 field->arg2,
1901 field->arg3);
1902 break;
1903 }
1904 case TRACE_STACK: {
1905 struct stack_entry *field;
1906
1907 trace_assign_type(field, entry);
1908
1909 for (i = 0; i < FTRACE_STACK_ENTRIES; i++) {
1910 if (i)
1911 trace_seq_puts(s, " <= ");
1912 seq_print_ip_sym(s, field->caller[i], sym_flags);
1913 }
1914 trace_seq_puts(s, "\n");
1915 break;
1916 }
1917 case TRACE_PRINT: {
1918 struct print_entry *field;
1919
1920 trace_assign_type(field, entry);
1921 1436
1922 seq_print_ip_sym(s, field->ip, sym_flags); 1437 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
1923 trace_seq_printf(s, ": %s", field->buf); 1438 if (!trace_print_lat_context(iter))
1924 if (entry->flags & TRACE_FLAG_CONT) 1439 goto partial;
1925 trace_seq_print_cont(s, iter);
1926 break;
1927 } 1440 }
1928 case TRACE_BRANCH: {
1929 struct trace_branch *field;
1930 1441
1931 trace_assign_type(field, entry); 1442 if (event)
1443 return event->latency_trace(iter, sym_flags);
1932 1444
1933 trace_seq_printf(s, "[%s] %s:%s:%d\n", 1445 if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
1934 field->correct ? " ok " : " MISS ", 1446 goto partial;
1935 field->func,
1936 field->file,
1937 field->line);
1938 break;
1939 }
1940 case TRACE_USER_STACK: {
1941 struct userstack_entry *field;
1942 1447
1943 trace_assign_type(field, entry);
1944
1945 seq_print_userip_objs(field, s, sym_flags);
1946 trace_seq_putc(s, '\n');
1947 break;
1948 }
1949 default:
1950 trace_seq_printf(s, "Unknown type %d\n", entry->type);
1951 }
1952 return TRACE_TYPE_HANDLED; 1448 return TRACE_TYPE_HANDLED;
1449partial:
1450 return TRACE_TYPE_PARTIAL_LINE;
1953} 1451}
1954 1452
1955static enum print_line_t print_trace_fmt(struct trace_iterator *iter) 1453static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
@@ -1957,313 +1455,78 @@ static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
1957 struct trace_seq *s = &iter->seq; 1455 struct trace_seq *s = &iter->seq;
1958 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK); 1456 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1959 struct trace_entry *entry; 1457 struct trace_entry *entry;
1960 unsigned long usec_rem; 1458 struct trace_event *event;
1961 unsigned long long t;
1962 unsigned long secs;
1963 char *comm;
1964 int ret;
1965 int S, T;
1966 int i;
1967 1459
1968 entry = iter->ent; 1460 entry = iter->ent;
1969 1461
1970 if (entry->type == TRACE_CONT)
1971 return TRACE_TYPE_HANDLED;
1972
1973 test_cpu_buff_start(iter); 1462 test_cpu_buff_start(iter);
1974 1463
1975 comm = trace_find_cmdline(iter->ent->pid); 1464 event = ftrace_find_event(entry->type);
1976
1977 t = ns2usecs(iter->ts);
1978 usec_rem = do_div(t, 1000000ULL);
1979 secs = (unsigned long)t;
1980
1981 ret = trace_seq_printf(s, "%16s-%-5d ", comm, entry->pid);
1982 if (!ret)
1983 return TRACE_TYPE_PARTIAL_LINE;
1984 ret = trace_seq_printf(s, "[%03d] ", iter->cpu);
1985 if (!ret)
1986 return TRACE_TYPE_PARTIAL_LINE;
1987 ret = trace_seq_printf(s, "%5lu.%06lu: ", secs, usec_rem);
1988 if (!ret)
1989 return TRACE_TYPE_PARTIAL_LINE;
1990
1991 switch (entry->type) {
1992 case TRACE_FN: {
1993 struct ftrace_entry *field;
1994
1995 trace_assign_type(field, entry);
1996
1997 ret = seq_print_ip_sym(s, field->ip, sym_flags);
1998 if (!ret)
1999 return TRACE_TYPE_PARTIAL_LINE;
2000 if ((sym_flags & TRACE_ITER_PRINT_PARENT) &&
2001 field->parent_ip) {
2002 ret = trace_seq_printf(s, " <-");
2003 if (!ret)
2004 return TRACE_TYPE_PARTIAL_LINE;
2005 ret = seq_print_ip_sym(s,
2006 field->parent_ip,
2007 sym_flags);
2008 if (!ret)
2009 return TRACE_TYPE_PARTIAL_LINE;
2010 }
2011 ret = trace_seq_printf(s, "\n");
2012 if (!ret)
2013 return TRACE_TYPE_PARTIAL_LINE;
2014 break;
2015 }
2016 case TRACE_CTX:
2017 case TRACE_WAKE: {
2018 struct ctx_switch_entry *field;
2019
2020 trace_assign_type(field, entry);
2021
2022 T = task_state_char(field->next_state);
2023 S = task_state_char(field->prev_state);
2024 ret = trace_seq_printf(s, " %5d:%3d:%c %s [%03d] %5d:%3d:%c\n",
2025 field->prev_pid,
2026 field->prev_prio,
2027 S,
2028 entry->type == TRACE_CTX ? "==>" : " +",
2029 field->next_cpu,
2030 field->next_pid,
2031 field->next_prio,
2032 T);
2033 if (!ret)
2034 return TRACE_TYPE_PARTIAL_LINE;
2035 break;
2036 }
2037 case TRACE_SPECIAL: {
2038 struct special_entry *field;
2039
2040 trace_assign_type(field, entry);
2041
2042 ret = trace_seq_printf(s, "# %ld %ld %ld\n",
2043 field->arg1,
2044 field->arg2,
2045 field->arg3);
2046 if (!ret)
2047 return TRACE_TYPE_PARTIAL_LINE;
2048 break;
2049 }
2050 case TRACE_STACK: {
2051 struct stack_entry *field;
2052
2053 trace_assign_type(field, entry);
2054
2055 for (i = 0; i < FTRACE_STACK_ENTRIES; i++) {
2056 if (i) {
2057 ret = trace_seq_puts(s, " <= ");
2058 if (!ret)
2059 return TRACE_TYPE_PARTIAL_LINE;
2060 }
2061 ret = seq_print_ip_sym(s, field->caller[i],
2062 sym_flags);
2063 if (!ret)
2064 return TRACE_TYPE_PARTIAL_LINE;
2065 }
2066 ret = trace_seq_puts(s, "\n");
2067 if (!ret)
2068 return TRACE_TYPE_PARTIAL_LINE;
2069 break;
2070 }
2071 case TRACE_PRINT: {
2072 struct print_entry *field;
2073
2074 trace_assign_type(field, entry);
2075 1465
2076 seq_print_ip_sym(s, field->ip, sym_flags); 1466 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2077 trace_seq_printf(s, ": %s", field->buf); 1467 if (!trace_print_context(iter))
2078 if (entry->flags & TRACE_FLAG_CONT) 1468 goto partial;
2079 trace_seq_print_cont(s, iter);
2080 break;
2081 }
2082 case TRACE_GRAPH_RET: {
2083 return print_graph_function(iter);
2084 }
2085 case TRACE_GRAPH_ENT: {
2086 return print_graph_function(iter);
2087 } 1469 }
2088 case TRACE_BRANCH: {
2089 struct trace_branch *field;
2090 1470
2091 trace_assign_type(field, entry); 1471 if (event)
1472 return event->trace(iter, sym_flags);
2092 1473
2093 trace_seq_printf(s, "[%s] %s:%s:%d\n", 1474 if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
2094 field->correct ? " ok " : " MISS ", 1475 goto partial;
2095 field->func,
2096 field->file,
2097 field->line);
2098 break;
2099 }
2100 case TRACE_USER_STACK: {
2101 struct userstack_entry *field;
2102 1476
2103 trace_assign_type(field, entry);
2104
2105 ret = seq_print_userip_objs(field, s, sym_flags);
2106 if (!ret)
2107 return TRACE_TYPE_PARTIAL_LINE;
2108 ret = trace_seq_putc(s, '\n');
2109 if (!ret)
2110 return TRACE_TYPE_PARTIAL_LINE;
2111 break;
2112 }
2113 }
2114 return TRACE_TYPE_HANDLED; 1477 return TRACE_TYPE_HANDLED;
1478partial:
1479 return TRACE_TYPE_PARTIAL_LINE;
2115} 1480}
2116 1481
2117static enum print_line_t print_raw_fmt(struct trace_iterator *iter) 1482static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
2118{ 1483{
2119 struct trace_seq *s = &iter->seq; 1484 struct trace_seq *s = &iter->seq;
2120 struct trace_entry *entry; 1485 struct trace_entry *entry;
2121 int ret; 1486 struct trace_event *event;
2122 int S, T;
2123 1487
2124 entry = iter->ent; 1488 entry = iter->ent;
2125 1489
2126 if (entry->type == TRACE_CONT) 1490 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2127 return TRACE_TYPE_HANDLED; 1491 if (!trace_seq_printf(s, "%d %d %llu ",
2128 1492 entry->pid, iter->cpu, iter->ts))
2129 ret = trace_seq_printf(s, "%d %d %llu ", 1493 goto partial;
2130 entry->pid, iter->cpu, iter->ts);
2131 if (!ret)
2132 return TRACE_TYPE_PARTIAL_LINE;
2133
2134 switch (entry->type) {
2135 case TRACE_FN: {
2136 struct ftrace_entry *field;
2137
2138 trace_assign_type(field, entry);
2139
2140 ret = trace_seq_printf(s, "%x %x\n",
2141 field->ip,
2142 field->parent_ip);
2143 if (!ret)
2144 return TRACE_TYPE_PARTIAL_LINE;
2145 break;
2146 }
2147 case TRACE_CTX:
2148 case TRACE_WAKE: {
2149 struct ctx_switch_entry *field;
2150
2151 trace_assign_type(field, entry);
2152
2153 T = task_state_char(field->next_state);
2154 S = entry->type == TRACE_WAKE ? '+' :
2155 task_state_char(field->prev_state);
2156 ret = trace_seq_printf(s, "%d %d %c %d %d %d %c\n",
2157 field->prev_pid,
2158 field->prev_prio,
2159 S,
2160 field->next_cpu,
2161 field->next_pid,
2162 field->next_prio,
2163 T);
2164 if (!ret)
2165 return TRACE_TYPE_PARTIAL_LINE;
2166 break;
2167 } 1494 }
2168 case TRACE_SPECIAL:
2169 case TRACE_USER_STACK:
2170 case TRACE_STACK: {
2171 struct special_entry *field;
2172 1495
2173 trace_assign_type(field, entry); 1496 event = ftrace_find_event(entry->type);
1497 if (event)
1498 return event->raw(iter, 0);
2174 1499
2175 ret = trace_seq_printf(s, "# %ld %ld %ld\n", 1500 if (!trace_seq_printf(s, "%d ?\n", entry->type))
2176 field->arg1, 1501 goto partial;
2177 field->arg2,
2178 field->arg3);
2179 if (!ret)
2180 return TRACE_TYPE_PARTIAL_LINE;
2181 break;
2182 }
2183 case TRACE_PRINT: {
2184 struct print_entry *field;
2185 1502
2186 trace_assign_type(field, entry);
2187
2188 trace_seq_printf(s, "# %lx %s", field->ip, field->buf);
2189 if (entry->flags & TRACE_FLAG_CONT)
2190 trace_seq_print_cont(s, iter);
2191 break;
2192 }
2193 }
2194 return TRACE_TYPE_HANDLED; 1503 return TRACE_TYPE_HANDLED;
1504partial:
1505 return TRACE_TYPE_PARTIAL_LINE;
2195} 1506}
2196 1507
2197#define SEQ_PUT_FIELD_RET(s, x) \
2198do { \
2199 if (!trace_seq_putmem(s, &(x), sizeof(x))) \
2200 return 0; \
2201} while (0)
2202
2203#define SEQ_PUT_HEX_FIELD_RET(s, x) \
2204do { \
2205 BUILD_BUG_ON(sizeof(x) > MAX_MEMHEX_BYTES); \
2206 if (!trace_seq_putmem_hex(s, &(x), sizeof(x))) \
2207 return 0; \
2208} while (0)
2209
2210static enum print_line_t print_hex_fmt(struct trace_iterator *iter) 1508static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
2211{ 1509{
2212 struct trace_seq *s = &iter->seq; 1510 struct trace_seq *s = &iter->seq;
2213 unsigned char newline = '\n'; 1511 unsigned char newline = '\n';
2214 struct trace_entry *entry; 1512 struct trace_entry *entry;
2215 int S, T; 1513 struct trace_event *event;
2216 1514
2217 entry = iter->ent; 1515 entry = iter->ent;
2218 1516
2219 if (entry->type == TRACE_CONT) 1517 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2220 return TRACE_TYPE_HANDLED; 1518 SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
2221 1519 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
2222 SEQ_PUT_HEX_FIELD_RET(s, entry->pid); 1520 SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
2223 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
2224 SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
2225
2226 switch (entry->type) {
2227 case TRACE_FN: {
2228 struct ftrace_entry *field;
2229
2230 trace_assign_type(field, entry);
2231
2232 SEQ_PUT_HEX_FIELD_RET(s, field->ip);
2233 SEQ_PUT_HEX_FIELD_RET(s, field->parent_ip);
2234 break;
2235 }
2236 case TRACE_CTX:
2237 case TRACE_WAKE: {
2238 struct ctx_switch_entry *field;
2239
2240 trace_assign_type(field, entry);
2241
2242 T = task_state_char(field->next_state);
2243 S = entry->type == TRACE_WAKE ? '+' :
2244 task_state_char(field->prev_state);
2245 SEQ_PUT_HEX_FIELD_RET(s, field->prev_pid);
2246 SEQ_PUT_HEX_FIELD_RET(s, field->prev_prio);
2247 SEQ_PUT_HEX_FIELD_RET(s, S);
2248 SEQ_PUT_HEX_FIELD_RET(s, field->next_cpu);
2249 SEQ_PUT_HEX_FIELD_RET(s, field->next_pid);
2250 SEQ_PUT_HEX_FIELD_RET(s, field->next_prio);
2251 SEQ_PUT_HEX_FIELD_RET(s, T);
2252 break;
2253 } 1521 }
2254 case TRACE_SPECIAL:
2255 case TRACE_USER_STACK:
2256 case TRACE_STACK: {
2257 struct special_entry *field;
2258 1522
2259 trace_assign_type(field, entry); 1523 event = ftrace_find_event(entry->type);
2260 1524 if (event) {
2261 SEQ_PUT_HEX_FIELD_RET(s, field->arg1); 1525 enum print_line_t ret = event->hex(iter, 0);
2262 SEQ_PUT_HEX_FIELD_RET(s, field->arg2); 1526 if (ret != TRACE_TYPE_HANDLED)
2263 SEQ_PUT_HEX_FIELD_RET(s, field->arg3); 1527 return ret;
2264 break;
2265 }
2266 } 1528 }
1529
2267 SEQ_PUT_FIELD_RET(s, newline); 1530 SEQ_PUT_FIELD_RET(s, newline);
2268 1531
2269 return TRACE_TYPE_HANDLED; 1532 return TRACE_TYPE_HANDLED;
@@ -2278,13 +1541,10 @@ static enum print_line_t print_printk_msg_only(struct trace_iterator *iter)
2278 1541
2279 trace_assign_type(field, entry); 1542 trace_assign_type(field, entry);
2280 1543
2281 ret = trace_seq_printf(s, field->buf); 1544 ret = trace_seq_printf(s, "%s", field->buf);
2282 if (!ret) 1545 if (!ret)
2283 return TRACE_TYPE_PARTIAL_LINE; 1546 return TRACE_TYPE_PARTIAL_LINE;
2284 1547
2285 if (entry->flags & TRACE_FLAG_CONT)
2286 trace_seq_print_cont(s, iter);
2287
2288 return TRACE_TYPE_HANDLED; 1548 return TRACE_TYPE_HANDLED;
2289} 1549}
2290 1550
@@ -2292,53 +1552,18 @@ static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
2292{ 1552{
2293 struct trace_seq *s = &iter->seq; 1553 struct trace_seq *s = &iter->seq;
2294 struct trace_entry *entry; 1554 struct trace_entry *entry;
1555 struct trace_event *event;
2295 1556
2296 entry = iter->ent; 1557 entry = iter->ent;
2297 1558
2298 if (entry->type == TRACE_CONT) 1559 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2299 return TRACE_TYPE_HANDLED; 1560 SEQ_PUT_FIELD_RET(s, entry->pid);
2300 1561 SEQ_PUT_FIELD_RET(s, iter->cpu);
2301 SEQ_PUT_FIELD_RET(s, entry->pid); 1562 SEQ_PUT_FIELD_RET(s, iter->ts);
2302 SEQ_PUT_FIELD_RET(s, entry->cpu);
2303 SEQ_PUT_FIELD_RET(s, iter->ts);
2304
2305 switch (entry->type) {
2306 case TRACE_FN: {
2307 struct ftrace_entry *field;
2308
2309 trace_assign_type(field, entry);
2310
2311 SEQ_PUT_FIELD_RET(s, field->ip);
2312 SEQ_PUT_FIELD_RET(s, field->parent_ip);
2313 break;
2314 } 1563 }
2315 case TRACE_CTX: {
2316 struct ctx_switch_entry *field;
2317
2318 trace_assign_type(field, entry);
2319 1564
2320 SEQ_PUT_FIELD_RET(s, field->prev_pid); 1565 event = ftrace_find_event(entry->type);
2321 SEQ_PUT_FIELD_RET(s, field->prev_prio); 1566 return event ? event->binary(iter, 0) : TRACE_TYPE_HANDLED;
2322 SEQ_PUT_FIELD_RET(s, field->prev_state);
2323 SEQ_PUT_FIELD_RET(s, field->next_pid);
2324 SEQ_PUT_FIELD_RET(s, field->next_prio);
2325 SEQ_PUT_FIELD_RET(s, field->next_state);
2326 break;
2327 }
2328 case TRACE_SPECIAL:
2329 case TRACE_USER_STACK:
2330 case TRACE_STACK: {
2331 struct special_entry *field;
2332
2333 trace_assign_type(field, entry);
2334
2335 SEQ_PUT_FIELD_RET(s, field->arg1);
2336 SEQ_PUT_FIELD_RET(s, field->arg2);
2337 SEQ_PUT_FIELD_RET(s, field->arg3);
2338 break;
2339 }
2340 }
2341 return 1;
2342} 1567}
2343 1568
2344static int trace_empty(struct trace_iterator *iter) 1569static int trace_empty(struct trace_iterator *iter)
@@ -2383,7 +1608,7 @@ static enum print_line_t print_trace_line(struct trace_iterator *iter)
2383 return print_raw_fmt(iter); 1608 return print_raw_fmt(iter);
2384 1609
2385 if (iter->iter_flags & TRACE_FILE_LAT_FMT) 1610 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2386 return print_lat_fmt(iter, iter->idx, iter->cpu); 1611 return print_lat_fmt(iter);
2387 1612
2388 return print_trace_fmt(iter); 1613 return print_trace_fmt(iter);
2389} 1614}
@@ -2505,7 +1730,7 @@ int tracing_open_generic(struct inode *inode, struct file *filp)
2505 return 0; 1730 return 0;
2506} 1731}
2507 1732
2508int tracing_release(struct inode *inode, struct file *file) 1733static int tracing_release(struct inode *inode, struct file *file)
2509{ 1734{
2510 struct seq_file *m = (struct seq_file *)file->private_data; 1735 struct seq_file *m = (struct seq_file *)file->private_data;
2511 struct trace_iterator *iter = m->private; 1736 struct trace_iterator *iter = m->private;
@@ -2748,7 +1973,7 @@ tracing_trace_options_read(struct file *filp, char __user *ubuf,
2748 struct tracer_opt *trace_opts = current_trace->flags->opts; 1973 struct tracer_opt *trace_opts = current_trace->flags->opts;
2749 1974
2750 1975
2751 /* calulate max size */ 1976 /* calculate max size */
2752 for (i = 0; trace_options[i]; i++) { 1977 for (i = 0; trace_options[i]; i++) {
2753 len += strlen(trace_options[i]); 1978 len += strlen(trace_options[i]);
2754 len += 3; /* "no" and space */ 1979 len += 3; /* "no" and space */
@@ -2930,7 +2155,7 @@ tracing_ctrl_write(struct file *filp, const char __user *ubuf,
2930{ 2155{
2931 struct trace_array *tr = filp->private_data; 2156 struct trace_array *tr = filp->private_data;
2932 char buf[64]; 2157 char buf[64];
2933 long val; 2158 unsigned long val;
2934 int ret; 2159 int ret;
2935 2160
2936 if (cnt >= sizeof(buf)) 2161 if (cnt >= sizeof(buf))
@@ -2985,7 +2210,13 @@ tracing_set_trace_read(struct file *filp, char __user *ubuf,
2985 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 2210 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2986} 2211}
2987 2212
2988static int tracing_set_tracer(char *buf) 2213int tracer_init(struct tracer *t, struct trace_array *tr)
2214{
2215 tracing_reset_online_cpus(tr);
2216 return t->init(tr);
2217}
2218
2219static int tracing_set_tracer(const char *buf)
2989{ 2220{
2990 struct trace_array *tr = &global_trace; 2221 struct trace_array *tr = &global_trace;
2991 struct tracer *t; 2222 struct tracer *t;
@@ -3009,7 +2240,7 @@ static int tracing_set_tracer(char *buf)
3009 2240
3010 current_trace = t; 2241 current_trace = t;
3011 if (t->init) { 2242 if (t->init) {
3012 ret = t->init(tr); 2243 ret = tracer_init(t, tr);
3013 if (ret) 2244 if (ret)
3014 goto out; 2245 goto out;
3015 } 2246 }
@@ -3072,9 +2303,9 @@ static ssize_t
3072tracing_max_lat_write(struct file *filp, const char __user *ubuf, 2303tracing_max_lat_write(struct file *filp, const char __user *ubuf,
3073 size_t cnt, loff_t *ppos) 2304 size_t cnt, loff_t *ppos)
3074{ 2305{
3075 long *ptr = filp->private_data; 2306 unsigned long *ptr = filp->private_data;
3076 char buf[64]; 2307 char buf[64];
3077 long val; 2308 unsigned long val;
3078 int ret; 2309 int ret;
3079 2310
3080 if (cnt >= sizeof(buf)) 2311 if (cnt >= sizeof(buf))
@@ -3167,67 +2398,60 @@ tracing_poll_pipe(struct file *filp, poll_table *poll_table)
3167 } 2398 }
3168} 2399}
3169 2400
3170/* 2401
3171 * Consumer reader. 2402void default_wait_pipe(struct trace_iterator *iter)
3172 */
3173static ssize_t
3174tracing_read_pipe(struct file *filp, char __user *ubuf,
3175 size_t cnt, loff_t *ppos)
3176{ 2403{
3177 struct trace_iterator *iter = filp->private_data; 2404 DEFINE_WAIT(wait);
3178 ssize_t sret;
3179 2405
3180 /* return any leftover data */ 2406 prepare_to_wait(&trace_wait, &wait, TASK_INTERRUPTIBLE);
3181 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3182 if (sret != -EBUSY)
3183 return sret;
3184 2407
3185 trace_seq_reset(&iter->seq); 2408 if (trace_empty(iter))
2409 schedule();
3186 2410
3187 mutex_lock(&trace_types_lock); 2411 finish_wait(&trace_wait, &wait);
3188 if (iter->trace->read) { 2412}
3189 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos); 2413
3190 if (sret) 2414/*
3191 goto out; 2415 * This is a make-shift waitqueue.
3192 } 2416 * A tracer might use this callback on some rare cases:
2417 *
2418 * 1) the current tracer might hold the runqueue lock when it wakes up
2419 * a reader, hence a deadlock (sched, function, and function graph tracers)
2420 * 2) the function tracers, trace all functions, we don't want
2421 * the overhead of calling wake_up and friends
2422 * (and tracing them too)
2423 *
2424 * Anyway, this is really very primitive wakeup.
2425 */
2426void poll_wait_pipe(struct trace_iterator *iter)
2427{
2428 set_current_state(TASK_INTERRUPTIBLE);
2429 /* sleep for 100 msecs, and try again. */
2430 schedule_timeout(HZ / 10);
2431}
2432
2433/* Must be called with trace_types_lock mutex held. */
2434static int tracing_wait_pipe(struct file *filp)
2435{
2436 struct trace_iterator *iter = filp->private_data;
3193 2437
3194waitagain:
3195 sret = 0;
3196 while (trace_empty(iter)) { 2438 while (trace_empty(iter)) {
3197 2439
3198 if ((filp->f_flags & O_NONBLOCK)) { 2440 if ((filp->f_flags & O_NONBLOCK)) {
3199 sret = -EAGAIN; 2441 return -EAGAIN;
3200 goto out;
3201 } 2442 }
3202 2443
3203 /*
3204 * This is a make-shift waitqueue. The reason we don't use
3205 * an actual wait queue is because:
3206 * 1) we only ever have one waiter
3207 * 2) the tracing, traces all functions, we don't want
3208 * the overhead of calling wake_up and friends
3209 * (and tracing them too)
3210 * Anyway, this is really very primitive wakeup.
3211 */
3212 set_current_state(TASK_INTERRUPTIBLE);
3213 iter->tr->waiter = current;
3214
3215 mutex_unlock(&trace_types_lock); 2444 mutex_unlock(&trace_types_lock);
3216 2445
3217 /* sleep for 100 msecs, and try again. */ 2446 iter->trace->wait_pipe(iter);
3218 schedule_timeout(HZ/10);
3219 2447
3220 mutex_lock(&trace_types_lock); 2448 mutex_lock(&trace_types_lock);
3221 2449
3222 iter->tr->waiter = NULL; 2450 if (signal_pending(current))
3223 2451 return -EINTR;
3224 if (signal_pending(current)) {
3225 sret = -EINTR;
3226 goto out;
3227 }
3228 2452
3229 if (iter->trace != current_trace) 2453 if (iter->trace != current_trace)
3230 goto out; 2454 return 0;
3231 2455
3232 /* 2456 /*
3233 * We block until we read something and tracing is disabled. 2457 * We block until we read something and tracing is disabled.
@@ -3240,13 +2464,45 @@ waitagain:
3240 */ 2464 */
3241 if (!tracer_enabled && iter->pos) 2465 if (!tracer_enabled && iter->pos)
3242 break; 2466 break;
2467 }
2468
2469 return 1;
2470}
2471
2472/*
2473 * Consumer reader.
2474 */
2475static ssize_t
2476tracing_read_pipe(struct file *filp, char __user *ubuf,
2477 size_t cnt, loff_t *ppos)
2478{
2479 struct trace_iterator *iter = filp->private_data;
2480 ssize_t sret;
3243 2481
3244 continue; 2482 /* return any leftover data */
2483 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
2484 if (sret != -EBUSY)
2485 return sret;
2486
2487 trace_seq_reset(&iter->seq);
2488
2489 mutex_lock(&trace_types_lock);
2490 if (iter->trace->read) {
2491 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
2492 if (sret)
2493 goto out;
3245 } 2494 }
3246 2495
2496waitagain:
2497 sret = tracing_wait_pipe(filp);
2498 if (sret <= 0)
2499 goto out;
2500
3247 /* stop when tracing is finished */ 2501 /* stop when tracing is finished */
3248 if (trace_empty(iter)) 2502 if (trace_empty(iter)) {
2503 sret = 0;
3249 goto out; 2504 goto out;
2505 }
3250 2506
3251 if (cnt >= PAGE_SIZE) 2507 if (cnt >= PAGE_SIZE)
3252 cnt = PAGE_SIZE - 1; 2508 cnt = PAGE_SIZE - 1;
@@ -3267,8 +2523,8 @@ waitagain:
3267 iter->seq.len = len; 2523 iter->seq.len = len;
3268 break; 2524 break;
3269 } 2525 }
3270 2526 if (ret != TRACE_TYPE_NO_CONSUME)
3271 trace_consume(iter); 2527 trace_consume(iter);
3272 2528
3273 if (iter->seq.len >= cnt) 2529 if (iter->seq.len >= cnt)
3274 break; 2530 break;
@@ -3292,6 +2548,134 @@ out:
3292 return sret; 2548 return sret;
3293} 2549}
3294 2550
2551static void tracing_pipe_buf_release(struct pipe_inode_info *pipe,
2552 struct pipe_buffer *buf)
2553{
2554 __free_page(buf->page);
2555}
2556
2557static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
2558 unsigned int idx)
2559{
2560 __free_page(spd->pages[idx]);
2561}
2562
2563static struct pipe_buf_operations tracing_pipe_buf_ops = {
2564 .can_merge = 0,
2565 .map = generic_pipe_buf_map,
2566 .unmap = generic_pipe_buf_unmap,
2567 .confirm = generic_pipe_buf_confirm,
2568 .release = tracing_pipe_buf_release,
2569 .steal = generic_pipe_buf_steal,
2570 .get = generic_pipe_buf_get,
2571};
2572
2573static size_t
2574tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
2575{
2576 size_t count;
2577 int ret;
2578
2579 /* Seq buffer is page-sized, exactly what we need. */
2580 for (;;) {
2581 count = iter->seq.len;
2582 ret = print_trace_line(iter);
2583 count = iter->seq.len - count;
2584 if (rem < count) {
2585 rem = 0;
2586 iter->seq.len -= count;
2587 break;
2588 }
2589 if (ret == TRACE_TYPE_PARTIAL_LINE) {
2590 iter->seq.len -= count;
2591 break;
2592 }
2593
2594 trace_consume(iter);
2595 rem -= count;
2596 if (!find_next_entry_inc(iter)) {
2597 rem = 0;
2598 iter->ent = NULL;
2599 break;
2600 }
2601 }
2602
2603 return rem;
2604}
2605
2606static ssize_t tracing_splice_read_pipe(struct file *filp,
2607 loff_t *ppos,
2608 struct pipe_inode_info *pipe,
2609 size_t len,
2610 unsigned int flags)
2611{
2612 struct page *pages[PIPE_BUFFERS];
2613 struct partial_page partial[PIPE_BUFFERS];
2614 struct trace_iterator *iter = filp->private_data;
2615 struct splice_pipe_desc spd = {
2616 .pages = pages,
2617 .partial = partial,
2618 .nr_pages = 0, /* This gets updated below. */
2619 .flags = flags,
2620 .ops = &tracing_pipe_buf_ops,
2621 .spd_release = tracing_spd_release_pipe,
2622 };
2623 ssize_t ret;
2624 size_t rem;
2625 unsigned int i;
2626
2627 mutex_lock(&trace_types_lock);
2628
2629 if (iter->trace->splice_read) {
2630 ret = iter->trace->splice_read(iter, filp,
2631 ppos, pipe, len, flags);
2632 if (ret)
2633 goto out_err;
2634 }
2635
2636 ret = tracing_wait_pipe(filp);
2637 if (ret <= 0)
2638 goto out_err;
2639
2640 if (!iter->ent && !find_next_entry_inc(iter)) {
2641 ret = -EFAULT;
2642 goto out_err;
2643 }
2644
2645 /* Fill as many pages as possible. */
2646 for (i = 0, rem = len; i < PIPE_BUFFERS && rem; i++) {
2647 pages[i] = alloc_page(GFP_KERNEL);
2648 if (!pages[i])
2649 break;
2650
2651 rem = tracing_fill_pipe_page(rem, iter);
2652
2653 /* Copy the data into the page, so we can start over. */
2654 ret = trace_seq_to_buffer(&iter->seq,
2655 page_address(pages[i]),
2656 iter->seq.len);
2657 if (ret < 0) {
2658 __free_page(pages[i]);
2659 break;
2660 }
2661 partial[i].offset = 0;
2662 partial[i].len = iter->seq.len;
2663
2664 trace_seq_reset(&iter->seq);
2665 }
2666
2667 mutex_unlock(&trace_types_lock);
2668
2669 spd.nr_pages = i;
2670
2671 return splice_to_pipe(pipe, &spd);
2672
2673out_err:
2674 mutex_unlock(&trace_types_lock);
2675
2676 return ret;
2677}
2678
3295static ssize_t 2679static ssize_t
3296tracing_entries_read(struct file *filp, char __user *ubuf, 2680tracing_entries_read(struct file *filp, char __user *ubuf,
3297 size_t cnt, loff_t *ppos) 2681 size_t cnt, loff_t *ppos)
@@ -3455,6 +2839,7 @@ static struct file_operations tracing_pipe_fops = {
3455 .open = tracing_open_pipe, 2839 .open = tracing_open_pipe,
3456 .poll = tracing_poll_pipe, 2840 .poll = tracing_poll_pipe,
3457 .read = tracing_read_pipe, 2841 .read = tracing_read_pipe,
2842 .splice_read = tracing_splice_read_pipe,
3458 .release = tracing_release_pipe, 2843 .release = tracing_release_pipe,
3459}; 2844};
3460 2845
@@ -3653,18 +3038,16 @@ int trace_vprintk(unsigned long ip, int depth, const char *fmt, va_list args)
3653 trace_buf[len] = 0; 3038 trace_buf[len] = 0;
3654 3039
3655 size = sizeof(*entry) + len + 1; 3040 size = sizeof(*entry) + len + 1;
3656 event = ring_buffer_lock_reserve(tr->buffer, size, &irq_flags); 3041 event = trace_buffer_lock_reserve(tr, TRACE_PRINT, size, irq_flags, pc);
3657 if (!event) 3042 if (!event)
3658 goto out_unlock; 3043 goto out_unlock;
3659 entry = ring_buffer_event_data(event); 3044 entry = ring_buffer_event_data(event);
3660 tracing_generic_entry_update(&entry->ent, irq_flags, pc);
3661 entry->ent.type = TRACE_PRINT;
3662 entry->ip = ip; 3045 entry->ip = ip;
3663 entry->depth = depth; 3046 entry->depth = depth;
3664 3047
3665 memcpy(&entry->buf, trace_buf, len); 3048 memcpy(&entry->buf, trace_buf, len);
3666 entry->buf[len] = 0; 3049 entry->buf[len] = 0;
3667 ring_buffer_unlock_commit(tr->buffer, event, irq_flags); 3050 ring_buffer_unlock_commit(tr->buffer, event);
3668 3051
3669 out_unlock: 3052 out_unlock:
3670 spin_unlock_irqrestore(&trace_buf_lock, irq_flags); 3053 spin_unlock_irqrestore(&trace_buf_lock, irq_flags);
@@ -3691,6 +3074,15 @@ int __ftrace_printk(unsigned long ip, const char *fmt, ...)
3691} 3074}
3692EXPORT_SYMBOL_GPL(__ftrace_printk); 3075EXPORT_SYMBOL_GPL(__ftrace_printk);
3693 3076
3077int __ftrace_vprintk(unsigned long ip, const char *fmt, va_list ap)
3078{
3079 if (!(trace_flags & TRACE_ITER_PRINTK))
3080 return 0;
3081
3082 return trace_vprintk(ip, task_curr_ret_stack(current), fmt, ap);
3083}
3084EXPORT_SYMBOL_GPL(__ftrace_vprintk);
3085
3694static int trace_panic_handler(struct notifier_block *this, 3086static int trace_panic_handler(struct notifier_block *this,
3695 unsigned long event, void *unused) 3087 unsigned long event, void *unused)
3696{ 3088{
@@ -3871,14 +3263,10 @@ __init static int tracer_alloc_buffers(void)
3871 trace_init_cmdlines(); 3263 trace_init_cmdlines();
3872 3264
3873 register_tracer(&nop_trace); 3265 register_tracer(&nop_trace);
3266 current_trace = &nop_trace;
3874#ifdef CONFIG_BOOT_TRACER 3267#ifdef CONFIG_BOOT_TRACER
3875 register_tracer(&boot_tracer); 3268 register_tracer(&boot_tracer);
3876 current_trace = &boot_tracer;
3877 current_trace->init(&global_trace);
3878#else
3879 current_trace = &nop_trace;
3880#endif 3269#endif
3881
3882 /* All seems OK, enable tracing */ 3270 /* All seems OK, enable tracing */
3883 tracing_disabled = 0; 3271 tracing_disabled = 0;
3884 3272
@@ -3895,5 +3283,26 @@ out_free_buffer_mask:
3895out: 3283out:
3896 return ret; 3284 return ret;
3897} 3285}
3286
3287__init static int clear_boot_tracer(void)
3288{
3289 /*
3290 * The default tracer at boot buffer is an init section.
3291 * This function is called in lateinit. If we did not
3292 * find the boot tracer, then clear it out, to prevent
3293 * later registration from accessing the buffer that is
3294 * about to be freed.
3295 */
3296 if (!default_bootup_tracer)
3297 return 0;
3298
3299 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
3300 default_bootup_tracer);
3301 default_bootup_tracer = NULL;
3302
3303 return 0;
3304}
3305
3898early_initcall(tracer_alloc_buffers); 3306early_initcall(tracer_alloc_buffers);
3899fs_initcall(tracer_init_debugfs); 3307fs_initcall(tracer_init_debugfs);
3308late_initcall(clear_boot_tracer);
diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h
index 4d3d381bfd95..eed732c151fc 100644
--- a/kernel/trace/trace.h
+++ b/kernel/trace/trace.h
@@ -9,6 +9,8 @@
9#include <linux/mmiotrace.h> 9#include <linux/mmiotrace.h>
10#include <linux/ftrace.h> 10#include <linux/ftrace.h>
11#include <trace/boot.h> 11#include <trace/boot.h>
12#include <trace/kmemtrace.h>
13#include <trace/power.h>
12 14
13enum trace_type { 15enum trace_type {
14 __TRACE_FIRST_TYPE = 0, 16 __TRACE_FIRST_TYPE = 0,
@@ -16,7 +18,6 @@ enum trace_type {
16 TRACE_FN, 18 TRACE_FN,
17 TRACE_CTX, 19 TRACE_CTX,
18 TRACE_WAKE, 20 TRACE_WAKE,
19 TRACE_CONT,
20 TRACE_STACK, 21 TRACE_STACK,
21 TRACE_PRINT, 22 TRACE_PRINT,
22 TRACE_SPECIAL, 23 TRACE_SPECIAL,
@@ -29,9 +30,12 @@ enum trace_type {
29 TRACE_GRAPH_ENT, 30 TRACE_GRAPH_ENT,
30 TRACE_USER_STACK, 31 TRACE_USER_STACK,
31 TRACE_HW_BRANCHES, 32 TRACE_HW_BRANCHES,
33 TRACE_KMEM_ALLOC,
34 TRACE_KMEM_FREE,
32 TRACE_POWER, 35 TRACE_POWER,
36 TRACE_BLK,
33 37
34 __TRACE_LAST_TYPE 38 __TRACE_LAST_TYPE,
35}; 39};
36 40
37/* 41/*
@@ -42,7 +46,6 @@ enum trace_type {
42 */ 46 */
43struct trace_entry { 47struct trace_entry {
44 unsigned char type; 48 unsigned char type;
45 unsigned char cpu;
46 unsigned char flags; 49 unsigned char flags;
47 unsigned char preempt_count; 50 unsigned char preempt_count;
48 int pid; 51 int pid;
@@ -60,13 +63,13 @@ struct ftrace_entry {
60 63
61/* Function call entry */ 64/* Function call entry */
62struct ftrace_graph_ent_entry { 65struct ftrace_graph_ent_entry {
63 struct trace_entry ent; 66 struct trace_entry ent;
64 struct ftrace_graph_ent graph_ent; 67 struct ftrace_graph_ent graph_ent;
65}; 68};
66 69
67/* Function return entry */ 70/* Function return entry */
68struct ftrace_graph_ret_entry { 71struct ftrace_graph_ret_entry {
69 struct trace_entry ent; 72 struct trace_entry ent;
70 struct ftrace_graph_ret ret; 73 struct ftrace_graph_ret ret;
71}; 74};
72extern struct tracer boot_tracer; 75extern struct tracer boot_tracer;
@@ -170,6 +173,24 @@ struct trace_power {
170 struct power_trace state_data; 173 struct power_trace state_data;
171}; 174};
172 175
176struct kmemtrace_alloc_entry {
177 struct trace_entry ent;
178 enum kmemtrace_type_id type_id;
179 unsigned long call_site;
180 const void *ptr;
181 size_t bytes_req;
182 size_t bytes_alloc;
183 gfp_t gfp_flags;
184 int node;
185};
186
187struct kmemtrace_free_entry {
188 struct trace_entry ent;
189 enum kmemtrace_type_id type_id;
190 unsigned long call_site;
191 const void *ptr;
192};
193
173/* 194/*
174 * trace_flag_type is an enumeration that holds different 195 * trace_flag_type is an enumeration that holds different
175 * states when a trace occurs. These are: 196 * states when a trace occurs. These are:
@@ -178,7 +199,6 @@ struct trace_power {
178 * NEED_RESCED - reschedule is requested 199 * NEED_RESCED - reschedule is requested
179 * HARDIRQ - inside an interrupt handler 200 * HARDIRQ - inside an interrupt handler
180 * SOFTIRQ - inside a softirq handler 201 * SOFTIRQ - inside a softirq handler
181 * CONT - multiple entries hold the trace item
182 */ 202 */
183enum trace_flag_type { 203enum trace_flag_type {
184 TRACE_FLAG_IRQS_OFF = 0x01, 204 TRACE_FLAG_IRQS_OFF = 0x01,
@@ -186,7 +206,6 @@ enum trace_flag_type {
186 TRACE_FLAG_NEED_RESCHED = 0x04, 206 TRACE_FLAG_NEED_RESCHED = 0x04,
187 TRACE_FLAG_HARDIRQ = 0x08, 207 TRACE_FLAG_HARDIRQ = 0x08,
188 TRACE_FLAG_SOFTIRQ = 0x10, 208 TRACE_FLAG_SOFTIRQ = 0x10,
189 TRACE_FLAG_CONT = 0x20,
190}; 209};
191 210
192#define TRACE_BUF_SIZE 1024 211#define TRACE_BUF_SIZE 1024
@@ -262,7 +281,6 @@ extern void __ftrace_bad_type(void);
262 do { \ 281 do { \
263 IF_ASSIGN(var, ent, struct ftrace_entry, TRACE_FN); \ 282 IF_ASSIGN(var, ent, struct ftrace_entry, TRACE_FN); \
264 IF_ASSIGN(var, ent, struct ctx_switch_entry, 0); \ 283 IF_ASSIGN(var, ent, struct ctx_switch_entry, 0); \
265 IF_ASSIGN(var, ent, struct trace_field_cont, TRACE_CONT); \
266 IF_ASSIGN(var, ent, struct stack_entry, TRACE_STACK); \ 284 IF_ASSIGN(var, ent, struct stack_entry, TRACE_STACK); \
267 IF_ASSIGN(var, ent, struct userstack_entry, TRACE_USER_STACK);\ 285 IF_ASSIGN(var, ent, struct userstack_entry, TRACE_USER_STACK);\
268 IF_ASSIGN(var, ent, struct print_entry, TRACE_PRINT); \ 286 IF_ASSIGN(var, ent, struct print_entry, TRACE_PRINT); \
@@ -280,6 +298,10 @@ extern void __ftrace_bad_type(void);
280 TRACE_GRAPH_RET); \ 298 TRACE_GRAPH_RET); \
281 IF_ASSIGN(var, ent, struct hw_branch_entry, TRACE_HW_BRANCHES);\ 299 IF_ASSIGN(var, ent, struct hw_branch_entry, TRACE_HW_BRANCHES);\
282 IF_ASSIGN(var, ent, struct trace_power, TRACE_POWER); \ 300 IF_ASSIGN(var, ent, struct trace_power, TRACE_POWER); \
301 IF_ASSIGN(var, ent, struct kmemtrace_alloc_entry, \
302 TRACE_KMEM_ALLOC); \
303 IF_ASSIGN(var, ent, struct kmemtrace_free_entry, \
304 TRACE_KMEM_FREE); \
283 __ftrace_bad_type(); \ 305 __ftrace_bad_type(); \
284 } while (0) 306 } while (0)
285 307
@@ -287,7 +309,8 @@ extern void __ftrace_bad_type(void);
287enum print_line_t { 309enum print_line_t {
288 TRACE_TYPE_PARTIAL_LINE = 0, /* Retry after flushing the seq */ 310 TRACE_TYPE_PARTIAL_LINE = 0, /* Retry after flushing the seq */
289 TRACE_TYPE_HANDLED = 1, 311 TRACE_TYPE_HANDLED = 1,
290 TRACE_TYPE_UNHANDLED = 2 /* Relay to other output functions */ 312 TRACE_TYPE_UNHANDLED = 2, /* Relay to other output functions */
313 TRACE_TYPE_NO_CONSUME = 3 /* Handled but ask to not consume */
291}; 314};
292 315
293 316
@@ -313,22 +336,45 @@ struct tracer_flags {
313/* Makes more easy to define a tracer opt */ 336/* Makes more easy to define a tracer opt */
314#define TRACER_OPT(s, b) .name = #s, .bit = b 337#define TRACER_OPT(s, b) .name = #s, .bit = b
315 338
316/* 339
317 * A specific tracer, represented by methods that operate on a trace array: 340/**
341 * struct tracer - a specific tracer and its callbacks to interact with debugfs
342 * @name: the name chosen to select it on the available_tracers file
343 * @init: called when one switches to this tracer (echo name > current_tracer)
344 * @reset: called when one switches to another tracer
345 * @start: called when tracing is unpaused (echo 1 > tracing_enabled)
346 * @stop: called when tracing is paused (echo 0 > tracing_enabled)
347 * @open: called when the trace file is opened
348 * @pipe_open: called when the trace_pipe file is opened
349 * @wait_pipe: override how the user waits for traces on trace_pipe
350 * @close: called when the trace file is released
351 * @read: override the default read callback on trace_pipe
352 * @splice_read: override the default splice_read callback on trace_pipe
353 * @selftest: selftest to run on boot (see trace_selftest.c)
354 * @print_headers: override the first lines that describe your columns
355 * @print_line: callback that prints a trace
356 * @set_flag: signals one of your private flags changed (trace_options file)
357 * @flags: your private flags
318 */ 358 */
319struct tracer { 359struct tracer {
320 const char *name; 360 const char *name;
321 /* Your tracer should raise a warning if init fails */
322 int (*init)(struct trace_array *tr); 361 int (*init)(struct trace_array *tr);
323 void (*reset)(struct trace_array *tr); 362 void (*reset)(struct trace_array *tr);
324 void (*start)(struct trace_array *tr); 363 void (*start)(struct trace_array *tr);
325 void (*stop)(struct trace_array *tr); 364 void (*stop)(struct trace_array *tr);
326 void (*open)(struct trace_iterator *iter); 365 void (*open)(struct trace_iterator *iter);
327 void (*pipe_open)(struct trace_iterator *iter); 366 void (*pipe_open)(struct trace_iterator *iter);
367 void (*wait_pipe)(struct trace_iterator *iter);
328 void (*close)(struct trace_iterator *iter); 368 void (*close)(struct trace_iterator *iter);
329 ssize_t (*read)(struct trace_iterator *iter, 369 ssize_t (*read)(struct trace_iterator *iter,
330 struct file *filp, char __user *ubuf, 370 struct file *filp, char __user *ubuf,
331 size_t cnt, loff_t *ppos); 371 size_t cnt, loff_t *ppos);
372 ssize_t (*splice_read)(struct trace_iterator *iter,
373 struct file *filp,
374 loff_t *ppos,
375 struct pipe_inode_info *pipe,
376 size_t len,
377 unsigned int flags);
332#ifdef CONFIG_FTRACE_STARTUP_TEST 378#ifdef CONFIG_FTRACE_STARTUP_TEST
333 int (*selftest)(struct tracer *trace, 379 int (*selftest)(struct tracer *trace,
334 struct trace_array *tr); 380 struct trace_array *tr);
@@ -340,6 +386,7 @@ struct tracer {
340 struct tracer *next; 386 struct tracer *next;
341 int print_max; 387 int print_max;
342 struct tracer_flags *flags; 388 struct tracer_flags *flags;
389 struct tracer_stat *stats;
343}; 390};
344 391
345struct trace_seq { 392struct trace_seq {
@@ -371,6 +418,7 @@ struct trace_iterator {
371 cpumask_var_t started; 418 cpumask_var_t started;
372}; 419};
373 420
421int tracer_init(struct tracer *t, struct trace_array *tr);
374int tracing_is_enabled(void); 422int tracing_is_enabled(void);
375void trace_wake_up(void); 423void trace_wake_up(void);
376void tracing_reset(struct trace_array *tr, int cpu); 424void tracing_reset(struct trace_array *tr, int cpu);
@@ -379,26 +427,42 @@ int tracing_open_generic(struct inode *inode, struct file *filp);
379struct dentry *tracing_init_dentry(void); 427struct dentry *tracing_init_dentry(void);
380void init_tracer_sysprof_debugfs(struct dentry *d_tracer); 428void init_tracer_sysprof_debugfs(struct dentry *d_tracer);
381 429
430struct ring_buffer_event;
431
432struct ring_buffer_event *trace_buffer_lock_reserve(struct trace_array *tr,
433 unsigned char type,
434 unsigned long len,
435 unsigned long flags,
436 int pc);
437void trace_buffer_unlock_commit(struct trace_array *tr,
438 struct ring_buffer_event *event,
439 unsigned long flags, int pc);
440
382struct trace_entry *tracing_get_trace_entry(struct trace_array *tr, 441struct trace_entry *tracing_get_trace_entry(struct trace_array *tr,
383 struct trace_array_cpu *data); 442 struct trace_array_cpu *data);
443
444struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
445 int *ent_cpu, u64 *ent_ts);
446
384void tracing_generic_entry_update(struct trace_entry *entry, 447void tracing_generic_entry_update(struct trace_entry *entry,
385 unsigned long flags, 448 unsigned long flags,
386 int pc); 449 int pc);
387 450
451void default_wait_pipe(struct trace_iterator *iter);
452void poll_wait_pipe(struct trace_iterator *iter);
453
388void ftrace(struct trace_array *tr, 454void ftrace(struct trace_array *tr,
389 struct trace_array_cpu *data, 455 struct trace_array_cpu *data,
390 unsigned long ip, 456 unsigned long ip,
391 unsigned long parent_ip, 457 unsigned long parent_ip,
392 unsigned long flags, int pc); 458 unsigned long flags, int pc);
393void tracing_sched_switch_trace(struct trace_array *tr, 459void tracing_sched_switch_trace(struct trace_array *tr,
394 struct trace_array_cpu *data,
395 struct task_struct *prev, 460 struct task_struct *prev,
396 struct task_struct *next, 461 struct task_struct *next,
397 unsigned long flags, int pc); 462 unsigned long flags, int pc);
398void tracing_record_cmdline(struct task_struct *tsk); 463void tracing_record_cmdline(struct task_struct *tsk);
399 464
400void tracing_sched_wakeup_trace(struct trace_array *tr, 465void tracing_sched_wakeup_trace(struct trace_array *tr,
401 struct trace_array_cpu *data,
402 struct task_struct *wakee, 466 struct task_struct *wakee,
403 struct task_struct *cur, 467 struct task_struct *cur,
404 unsigned long flags, int pc); 468 unsigned long flags, int pc);
@@ -408,14 +472,12 @@ void trace_special(struct trace_array *tr,
408 unsigned long arg2, 472 unsigned long arg2,
409 unsigned long arg3, int pc); 473 unsigned long arg3, int pc);
410void trace_function(struct trace_array *tr, 474void trace_function(struct trace_array *tr,
411 struct trace_array_cpu *data,
412 unsigned long ip, 475 unsigned long ip,
413 unsigned long parent_ip, 476 unsigned long parent_ip,
414 unsigned long flags, int pc); 477 unsigned long flags, int pc);
415 478
416void trace_graph_return(struct ftrace_graph_ret *trace); 479void trace_graph_return(struct ftrace_graph_ret *trace);
417int trace_graph_entry(struct ftrace_graph_ent *trace); 480int trace_graph_entry(struct ftrace_graph_ent *trace);
418void trace_hw_branch(struct trace_array *tr, u64 from, u64 to);
419 481
420void tracing_start_cmdline_record(void); 482void tracing_start_cmdline_record(void);
421void tracing_stop_cmdline_record(void); 483void tracing_stop_cmdline_record(void);
@@ -434,15 +496,11 @@ void update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu);
434void update_max_tr_single(struct trace_array *tr, 496void update_max_tr_single(struct trace_array *tr,
435 struct task_struct *tsk, int cpu); 497 struct task_struct *tsk, int cpu);
436 498
437extern cycle_t ftrace_now(int cpu); 499void __trace_stack(struct trace_array *tr,
500 unsigned long flags,
501 int skip, int pc);
438 502
439#ifdef CONFIG_FUNCTION_TRACER 503extern cycle_t ftrace_now(int cpu);
440void tracing_start_function_trace(void);
441void tracing_stop_function_trace(void);
442#else
443# define tracing_start_function_trace() do { } while (0)
444# define tracing_stop_function_trace() do { } while (0)
445#endif
446 504
447#ifdef CONFIG_CONTEXT_SWITCH_TRACER 505#ifdef CONFIG_CONTEXT_SWITCH_TRACER
448typedef void 506typedef void
@@ -456,10 +514,10 @@ struct tracer_switch_ops {
456 void *private; 514 void *private;
457 struct tracer_switch_ops *next; 515 struct tracer_switch_ops *next;
458}; 516};
459
460char *trace_find_cmdline(int pid);
461#endif /* CONFIG_CONTEXT_SWITCH_TRACER */ 517#endif /* CONFIG_CONTEXT_SWITCH_TRACER */
462 518
519extern char *trace_find_cmdline(int pid);
520
463#ifdef CONFIG_DYNAMIC_FTRACE 521#ifdef CONFIG_DYNAMIC_FTRACE
464extern unsigned long ftrace_update_tot_cnt; 522extern unsigned long ftrace_update_tot_cnt;
465#define DYN_FTRACE_TEST_NAME trace_selftest_dynamic_test_func 523#define DYN_FTRACE_TEST_NAME trace_selftest_dynamic_test_func
@@ -469,6 +527,8 @@ extern int DYN_FTRACE_TEST_NAME(void);
469#ifdef CONFIG_FTRACE_STARTUP_TEST 527#ifdef CONFIG_FTRACE_STARTUP_TEST
470extern int trace_selftest_startup_function(struct tracer *trace, 528extern int trace_selftest_startup_function(struct tracer *trace,
471 struct trace_array *tr); 529 struct trace_array *tr);
530extern int trace_selftest_startup_function_graph(struct tracer *trace,
531 struct trace_array *tr);
472extern int trace_selftest_startup_irqsoff(struct tracer *trace, 532extern int trace_selftest_startup_irqsoff(struct tracer *trace,
473 struct trace_array *tr); 533 struct trace_array *tr);
474extern int trace_selftest_startup_preemptoff(struct tracer *trace, 534extern int trace_selftest_startup_preemptoff(struct tracer *trace,
@@ -488,15 +548,6 @@ extern int trace_selftest_startup_branch(struct tracer *trace,
488#endif /* CONFIG_FTRACE_STARTUP_TEST */ 548#endif /* CONFIG_FTRACE_STARTUP_TEST */
489 549
490extern void *head_page(struct trace_array_cpu *data); 550extern void *head_page(struct trace_array_cpu *data);
491extern int trace_seq_printf(struct trace_seq *s, const char *fmt, ...);
492extern void trace_seq_print_cont(struct trace_seq *s,
493 struct trace_iterator *iter);
494
495extern int
496seq_print_ip_sym(struct trace_seq *s, unsigned long ip,
497 unsigned long sym_flags);
498extern ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf,
499 size_t cnt);
500extern long ns2usecs(cycle_t nsec); 551extern long ns2usecs(cycle_t nsec);
501extern int 552extern int
502trace_vprintk(unsigned long ip, int depth, const char *fmt, va_list args); 553trace_vprintk(unsigned long ip, int depth, const char *fmt, va_list args);
@@ -580,7 +631,8 @@ enum trace_iterator_flags {
580 TRACE_ITER_ANNOTATE = 0x2000, 631 TRACE_ITER_ANNOTATE = 0x2000,
581 TRACE_ITER_USERSTACKTRACE = 0x4000, 632 TRACE_ITER_USERSTACKTRACE = 0x4000,
582 TRACE_ITER_SYM_USEROBJ = 0x8000, 633 TRACE_ITER_SYM_USEROBJ = 0x8000,
583 TRACE_ITER_PRINTK_MSGONLY = 0x10000 634 TRACE_ITER_PRINTK_MSGONLY = 0x10000,
635 TRACE_ITER_CONTEXT_INFO = 0x20000 /* Print pid/cpu/time */
584}; 636};
585 637
586/* 638/*
@@ -601,12 +653,12 @@ extern struct tracer nop_trace;
601 * preempt_enable (after a disable), a schedule might take place 653 * preempt_enable (after a disable), a schedule might take place
602 * causing an infinite recursion. 654 * causing an infinite recursion.
603 * 655 *
604 * To prevent this, we read the need_recshed flag before 656 * To prevent this, we read the need_resched flag before
605 * disabling preemption. When we want to enable preemption we 657 * disabling preemption. When we want to enable preemption we
606 * check the flag, if it is set, then we call preempt_enable_no_resched. 658 * check the flag, if it is set, then we call preempt_enable_no_resched.
607 * Otherwise, we call preempt_enable. 659 * Otherwise, we call preempt_enable.
608 * 660 *
609 * The rational for doing the above is that if need resched is set 661 * The rational for doing the above is that if need_resched is set
610 * and we have yet to reschedule, we are either in an atomic location 662 * and we have yet to reschedule, we are either in an atomic location
611 * (where we do not need to check for scheduling) or we are inside 663 * (where we do not need to check for scheduling) or we are inside
612 * the scheduler and do not want to resched. 664 * the scheduler and do not want to resched.
@@ -627,7 +679,7 @@ static inline int ftrace_preempt_disable(void)
627 * 679 *
628 * This is a scheduler safe way to enable preemption and not miss 680 * This is a scheduler safe way to enable preemption and not miss
629 * any preemption checks. The disabled saved the state of preemption. 681 * any preemption checks. The disabled saved the state of preemption.
630 * If resched is set, then we were either inside an atomic or 682 * If resched is set, then we are either inside an atomic or
631 * are inside the scheduler (we would have already scheduled 683 * are inside the scheduler (we would have already scheduled
632 * otherwise). In this case, we do not want to call normal 684 * otherwise). In this case, we do not want to call normal
633 * preempt_enable, but preempt_enable_no_resched instead. 685 * preempt_enable, but preempt_enable_no_resched instead.
diff --git a/kernel/trace/trace_boot.c b/kernel/trace/trace_boot.c
index 366c8c333e13..7a30fc4c3642 100644
--- a/kernel/trace/trace_boot.c
+++ b/kernel/trace/trace_boot.c
@@ -11,6 +11,7 @@
11#include <linux/kallsyms.h> 11#include <linux/kallsyms.h>
12 12
13#include "trace.h" 13#include "trace.h"
14#include "trace_output.h"
14 15
15static struct trace_array *boot_trace; 16static struct trace_array *boot_trace;
16static bool pre_initcalls_finished; 17static bool pre_initcalls_finished;
@@ -27,13 +28,13 @@ void start_boot_trace(void)
27 28
28void enable_boot_trace(void) 29void enable_boot_trace(void)
29{ 30{
30 if (pre_initcalls_finished) 31 if (boot_trace && pre_initcalls_finished)
31 tracing_start_sched_switch_record(); 32 tracing_start_sched_switch_record();
32} 33}
33 34
34void disable_boot_trace(void) 35void disable_boot_trace(void)
35{ 36{
36 if (pre_initcalls_finished) 37 if (boot_trace && pre_initcalls_finished)
37 tracing_stop_sched_switch_record(); 38 tracing_stop_sched_switch_record();
38} 39}
39 40
@@ -42,6 +43,9 @@ static int boot_trace_init(struct trace_array *tr)
42 int cpu; 43 int cpu;
43 boot_trace = tr; 44 boot_trace = tr;
44 45
46 if (!tr)
47 return 0;
48
45 for_each_cpu(cpu, cpu_possible_mask) 49 for_each_cpu(cpu, cpu_possible_mask)
46 tracing_reset(tr, cpu); 50 tracing_reset(tr, cpu);
47 51
@@ -128,10 +132,9 @@ void trace_boot_call(struct boot_trace_call *bt, initcall_t fn)
128{ 132{
129 struct ring_buffer_event *event; 133 struct ring_buffer_event *event;
130 struct trace_boot_call *entry; 134 struct trace_boot_call *entry;
131 unsigned long irq_flags;
132 struct trace_array *tr = boot_trace; 135 struct trace_array *tr = boot_trace;
133 136
134 if (!pre_initcalls_finished) 137 if (!tr || !pre_initcalls_finished)
135 return; 138 return;
136 139
137 /* Get its name now since this function could 140 /* Get its name now since this function could
@@ -140,18 +143,13 @@ void trace_boot_call(struct boot_trace_call *bt, initcall_t fn)
140 sprint_symbol(bt->func, (unsigned long)fn); 143 sprint_symbol(bt->func, (unsigned long)fn);
141 preempt_disable(); 144 preempt_disable();
142 145
143 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry), 146 event = trace_buffer_lock_reserve(tr, TRACE_BOOT_CALL,
144 &irq_flags); 147 sizeof(*entry), 0, 0);
145 if (!event) 148 if (!event)
146 goto out; 149 goto out;
147 entry = ring_buffer_event_data(event); 150 entry = ring_buffer_event_data(event);
148 tracing_generic_entry_update(&entry->ent, 0, 0);
149 entry->ent.type = TRACE_BOOT_CALL;
150 entry->boot_call = *bt; 151 entry->boot_call = *bt;
151 ring_buffer_unlock_commit(tr->buffer, event, irq_flags); 152 trace_buffer_unlock_commit(tr, event, 0, 0);
152
153 trace_wake_up();
154
155 out: 153 out:
156 preempt_enable(); 154 preempt_enable();
157} 155}
@@ -160,27 +158,21 @@ void trace_boot_ret(struct boot_trace_ret *bt, initcall_t fn)
160{ 158{
161 struct ring_buffer_event *event; 159 struct ring_buffer_event *event;
162 struct trace_boot_ret *entry; 160 struct trace_boot_ret *entry;
163 unsigned long irq_flags;
164 struct trace_array *tr = boot_trace; 161 struct trace_array *tr = boot_trace;
165 162
166 if (!pre_initcalls_finished) 163 if (!tr || !pre_initcalls_finished)
167 return; 164 return;
168 165
169 sprint_symbol(bt->func, (unsigned long)fn); 166 sprint_symbol(bt->func, (unsigned long)fn);
170 preempt_disable(); 167 preempt_disable();
171 168
172 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry), 169 event = trace_buffer_lock_reserve(tr, TRACE_BOOT_RET,
173 &irq_flags); 170 sizeof(*entry), 0, 0);
174 if (!event) 171 if (!event)
175 goto out; 172 goto out;
176 entry = ring_buffer_event_data(event); 173 entry = ring_buffer_event_data(event);
177 tracing_generic_entry_update(&entry->ent, 0, 0);
178 entry->ent.type = TRACE_BOOT_RET;
179 entry->boot_ret = *bt; 174 entry->boot_ret = *bt;
180 ring_buffer_unlock_commit(tr->buffer, event, irq_flags); 175 trace_buffer_unlock_commit(tr, event, 0, 0);
181
182 trace_wake_up();
183
184 out: 176 out:
185 preempt_enable(); 177 preempt_enable();
186} 178}
diff --git a/kernel/trace/trace_branch.c b/kernel/trace/trace_branch.c
index 6c00feb3bac7..c2e68d440c4d 100644
--- a/kernel/trace/trace_branch.c
+++ b/kernel/trace/trace_branch.c
@@ -14,12 +14,17 @@
14#include <linux/hash.h> 14#include <linux/hash.h>
15#include <linux/fs.h> 15#include <linux/fs.h>
16#include <asm/local.h> 16#include <asm/local.h>
17
17#include "trace.h" 18#include "trace.h"
19#include "trace_stat.h"
20#include "trace_output.h"
18 21
19#ifdef CONFIG_BRANCH_TRACER 22#ifdef CONFIG_BRANCH_TRACER
20 23
24static struct tracer branch_trace;
21static int branch_tracing_enabled __read_mostly; 25static int branch_tracing_enabled __read_mostly;
22static DEFINE_MUTEX(branch_tracing_mutex); 26static DEFINE_MUTEX(branch_tracing_mutex);
27
23static struct trace_array *branch_tracer; 28static struct trace_array *branch_tracer;
24 29
25static void 30static void
@@ -28,7 +33,7 @@ probe_likely_condition(struct ftrace_branch_data *f, int val, int expect)
28 struct trace_array *tr = branch_tracer; 33 struct trace_array *tr = branch_tracer;
29 struct ring_buffer_event *event; 34 struct ring_buffer_event *event;
30 struct trace_branch *entry; 35 struct trace_branch *entry;
31 unsigned long flags, irq_flags; 36 unsigned long flags;
32 int cpu, pc; 37 int cpu, pc;
33 const char *p; 38 const char *p;
34 39
@@ -47,15 +52,13 @@ probe_likely_condition(struct ftrace_branch_data *f, int val, int expect)
47 if (atomic_inc_return(&tr->data[cpu]->disabled) != 1) 52 if (atomic_inc_return(&tr->data[cpu]->disabled) != 1)
48 goto out; 53 goto out;
49 54
50 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry), 55 pc = preempt_count();
51 &irq_flags); 56 event = trace_buffer_lock_reserve(tr, TRACE_BRANCH,
57 sizeof(*entry), flags, pc);
52 if (!event) 58 if (!event)
53 goto out; 59 goto out;
54 60
55 pc = preempt_count();
56 entry = ring_buffer_event_data(event); 61 entry = ring_buffer_event_data(event);
57 tracing_generic_entry_update(&entry->ent, flags, pc);
58 entry->ent.type = TRACE_BRANCH;
59 62
60 /* Strip off the path, only save the file */ 63 /* Strip off the path, only save the file */
61 p = f->file + strlen(f->file); 64 p = f->file + strlen(f->file);
@@ -70,7 +73,7 @@ probe_likely_condition(struct ftrace_branch_data *f, int val, int expect)
70 entry->line = f->line; 73 entry->line = f->line;
71 entry->correct = val == expect; 74 entry->correct = val == expect;
72 75
73 ring_buffer_unlock_commit(tr->buffer, event, irq_flags); 76 ring_buffer_unlock_commit(tr->buffer, event);
74 77
75 out: 78 out:
76 atomic_dec(&tr->data[cpu]->disabled); 79 atomic_dec(&tr->data[cpu]->disabled);
@@ -88,8 +91,6 @@ void trace_likely_condition(struct ftrace_branch_data *f, int val, int expect)
88 91
89int enable_branch_tracing(struct trace_array *tr) 92int enable_branch_tracing(struct trace_array *tr)
90{ 93{
91 int ret = 0;
92
93 mutex_lock(&branch_tracing_mutex); 94 mutex_lock(&branch_tracing_mutex);
94 branch_tracer = tr; 95 branch_tracer = tr;
95 /* 96 /*
@@ -100,7 +101,7 @@ int enable_branch_tracing(struct trace_array *tr)
100 branch_tracing_enabled++; 101 branch_tracing_enabled++;
101 mutex_unlock(&branch_tracing_mutex); 102 mutex_unlock(&branch_tracing_mutex);
102 103
103 return ret; 104 return 0;
104} 105}
105 106
106void disable_branch_tracing(void) 107void disable_branch_tracing(void)
@@ -128,11 +129,6 @@ static void stop_branch_trace(struct trace_array *tr)
128 129
129static int branch_trace_init(struct trace_array *tr) 130static int branch_trace_init(struct trace_array *tr)
130{ 131{
131 int cpu;
132
133 for_each_online_cpu(cpu)
134 tracing_reset(tr, cpu);
135
136 start_branch_trace(tr); 132 start_branch_trace(tr);
137 return 0; 133 return 0;
138} 134}
@@ -142,22 +138,54 @@ static void branch_trace_reset(struct trace_array *tr)
142 stop_branch_trace(tr); 138 stop_branch_trace(tr);
143} 139}
144 140
145struct tracer branch_trace __read_mostly = 141static enum print_line_t trace_branch_print(struct trace_iterator *iter,
142 int flags)
143{
144 struct trace_branch *field;
145
146 trace_assign_type(field, iter->ent);
147
148 if (trace_seq_printf(&iter->seq, "[%s] %s:%s:%d\n",
149 field->correct ? " ok " : " MISS ",
150 field->func,
151 field->file,
152 field->line))
153 return TRACE_TYPE_PARTIAL_LINE;
154
155 return TRACE_TYPE_HANDLED;
156}
157
158
159static struct trace_event trace_branch_event = {
160 .type = TRACE_BRANCH,
161 .trace = trace_branch_print,
162 .latency_trace = trace_branch_print,
163};
164
165static struct tracer branch_trace __read_mostly =
146{ 166{
147 .name = "branch", 167 .name = "branch",
148 .init = branch_trace_init, 168 .init = branch_trace_init,
149 .reset = branch_trace_reset, 169 .reset = branch_trace_reset,
150#ifdef CONFIG_FTRACE_SELFTEST 170#ifdef CONFIG_FTRACE_SELFTEST
151 .selftest = trace_selftest_startup_branch, 171 .selftest = trace_selftest_startup_branch,
152#endif 172#endif /* CONFIG_FTRACE_SELFTEST */
153}; 173};
154 174
155__init static int init_branch_trace(void) 175__init static int init_branch_tracer(void)
156{ 176{
177 int ret;
178
179 ret = register_ftrace_event(&trace_branch_event);
180 if (!ret) {
181 printk(KERN_WARNING "Warning: could not register "
182 "branch events\n");
183 return 1;
184 }
157 return register_tracer(&branch_trace); 185 return register_tracer(&branch_trace);
158} 186}
187device_initcall(init_branch_tracer);
159 188
160device_initcall(init_branch_trace);
161#else 189#else
162static inline 190static inline
163void trace_likely_condition(struct ftrace_branch_data *f, int val, int expect) 191void trace_likely_condition(struct ftrace_branch_data *f, int val, int expect)
@@ -183,66 +211,39 @@ void ftrace_likely_update(struct ftrace_branch_data *f, int val, int expect)
183} 211}
184EXPORT_SYMBOL(ftrace_likely_update); 212EXPORT_SYMBOL(ftrace_likely_update);
185 213
186struct ftrace_pointer { 214extern unsigned long __start_annotated_branch_profile[];
187 void *start; 215extern unsigned long __stop_annotated_branch_profile[];
188 void *stop;
189 int hit;
190};
191 216
192static void * 217static int annotated_branch_stat_headers(struct seq_file *m)
193t_next(struct seq_file *m, void *v, loff_t *pos)
194{ 218{
195 const struct ftrace_pointer *f = m->private; 219 seq_printf(m, " correct incorrect %% ");
196 struct ftrace_branch_data *p = v; 220 seq_printf(m, " Function "
197 221 " File Line\n"
198 (*pos)++; 222 " ------- --------- - "
199 223 " -------- "
200 if (v == (void *)1) 224 " ---- ----\n");
201 return f->start; 225 return 0;
202
203 ++p;
204
205 if ((void *)p >= (void *)f->stop)
206 return NULL;
207
208 return p;
209} 226}
210 227
211static void *t_start(struct seq_file *m, loff_t *pos) 228static inline long get_incorrect_percent(struct ftrace_branch_data *p)
212{ 229{
213 void *t = (void *)1; 230 long percent;
214 loff_t l = 0;
215
216 for (; t && l < *pos; t = t_next(m, t, &l))
217 ;
218 231
219 return t; 232 if (p->correct) {
220} 233 percent = p->incorrect * 100;
234 percent /= p->correct + p->incorrect;
235 } else
236 percent = p->incorrect ? 100 : -1;
221 237
222static void t_stop(struct seq_file *m, void *p) 238 return percent;
223{
224} 239}
225 240
226static int t_show(struct seq_file *m, void *v) 241static int branch_stat_show(struct seq_file *m, void *v)
227{ 242{
228 const struct ftrace_pointer *fp = m->private;
229 struct ftrace_branch_data *p = v; 243 struct ftrace_branch_data *p = v;
230 const char *f; 244 const char *f;
231 long percent; 245 long percent;
232 246
233 if (v == (void *)1) {
234 if (fp->hit)
235 seq_printf(m, " miss hit %% ");
236 else
237 seq_printf(m, " correct incorrect %% ");
238 seq_printf(m, " Function "
239 " File Line\n"
240 " ------- --------- - "
241 " -------- "
242 " ---- ----\n");
243 return 0;
244 }
245
246 /* Only print the file, not the path */ 247 /* Only print the file, not the path */
247 f = p->file + strlen(p->file); 248 f = p->file + strlen(p->file);
248 while (f >= p->file && *f != '/') 249 while (f >= p->file && *f != '/')
@@ -252,11 +253,7 @@ static int t_show(struct seq_file *m, void *v)
252 /* 253 /*
253 * The miss is overlayed on correct, and hit on incorrect. 254 * The miss is overlayed on correct, and hit on incorrect.
254 */ 255 */
255 if (p->correct) { 256 percent = get_incorrect_percent(p);
256 percent = p->incorrect * 100;
257 percent /= p->correct + p->incorrect;
258 } else
259 percent = p->incorrect ? 100 : -1;
260 257
261 seq_printf(m, "%8lu %8lu ", p->correct, p->incorrect); 258 seq_printf(m, "%8lu %8lu ", p->correct, p->incorrect);
262 if (percent < 0) 259 if (percent < 0)
@@ -267,76 +264,118 @@ static int t_show(struct seq_file *m, void *v)
267 return 0; 264 return 0;
268} 265}
269 266
270static struct seq_operations tracing_likely_seq_ops = { 267static void *annotated_branch_stat_start(void)
271 .start = t_start, 268{
272 .next = t_next, 269 return __start_annotated_branch_profile;
273 .stop = t_stop, 270}
274 .show = t_show, 271
272static void *
273annotated_branch_stat_next(void *v, int idx)
274{
275 struct ftrace_branch_data *p = v;
276
277 ++p;
278
279 if ((void *)p >= (void *)__stop_annotated_branch_profile)
280 return NULL;
281
282 return p;
283}
284
285static int annotated_branch_stat_cmp(void *p1, void *p2)
286{
287 struct ftrace_branch_data *a = p1;
288 struct ftrace_branch_data *b = p2;
289
290 long percent_a, percent_b;
291
292 percent_a = get_incorrect_percent(a);
293 percent_b = get_incorrect_percent(b);
294
295 if (percent_a < percent_b)
296 return -1;
297 if (percent_a > percent_b)
298 return 1;
299 else
300 return 0;
301}
302
303static struct tracer_stat annotated_branch_stats = {
304 .name = "branch_annotated",
305 .stat_start = annotated_branch_stat_start,
306 .stat_next = annotated_branch_stat_next,
307 .stat_cmp = annotated_branch_stat_cmp,
308 .stat_headers = annotated_branch_stat_headers,
309 .stat_show = branch_stat_show
275}; 310};
276 311
277static int tracing_branch_open(struct inode *inode, struct file *file) 312__init static int init_annotated_branch_stats(void)
278{ 313{
279 int ret; 314 int ret;
280 315
281 ret = seq_open(file, &tracing_likely_seq_ops); 316 ret = register_stat_tracer(&annotated_branch_stats);
282 if (!ret) { 317 if (!ret) {
283 struct seq_file *m = file->private_data; 318 printk(KERN_WARNING "Warning: could not register "
284 m->private = (void *)inode->i_private; 319 "annotated branches stats\n");
320 return 1;
285 } 321 }
286 322 return 0;
287 return ret;
288} 323}
289 324fs_initcall(init_annotated_branch_stats);
290static const struct file_operations tracing_branch_fops = {
291 .open = tracing_branch_open,
292 .read = seq_read,
293 .llseek = seq_lseek,
294};
295 325
296#ifdef CONFIG_PROFILE_ALL_BRANCHES 326#ifdef CONFIG_PROFILE_ALL_BRANCHES
327
297extern unsigned long __start_branch_profile[]; 328extern unsigned long __start_branch_profile[];
298extern unsigned long __stop_branch_profile[]; 329extern unsigned long __stop_branch_profile[];
299 330
300static const struct ftrace_pointer ftrace_branch_pos = { 331static int all_branch_stat_headers(struct seq_file *m)
301 .start = __start_branch_profile, 332{
302 .stop = __stop_branch_profile, 333 seq_printf(m, " miss hit %% ");
303 .hit = 1, 334 seq_printf(m, " Function "
304}; 335 " File Line\n"
336 " ------- --------- - "
337 " -------- "
338 " ---- ----\n");
339 return 0;
340}
305 341
306#endif /* CONFIG_PROFILE_ALL_BRANCHES */ 342static void *all_branch_stat_start(void)
343{
344 return __start_branch_profile;
345}
307 346
308extern unsigned long __start_annotated_branch_profile[]; 347static void *
309extern unsigned long __stop_annotated_branch_profile[]; 348all_branch_stat_next(void *v, int idx)
349{
350 struct ftrace_branch_data *p = v;
310 351
311static const struct ftrace_pointer ftrace_annotated_branch_pos = { 352 ++p;
312 .start = __start_annotated_branch_profile,
313 .stop = __stop_annotated_branch_profile,
314};
315 353
316static __init int ftrace_branch_init(void) 354 if ((void *)p >= (void *)__stop_branch_profile)
317{ 355 return NULL;
318 struct dentry *d_tracer;
319 struct dentry *entry;
320 356
321 d_tracer = tracing_init_dentry(); 357 return p;
358}
322 359
323 entry = debugfs_create_file("profile_annotated_branch", 0444, d_tracer, 360static struct tracer_stat all_branch_stats = {
324 (void *)&ftrace_annotated_branch_pos, 361 .name = "branch_all",
325 &tracing_branch_fops); 362 .stat_start = all_branch_stat_start,
326 if (!entry) 363 .stat_next = all_branch_stat_next,
327 pr_warning("Could not create debugfs " 364 .stat_headers = all_branch_stat_headers,
328 "'profile_annotatet_branch' entry\n"); 365 .stat_show = branch_stat_show
366};
329 367
330#ifdef CONFIG_PROFILE_ALL_BRANCHES 368__init static int all_annotated_branch_stats(void)
331 entry = debugfs_create_file("profile_branch", 0444, d_tracer, 369{
332 (void *)&ftrace_branch_pos, 370 int ret;
333 &tracing_branch_fops);
334 if (!entry)
335 pr_warning("Could not create debugfs"
336 " 'profile_branch' entry\n");
337#endif
338 371
372 ret = register_stat_tracer(&all_branch_stats);
373 if (!ret) {
374 printk(KERN_WARNING "Warning: could not register "
375 "all branches stats\n");
376 return 1;
377 }
339 return 0; 378 return 0;
340} 379}
341 380fs_initcall(all_annotated_branch_stats);
342device_initcall(ftrace_branch_init); 381#endif /* CONFIG_PROFILE_ALL_BRANCHES */
diff --git a/kernel/trace/trace_functions.c b/kernel/trace/trace_functions.c
index 9236d7e25a16..c9a0b7df44ff 100644
--- a/kernel/trace/trace_functions.c
+++ b/kernel/trace/trace_functions.c
@@ -9,6 +9,7 @@
9 * Copyright (C) 2004-2006 Ingo Molnar 9 * Copyright (C) 2004-2006 Ingo Molnar
10 * Copyright (C) 2004 William Lee Irwin III 10 * Copyright (C) 2004 William Lee Irwin III
11 */ 11 */
12#include <linux/ring_buffer.h>
12#include <linux/debugfs.h> 13#include <linux/debugfs.h>
13#include <linux/uaccess.h> 14#include <linux/uaccess.h>
14#include <linux/ftrace.h> 15#include <linux/ftrace.h>
@@ -16,52 +17,388 @@
16 17
17#include "trace.h" 18#include "trace.h"
18 19
19static void start_function_trace(struct trace_array *tr) 20/* function tracing enabled */
21static int ftrace_function_enabled;
22
23static struct trace_array *func_trace;
24
25static void tracing_start_function_trace(void);
26static void tracing_stop_function_trace(void);
27
28static int function_trace_init(struct trace_array *tr)
20{ 29{
30 func_trace = tr;
21 tr->cpu = get_cpu(); 31 tr->cpu = get_cpu();
22 tracing_reset_online_cpus(tr);
23 put_cpu(); 32 put_cpu();
24 33
25 tracing_start_cmdline_record(); 34 tracing_start_cmdline_record();
26 tracing_start_function_trace(); 35 tracing_start_function_trace();
36 return 0;
27} 37}
28 38
29static void stop_function_trace(struct trace_array *tr) 39static void function_trace_reset(struct trace_array *tr)
30{ 40{
31 tracing_stop_function_trace(); 41 tracing_stop_function_trace();
32 tracing_stop_cmdline_record(); 42 tracing_stop_cmdline_record();
33} 43}
34 44
35static int function_trace_init(struct trace_array *tr) 45static void function_trace_start(struct trace_array *tr)
36{ 46{
37 start_function_trace(tr); 47 tracing_reset_online_cpus(tr);
38 return 0;
39} 48}
40 49
41static void function_trace_reset(struct trace_array *tr) 50static void
51function_trace_call_preempt_only(unsigned long ip, unsigned long parent_ip)
52{
53 struct trace_array *tr = func_trace;
54 struct trace_array_cpu *data;
55 unsigned long flags;
56 long disabled;
57 int cpu, resched;
58 int pc;
59
60 if (unlikely(!ftrace_function_enabled))
61 return;
62
63 pc = preempt_count();
64 resched = ftrace_preempt_disable();
65 local_save_flags(flags);
66 cpu = raw_smp_processor_id();
67 data = tr->data[cpu];
68 disabled = atomic_inc_return(&data->disabled);
69
70 if (likely(disabled == 1))
71 trace_function(tr, ip, parent_ip, flags, pc);
72
73 atomic_dec(&data->disabled);
74 ftrace_preempt_enable(resched);
75}
76
77static void
78function_trace_call(unsigned long ip, unsigned long parent_ip)
42{ 79{
43 stop_function_trace(tr); 80 struct trace_array *tr = func_trace;
81 struct trace_array_cpu *data;
82 unsigned long flags;
83 long disabled;
84 int cpu;
85 int pc;
86
87 if (unlikely(!ftrace_function_enabled))
88 return;
89
90 /*
91 * Need to use raw, since this must be called before the
92 * recursive protection is performed.
93 */
94 local_irq_save(flags);
95 cpu = raw_smp_processor_id();
96 data = tr->data[cpu];
97 disabled = atomic_inc_return(&data->disabled);
98
99 if (likely(disabled == 1)) {
100 pc = preempt_count();
101 trace_function(tr, ip, parent_ip, flags, pc);
102 }
103
104 atomic_dec(&data->disabled);
105 local_irq_restore(flags);
44} 106}
45 107
46static void function_trace_start(struct trace_array *tr) 108static void
109function_stack_trace_call(unsigned long ip, unsigned long parent_ip)
47{ 110{
48 tracing_reset_online_cpus(tr); 111 struct trace_array *tr = func_trace;
112 struct trace_array_cpu *data;
113 unsigned long flags;
114 long disabled;
115 int cpu;
116 int pc;
117
118 if (unlikely(!ftrace_function_enabled))
119 return;
120
121 /*
122 * Need to use raw, since this must be called before the
123 * recursive protection is performed.
124 */
125 local_irq_save(flags);
126 cpu = raw_smp_processor_id();
127 data = tr->data[cpu];
128 disabled = atomic_inc_return(&data->disabled);
129
130 if (likely(disabled == 1)) {
131 pc = preempt_count();
132 trace_function(tr, ip, parent_ip, flags, pc);
133 /*
134 * skip over 5 funcs:
135 * __ftrace_trace_stack,
136 * __trace_stack,
137 * function_stack_trace_call
138 * ftrace_list_func
139 * ftrace_call
140 */
141 __trace_stack(tr, flags, 5, pc);
142 }
143
144 atomic_dec(&data->disabled);
145 local_irq_restore(flags);
146}
147
148
149static struct ftrace_ops trace_ops __read_mostly =
150{
151 .func = function_trace_call,
152};
153
154static struct ftrace_ops trace_stack_ops __read_mostly =
155{
156 .func = function_stack_trace_call,
157};
158
159/* Our two options */
160enum {
161 TRACE_FUNC_OPT_STACK = 0x1,
162};
163
164static struct tracer_opt func_opts[] = {
165#ifdef CONFIG_STACKTRACE
166 { TRACER_OPT(func_stack_trace, TRACE_FUNC_OPT_STACK) },
167#endif
168 { } /* Always set a last empty entry */
169};
170
171static struct tracer_flags func_flags = {
172 .val = 0, /* By default: all flags disabled */
173 .opts = func_opts
174};
175
176static void tracing_start_function_trace(void)
177{
178 ftrace_function_enabled = 0;
179
180 if (trace_flags & TRACE_ITER_PREEMPTONLY)
181 trace_ops.func = function_trace_call_preempt_only;
182 else
183 trace_ops.func = function_trace_call;
184
185 if (func_flags.val & TRACE_FUNC_OPT_STACK)
186 register_ftrace_function(&trace_stack_ops);
187 else
188 register_ftrace_function(&trace_ops);
189
190 ftrace_function_enabled = 1;
191}
192
193static void tracing_stop_function_trace(void)
194{
195 ftrace_function_enabled = 0;
196 /* OK if they are not registered */
197 unregister_ftrace_function(&trace_stack_ops);
198 unregister_ftrace_function(&trace_ops);
199}
200
201static int func_set_flag(u32 old_flags, u32 bit, int set)
202{
203 if (bit == TRACE_FUNC_OPT_STACK) {
204 /* do nothing if already set */
205 if (!!set == !!(func_flags.val & TRACE_FUNC_OPT_STACK))
206 return 0;
207
208 if (set) {
209 unregister_ftrace_function(&trace_ops);
210 register_ftrace_function(&trace_stack_ops);
211 } else {
212 unregister_ftrace_function(&trace_stack_ops);
213 register_ftrace_function(&trace_ops);
214 }
215
216 return 0;
217 }
218
219 return -EINVAL;
49} 220}
50 221
51static struct tracer function_trace __read_mostly = 222static struct tracer function_trace __read_mostly =
52{ 223{
53 .name = "function", 224 .name = "function",
54 .init = function_trace_init, 225 .init = function_trace_init,
55 .reset = function_trace_reset, 226 .reset = function_trace_reset,
56 .start = function_trace_start, 227 .start = function_trace_start,
228 .wait_pipe = poll_wait_pipe,
229 .flags = &func_flags,
230 .set_flag = func_set_flag,
57#ifdef CONFIG_FTRACE_SELFTEST 231#ifdef CONFIG_FTRACE_SELFTEST
58 .selftest = trace_selftest_startup_function, 232 .selftest = trace_selftest_startup_function,
59#endif 233#endif
60}; 234};
61 235
236#ifdef CONFIG_DYNAMIC_FTRACE
237static void
238ftrace_traceon(unsigned long ip, unsigned long parent_ip, void **data)
239{
240 long *count = (long *)data;
241
242 if (tracing_is_on())
243 return;
244
245 if (!*count)
246 return;
247
248 if (*count != -1)
249 (*count)--;
250
251 tracing_on();
252}
253
254static void
255ftrace_traceoff(unsigned long ip, unsigned long parent_ip, void **data)
256{
257 long *count = (long *)data;
258
259 if (!tracing_is_on())
260 return;
261
262 if (!*count)
263 return;
264
265 if (*count != -1)
266 (*count)--;
267
268 tracing_off();
269}
270
271static int
272ftrace_trace_onoff_print(struct seq_file *m, unsigned long ip,
273 struct ftrace_probe_ops *ops, void *data);
274
275static struct ftrace_probe_ops traceon_probe_ops = {
276 .func = ftrace_traceon,
277 .print = ftrace_trace_onoff_print,
278};
279
280static struct ftrace_probe_ops traceoff_probe_ops = {
281 .func = ftrace_traceoff,
282 .print = ftrace_trace_onoff_print,
283};
284
285static int
286ftrace_trace_onoff_print(struct seq_file *m, unsigned long ip,
287 struct ftrace_probe_ops *ops, void *data)
288{
289 char str[KSYM_SYMBOL_LEN];
290 long count = (long)data;
291
292 kallsyms_lookup(ip, NULL, NULL, NULL, str);
293 seq_printf(m, "%s:", str);
294
295 if (ops == &traceon_probe_ops)
296 seq_printf(m, "traceon");
297 else
298 seq_printf(m, "traceoff");
299
300 if (count == -1)
301 seq_printf(m, ":unlimited\n");
302 else
303 seq_printf(m, ":count=%ld", count);
304 seq_putc(m, '\n');
305
306 return 0;
307}
308
309static int
310ftrace_trace_onoff_unreg(char *glob, char *cmd, char *param)
311{
312 struct ftrace_probe_ops *ops;
313
314 /* we register both traceon and traceoff to this callback */
315 if (strcmp(cmd, "traceon") == 0)
316 ops = &traceon_probe_ops;
317 else
318 ops = &traceoff_probe_ops;
319
320 unregister_ftrace_function_probe_func(glob, ops);
321
322 return 0;
323}
324
325static int
326ftrace_trace_onoff_callback(char *glob, char *cmd, char *param, int enable)
327{
328 struct ftrace_probe_ops *ops;
329 void *count = (void *)-1;
330 char *number;
331 int ret;
332
333 /* hash funcs only work with set_ftrace_filter */
334 if (!enable)
335 return -EINVAL;
336
337 if (glob[0] == '!')
338 return ftrace_trace_onoff_unreg(glob+1, cmd, param);
339
340 /* we register both traceon and traceoff to this callback */
341 if (strcmp(cmd, "traceon") == 0)
342 ops = &traceon_probe_ops;
343 else
344 ops = &traceoff_probe_ops;
345
346 if (!param)
347 goto out_reg;
348
349 number = strsep(&param, ":");
350
351 if (!strlen(number))
352 goto out_reg;
353
354 /*
355 * We use the callback data field (which is a pointer)
356 * as our counter.
357 */
358 ret = strict_strtoul(number, 0, (unsigned long *)&count);
359 if (ret)
360 return ret;
361
362 out_reg:
363 ret = register_ftrace_function_probe(glob, ops, count);
364
365 return ret;
366}
367
368static struct ftrace_func_command ftrace_traceon_cmd = {
369 .name = "traceon",
370 .func = ftrace_trace_onoff_callback,
371};
372
373static struct ftrace_func_command ftrace_traceoff_cmd = {
374 .name = "traceoff",
375 .func = ftrace_trace_onoff_callback,
376};
377
378static int __init init_func_cmd_traceon(void)
379{
380 int ret;
381
382 ret = register_ftrace_command(&ftrace_traceoff_cmd);
383 if (ret)
384 return ret;
385
386 ret = register_ftrace_command(&ftrace_traceon_cmd);
387 if (ret)
388 unregister_ftrace_command(&ftrace_traceoff_cmd);
389 return ret;
390}
391#else
392static inline int init_func_cmd_traceon(void)
393{
394 return 0;
395}
396#endif /* CONFIG_DYNAMIC_FTRACE */
397
62static __init int init_function_trace(void) 398static __init int init_function_trace(void)
63{ 399{
400 init_func_cmd_traceon();
64 return register_tracer(&function_trace); 401 return register_tracer(&function_trace);
65} 402}
66
67device_initcall(init_function_trace); 403device_initcall(init_function_trace);
404
diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c
index 930c08e5b38e..c009553a8e81 100644
--- a/kernel/trace/trace_functions_graph.c
+++ b/kernel/trace/trace_functions_graph.c
@@ -1,7 +1,7 @@
1/* 1/*
2 * 2 *
3 * Function graph tracer. 3 * Function graph tracer.
4 * Copyright (c) 2008 Frederic Weisbecker <fweisbec@gmail.com> 4 * Copyright (c) 2008-2009 Frederic Weisbecker <fweisbec@gmail.com>
5 * Mostly borrowed from function tracer which 5 * Mostly borrowed from function tracer which
6 * is Copyright (c) Steven Rostedt <srostedt@redhat.com> 6 * is Copyright (c) Steven Rostedt <srostedt@redhat.com>
7 * 7 *
@@ -12,6 +12,7 @@
12#include <linux/fs.h> 12#include <linux/fs.h>
13 13
14#include "trace.h" 14#include "trace.h"
15#include "trace_output.h"
15 16
16#define TRACE_GRAPH_INDENT 2 17#define TRACE_GRAPH_INDENT 2
17 18
@@ -20,9 +21,11 @@
20#define TRACE_GRAPH_PRINT_CPU 0x2 21#define TRACE_GRAPH_PRINT_CPU 0x2
21#define TRACE_GRAPH_PRINT_OVERHEAD 0x4 22#define TRACE_GRAPH_PRINT_OVERHEAD 0x4
22#define TRACE_GRAPH_PRINT_PROC 0x8 23#define TRACE_GRAPH_PRINT_PROC 0x8
24#define TRACE_GRAPH_PRINT_DURATION 0x10
25#define TRACE_GRAPH_PRINT_ABS_TIME 0X20
23 26
24static struct tracer_opt trace_opts[] = { 27static struct tracer_opt trace_opts[] = {
25 /* Display overruns ? */ 28 /* Display overruns? (for self-debug purpose) */
26 { TRACER_OPT(funcgraph-overrun, TRACE_GRAPH_PRINT_OVERRUN) }, 29 { TRACER_OPT(funcgraph-overrun, TRACE_GRAPH_PRINT_OVERRUN) },
27 /* Display CPU ? */ 30 /* Display CPU ? */
28 { TRACER_OPT(funcgraph-cpu, TRACE_GRAPH_PRINT_CPU) }, 31 { TRACER_OPT(funcgraph-cpu, TRACE_GRAPH_PRINT_CPU) },
@@ -30,26 +33,101 @@ static struct tracer_opt trace_opts[] = {
30 { TRACER_OPT(funcgraph-overhead, TRACE_GRAPH_PRINT_OVERHEAD) }, 33 { TRACER_OPT(funcgraph-overhead, TRACE_GRAPH_PRINT_OVERHEAD) },
31 /* Display proc name/pid */ 34 /* Display proc name/pid */
32 { TRACER_OPT(funcgraph-proc, TRACE_GRAPH_PRINT_PROC) }, 35 { TRACER_OPT(funcgraph-proc, TRACE_GRAPH_PRINT_PROC) },
36 /* Display duration of execution */
37 { TRACER_OPT(funcgraph-duration, TRACE_GRAPH_PRINT_DURATION) },
38 /* Display absolute time of an entry */
39 { TRACER_OPT(funcgraph-abstime, TRACE_GRAPH_PRINT_ABS_TIME) },
33 { } /* Empty entry */ 40 { } /* Empty entry */
34}; 41};
35 42
36static struct tracer_flags tracer_flags = { 43static struct tracer_flags tracer_flags = {
37 /* Don't display overruns and proc by default */ 44 /* Don't display overruns and proc by default */
38 .val = TRACE_GRAPH_PRINT_CPU | TRACE_GRAPH_PRINT_OVERHEAD, 45 .val = TRACE_GRAPH_PRINT_CPU | TRACE_GRAPH_PRINT_OVERHEAD |
46 TRACE_GRAPH_PRINT_DURATION,
39 .opts = trace_opts 47 .opts = trace_opts
40}; 48};
41 49
42/* pid on the last trace processed */ 50/* pid on the last trace processed */
43static pid_t last_pid[NR_CPUS] = { [0 ... NR_CPUS-1] = -1 };
44 51
45static int graph_trace_init(struct trace_array *tr) 52
53/* Add a function return address to the trace stack on thread info.*/
54int
55ftrace_push_return_trace(unsigned long ret, unsigned long long time,
56 unsigned long func, int *depth)
57{
58 int index;
59
60 if (!current->ret_stack)
61 return -EBUSY;
62
63 /* The return trace stack is full */
64 if (current->curr_ret_stack == FTRACE_RETFUNC_DEPTH - 1) {
65 atomic_inc(&current->trace_overrun);
66 return -EBUSY;
67 }
68
69 index = ++current->curr_ret_stack;
70 barrier();
71 current->ret_stack[index].ret = ret;
72 current->ret_stack[index].func = func;
73 current->ret_stack[index].calltime = time;
74 *depth = index;
75
76 return 0;
77}
78
79/* Retrieve a function return address to the trace stack on thread info.*/
80void
81ftrace_pop_return_trace(struct ftrace_graph_ret *trace, unsigned long *ret)
46{ 82{
47 int cpu, ret; 83 int index;
84
85 index = current->curr_ret_stack;
48 86
49 for_each_online_cpu(cpu) 87 if (unlikely(index < 0)) {
50 tracing_reset(tr, cpu); 88 ftrace_graph_stop();
89 WARN_ON(1);
90 /* Might as well panic, otherwise we have no where to go */
91 *ret = (unsigned long)panic;
92 return;
93 }
94
95 *ret = current->ret_stack[index].ret;
96 trace->func = current->ret_stack[index].func;
97 trace->calltime = current->ret_stack[index].calltime;
98 trace->overrun = atomic_read(&current->trace_overrun);
99 trace->depth = index;
100 barrier();
101 current->curr_ret_stack--;
102
103}
104
105/*
106 * Send the trace to the ring-buffer.
107 * @return the original return address.
108 */
109unsigned long ftrace_return_to_handler(void)
110{
111 struct ftrace_graph_ret trace;
112 unsigned long ret;
113
114 ftrace_pop_return_trace(&trace, &ret);
115 trace.rettime = cpu_clock(raw_smp_processor_id());
116 ftrace_graph_return(&trace);
117
118 if (unlikely(!ret)) {
119 ftrace_graph_stop();
120 WARN_ON(1);
121 /* Might as well panic. What else to do? */
122 ret = (unsigned long)panic;
123 }
124
125 return ret;
126}
51 127
52 ret = register_ftrace_graph(&trace_graph_return, 128static int graph_trace_init(struct trace_array *tr)
129{
130 int ret = register_ftrace_graph(&trace_graph_return,
53 &trace_graph_entry); 131 &trace_graph_entry);
54 if (ret) 132 if (ret)
55 return ret; 133 return ret;
@@ -153,17 +231,25 @@ print_graph_proc(struct trace_seq *s, pid_t pid)
153 231
154/* If the pid changed since the last trace, output this event */ 232/* If the pid changed since the last trace, output this event */
155static enum print_line_t 233static enum print_line_t
156verif_pid(struct trace_seq *s, pid_t pid, int cpu) 234verif_pid(struct trace_seq *s, pid_t pid, int cpu, pid_t *last_pids_cpu)
157{ 235{
158 pid_t prev_pid; 236 pid_t prev_pid;
237 pid_t *last_pid;
159 int ret; 238 int ret;
160 239
161 if (last_pid[cpu] != -1 && last_pid[cpu] == pid) 240 if (!last_pids_cpu)
241 return TRACE_TYPE_HANDLED;
242
243 last_pid = per_cpu_ptr(last_pids_cpu, cpu);
244
245 if (*last_pid == pid)
162 return TRACE_TYPE_HANDLED; 246 return TRACE_TYPE_HANDLED;
163 247
164 prev_pid = last_pid[cpu]; 248 prev_pid = *last_pid;
165 last_pid[cpu] = pid; 249 *last_pid = pid;
166 250
251 if (prev_pid == -1)
252 return TRACE_TYPE_HANDLED;
167/* 253/*
168 * Context-switch trace line: 254 * Context-switch trace line:
169 255
@@ -175,34 +261,34 @@ verif_pid(struct trace_seq *s, pid_t pid, int cpu)
175 ret = trace_seq_printf(s, 261 ret = trace_seq_printf(s,
176 " ------------------------------------------\n"); 262 " ------------------------------------------\n");
177 if (!ret) 263 if (!ret)
178 TRACE_TYPE_PARTIAL_LINE; 264 return TRACE_TYPE_PARTIAL_LINE;
179 265
180 ret = print_graph_cpu(s, cpu); 266 ret = print_graph_cpu(s, cpu);
181 if (ret == TRACE_TYPE_PARTIAL_LINE) 267 if (ret == TRACE_TYPE_PARTIAL_LINE)
182 TRACE_TYPE_PARTIAL_LINE; 268 return TRACE_TYPE_PARTIAL_LINE;
183 269
184 ret = print_graph_proc(s, prev_pid); 270 ret = print_graph_proc(s, prev_pid);
185 if (ret == TRACE_TYPE_PARTIAL_LINE) 271 if (ret == TRACE_TYPE_PARTIAL_LINE)
186 TRACE_TYPE_PARTIAL_LINE; 272 return TRACE_TYPE_PARTIAL_LINE;
187 273
188 ret = trace_seq_printf(s, " => "); 274 ret = trace_seq_printf(s, " => ");
189 if (!ret) 275 if (!ret)
190 TRACE_TYPE_PARTIAL_LINE; 276 return TRACE_TYPE_PARTIAL_LINE;
191 277
192 ret = print_graph_proc(s, pid); 278 ret = print_graph_proc(s, pid);
193 if (ret == TRACE_TYPE_PARTIAL_LINE) 279 if (ret == TRACE_TYPE_PARTIAL_LINE)
194 TRACE_TYPE_PARTIAL_LINE; 280 return TRACE_TYPE_PARTIAL_LINE;
195 281
196 ret = trace_seq_printf(s, 282 ret = trace_seq_printf(s,
197 "\n ------------------------------------------\n\n"); 283 "\n ------------------------------------------\n\n");
198 if (!ret) 284 if (!ret)
199 TRACE_TYPE_PARTIAL_LINE; 285 return TRACE_TYPE_PARTIAL_LINE;
200 286
201 return ret; 287 return TRACE_TYPE_HANDLED;
202} 288}
203 289
204static bool 290static struct ftrace_graph_ret_entry *
205trace_branch_is_leaf(struct trace_iterator *iter, 291get_return_for_leaf(struct trace_iterator *iter,
206 struct ftrace_graph_ent_entry *curr) 292 struct ftrace_graph_ent_entry *curr)
207{ 293{
208 struct ring_buffer_iter *ring_iter; 294 struct ring_buffer_iter *ring_iter;
@@ -211,65 +297,123 @@ trace_branch_is_leaf(struct trace_iterator *iter,
211 297
212 ring_iter = iter->buffer_iter[iter->cpu]; 298 ring_iter = iter->buffer_iter[iter->cpu];
213 299
214 if (!ring_iter) 300 /* First peek to compare current entry and the next one */
215 return false; 301 if (ring_iter)
216 302 event = ring_buffer_iter_peek(ring_iter, NULL);
217 event = ring_buffer_iter_peek(ring_iter, NULL); 303 else {
304 /* We need to consume the current entry to see the next one */
305 ring_buffer_consume(iter->tr->buffer, iter->cpu, NULL);
306 event = ring_buffer_peek(iter->tr->buffer, iter->cpu,
307 NULL);
308 }
218 309
219 if (!event) 310 if (!event)
220 return false; 311 return NULL;
221 312
222 next = ring_buffer_event_data(event); 313 next = ring_buffer_event_data(event);
223 314
224 if (next->ent.type != TRACE_GRAPH_RET) 315 if (next->ent.type != TRACE_GRAPH_RET)
225 return false; 316 return NULL;
226 317
227 if (curr->ent.pid != next->ent.pid || 318 if (curr->ent.pid != next->ent.pid ||
228 curr->graph_ent.func != next->ret.func) 319 curr->graph_ent.func != next->ret.func)
229 return false; 320 return NULL;
321
322 /* this is a leaf, now advance the iterator */
323 if (ring_iter)
324 ring_buffer_read(ring_iter, NULL);
325
326 return next;
327}
328
329/* Signal a overhead of time execution to the output */
330static int
331print_graph_overhead(unsigned long long duration, struct trace_seq *s)
332{
333 /* If duration disappear, we don't need anything */
334 if (!(tracer_flags.val & TRACE_GRAPH_PRINT_DURATION))
335 return 1;
336
337 /* Non nested entry or return */
338 if (duration == -1)
339 return trace_seq_printf(s, " ");
340
341 if (tracer_flags.val & TRACE_GRAPH_PRINT_OVERHEAD) {
342 /* Duration exceeded 100 msecs */
343 if (duration > 100000ULL)
344 return trace_seq_printf(s, "! ");
345
346 /* Duration exceeded 10 msecs */
347 if (duration > 10000ULL)
348 return trace_seq_printf(s, "+ ");
349 }
230 350
231 return true; 351 return trace_seq_printf(s, " ");
352}
353
354static int print_graph_abs_time(u64 t, struct trace_seq *s)
355{
356 unsigned long usecs_rem;
357
358 usecs_rem = do_div(t, NSEC_PER_SEC);
359 usecs_rem /= 1000;
360
361 return trace_seq_printf(s, "%5lu.%06lu | ",
362 (unsigned long)t, usecs_rem);
232} 363}
233 364
234static enum print_line_t 365static enum print_line_t
235print_graph_irq(struct trace_seq *s, unsigned long addr, 366print_graph_irq(struct trace_iterator *iter, unsigned long addr,
236 enum trace_type type, int cpu, pid_t pid) 367 enum trace_type type, int cpu, pid_t pid)
237{ 368{
238 int ret; 369 int ret;
370 struct trace_seq *s = &iter->seq;
239 371
240 if (addr < (unsigned long)__irqentry_text_start || 372 if (addr < (unsigned long)__irqentry_text_start ||
241 addr >= (unsigned long)__irqentry_text_end) 373 addr >= (unsigned long)__irqentry_text_end)
242 return TRACE_TYPE_UNHANDLED; 374 return TRACE_TYPE_UNHANDLED;
243 375
244 if (type == TRACE_GRAPH_ENT) { 376 /* Absolute time */
245 ret = trace_seq_printf(s, "==========> | "); 377 if (tracer_flags.val & TRACE_GRAPH_PRINT_ABS_TIME) {
246 } else { 378 ret = print_graph_abs_time(iter->ts, s);
247 /* Cpu */ 379 if (!ret)
248 if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU) { 380 return TRACE_TYPE_PARTIAL_LINE;
249 ret = print_graph_cpu(s, cpu); 381 }
250 if (ret == TRACE_TYPE_PARTIAL_LINE)
251 return TRACE_TYPE_PARTIAL_LINE;
252 }
253 /* Proc */
254 if (tracer_flags.val & TRACE_GRAPH_PRINT_PROC) {
255 ret = print_graph_proc(s, pid);
256 if (ret == TRACE_TYPE_PARTIAL_LINE)
257 return TRACE_TYPE_PARTIAL_LINE;
258 382
259 ret = trace_seq_printf(s, " | "); 383 /* Cpu */
260 if (!ret) 384 if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU) {
261 return TRACE_TYPE_PARTIAL_LINE; 385 ret = print_graph_cpu(s, cpu);
262 } 386 if (ret == TRACE_TYPE_PARTIAL_LINE)
387 return TRACE_TYPE_PARTIAL_LINE;
388 }
389 /* Proc */
390 if (tracer_flags.val & TRACE_GRAPH_PRINT_PROC) {
391 ret = print_graph_proc(s, pid);
392 if (ret == TRACE_TYPE_PARTIAL_LINE)
393 return TRACE_TYPE_PARTIAL_LINE;
394 ret = trace_seq_printf(s, " | ");
395 if (!ret)
396 return TRACE_TYPE_PARTIAL_LINE;
397 }
263 398
264 /* No overhead */ 399 /* No overhead */
265 if (tracer_flags.val & TRACE_GRAPH_PRINT_OVERHEAD) { 400 ret = print_graph_overhead(-1, s);
266 ret = trace_seq_printf(s, " "); 401 if (!ret)
267 if (!ret) 402 return TRACE_TYPE_PARTIAL_LINE;
268 return TRACE_TYPE_PARTIAL_LINE; 403
269 } 404 if (type == TRACE_GRAPH_ENT)
405 ret = trace_seq_printf(s, "==========>");
406 else
407 ret = trace_seq_printf(s, "<==========");
408
409 if (!ret)
410 return TRACE_TYPE_PARTIAL_LINE;
411
412 /* Don't close the duration column if haven't one */
413 if (tracer_flags.val & TRACE_GRAPH_PRINT_DURATION)
414 trace_seq_printf(s, " |");
415 ret = trace_seq_printf(s, "\n");
270 416
271 ret = trace_seq_printf(s, "<========== |\n");
272 }
273 if (!ret) 417 if (!ret)
274 return TRACE_TYPE_PARTIAL_LINE; 418 return TRACE_TYPE_PARTIAL_LINE;
275 return TRACE_TYPE_HANDLED; 419 return TRACE_TYPE_HANDLED;
@@ -288,7 +432,7 @@ print_graph_duration(unsigned long long duration, struct trace_seq *s)
288 sprintf(msecs_str, "%lu", (unsigned long) duration); 432 sprintf(msecs_str, "%lu", (unsigned long) duration);
289 433
290 /* Print msecs */ 434 /* Print msecs */
291 ret = trace_seq_printf(s, msecs_str); 435 ret = trace_seq_printf(s, "%s", msecs_str);
292 if (!ret) 436 if (!ret)
293 return TRACE_TYPE_PARTIAL_LINE; 437 return TRACE_TYPE_PARTIAL_LINE;
294 438
@@ -321,51 +465,33 @@ print_graph_duration(unsigned long long duration, struct trace_seq *s)
321 465
322} 466}
323 467
324/* Signal a overhead of time execution to the output */
325static int
326print_graph_overhead(unsigned long long duration, struct trace_seq *s)
327{
328 /* Duration exceeded 100 msecs */
329 if (duration > 100000ULL)
330 return trace_seq_printf(s, "! ");
331
332 /* Duration exceeded 10 msecs */
333 if (duration > 10000ULL)
334 return trace_seq_printf(s, "+ ");
335
336 return trace_seq_printf(s, " ");
337}
338
339/* Case of a leaf function on its call entry */ 468/* Case of a leaf function on its call entry */
340static enum print_line_t 469static enum print_line_t
341print_graph_entry_leaf(struct trace_iterator *iter, 470print_graph_entry_leaf(struct trace_iterator *iter,
342 struct ftrace_graph_ent_entry *entry, struct trace_seq *s) 471 struct ftrace_graph_ent_entry *entry,
472 struct ftrace_graph_ret_entry *ret_entry, struct trace_seq *s)
343{ 473{
344 struct ftrace_graph_ret_entry *ret_entry;
345 struct ftrace_graph_ret *graph_ret; 474 struct ftrace_graph_ret *graph_ret;
346 struct ring_buffer_event *event;
347 struct ftrace_graph_ent *call; 475 struct ftrace_graph_ent *call;
348 unsigned long long duration; 476 unsigned long long duration;
349 int ret; 477 int ret;
350 int i; 478 int i;
351 479
352 event = ring_buffer_read(iter->buffer_iter[iter->cpu], NULL);
353 ret_entry = ring_buffer_event_data(event);
354 graph_ret = &ret_entry->ret; 480 graph_ret = &ret_entry->ret;
355 call = &entry->graph_ent; 481 call = &entry->graph_ent;
356 duration = graph_ret->rettime - graph_ret->calltime; 482 duration = graph_ret->rettime - graph_ret->calltime;
357 483
358 /* Overhead */ 484 /* Overhead */
359 if (tracer_flags.val & TRACE_GRAPH_PRINT_OVERHEAD) { 485 ret = print_graph_overhead(duration, s);
360 ret = print_graph_overhead(duration, s); 486 if (!ret)
361 if (!ret) 487 return TRACE_TYPE_PARTIAL_LINE;
362 return TRACE_TYPE_PARTIAL_LINE;
363 }
364 488
365 /* Duration */ 489 /* Duration */
366 ret = print_graph_duration(duration, s); 490 if (tracer_flags.val & TRACE_GRAPH_PRINT_DURATION) {
367 if (ret == TRACE_TYPE_PARTIAL_LINE) 491 ret = print_graph_duration(duration, s);
368 return TRACE_TYPE_PARTIAL_LINE; 492 if (ret == TRACE_TYPE_PARTIAL_LINE)
493 return TRACE_TYPE_PARTIAL_LINE;
494 }
369 495
370 /* Function */ 496 /* Function */
371 for (i = 0; i < call->depth * TRACE_GRAPH_INDENT; i++) { 497 for (i = 0; i < call->depth * TRACE_GRAPH_INDENT; i++) {
@@ -394,25 +520,17 @@ print_graph_entry_nested(struct ftrace_graph_ent_entry *entry,
394 struct ftrace_graph_ent *call = &entry->graph_ent; 520 struct ftrace_graph_ent *call = &entry->graph_ent;
395 521
396 /* No overhead */ 522 /* No overhead */
397 if (tracer_flags.val & TRACE_GRAPH_PRINT_OVERHEAD) { 523 ret = print_graph_overhead(-1, s);
398 ret = trace_seq_printf(s, " "); 524 if (!ret)
399 if (!ret) 525 return TRACE_TYPE_PARTIAL_LINE;
400 return TRACE_TYPE_PARTIAL_LINE;
401 }
402 526
403 /* Interrupt */ 527 /* No time */
404 ret = print_graph_irq(s, call->func, TRACE_GRAPH_ENT, cpu, pid); 528 if (tracer_flags.val & TRACE_GRAPH_PRINT_DURATION) {
405 if (ret == TRACE_TYPE_UNHANDLED) {
406 /* No time */
407 ret = trace_seq_printf(s, " | "); 529 ret = trace_seq_printf(s, " | ");
408 if (!ret) 530 if (!ret)
409 return TRACE_TYPE_PARTIAL_LINE; 531 return TRACE_TYPE_PARTIAL_LINE;
410 } else {
411 if (ret == TRACE_TYPE_PARTIAL_LINE)
412 return TRACE_TYPE_PARTIAL_LINE;
413 } 532 }
414 533
415
416 /* Function */ 534 /* Function */
417 for (i = 0; i < call->depth * TRACE_GRAPH_INDENT; i++) { 535 for (i = 0; i < call->depth * TRACE_GRAPH_INDENT; i++) {
418 ret = trace_seq_printf(s, " "); 536 ret = trace_seq_printf(s, " ");
@@ -428,20 +546,40 @@ print_graph_entry_nested(struct ftrace_graph_ent_entry *entry,
428 if (!ret) 546 if (!ret)
429 return TRACE_TYPE_PARTIAL_LINE; 547 return TRACE_TYPE_PARTIAL_LINE;
430 548
431 return TRACE_TYPE_HANDLED; 549 /*
550 * we already consumed the current entry to check the next one
551 * and see if this is a leaf.
552 */
553 return TRACE_TYPE_NO_CONSUME;
432} 554}
433 555
434static enum print_line_t 556static enum print_line_t
435print_graph_entry(struct ftrace_graph_ent_entry *field, struct trace_seq *s, 557print_graph_entry(struct ftrace_graph_ent_entry *field, struct trace_seq *s,
436 struct trace_iterator *iter, int cpu) 558 struct trace_iterator *iter)
437{ 559{
438 int ret; 560 int ret;
561 int cpu = iter->cpu;
562 pid_t *last_entry = iter->private;
439 struct trace_entry *ent = iter->ent; 563 struct trace_entry *ent = iter->ent;
564 struct ftrace_graph_ent *call = &field->graph_ent;
565 struct ftrace_graph_ret_entry *leaf_ret;
440 566
441 /* Pid */ 567 /* Pid */
442 if (verif_pid(s, ent->pid, cpu) == TRACE_TYPE_PARTIAL_LINE) 568 if (verif_pid(s, ent->pid, cpu, last_entry) == TRACE_TYPE_PARTIAL_LINE)
443 return TRACE_TYPE_PARTIAL_LINE; 569 return TRACE_TYPE_PARTIAL_LINE;
444 570
571 /* Interrupt */
572 ret = print_graph_irq(iter, call->func, TRACE_GRAPH_ENT, cpu, ent->pid);
573 if (ret == TRACE_TYPE_PARTIAL_LINE)
574 return TRACE_TYPE_PARTIAL_LINE;
575
576 /* Absolute time */
577 if (tracer_flags.val & TRACE_GRAPH_PRINT_ABS_TIME) {
578 ret = print_graph_abs_time(iter->ts, s);
579 if (!ret)
580 return TRACE_TYPE_PARTIAL_LINE;
581 }
582
445 /* Cpu */ 583 /* Cpu */
446 if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU) { 584 if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU) {
447 ret = print_graph_cpu(s, cpu); 585 ret = print_graph_cpu(s, cpu);
@@ -460,8 +598,9 @@ print_graph_entry(struct ftrace_graph_ent_entry *field, struct trace_seq *s,
460 return TRACE_TYPE_PARTIAL_LINE; 598 return TRACE_TYPE_PARTIAL_LINE;
461 } 599 }
462 600
463 if (trace_branch_is_leaf(iter, field)) 601 leaf_ret = get_return_for_leaf(iter, field);
464 return print_graph_entry_leaf(iter, field, s); 602 if (leaf_ret)
603 return print_graph_entry_leaf(iter, field, leaf_ret, s);
465 else 604 else
466 return print_graph_entry_nested(field, s, iter->ent->pid, cpu); 605 return print_graph_entry_nested(field, s, iter->ent->pid, cpu);
467 606
@@ -469,16 +608,25 @@ print_graph_entry(struct ftrace_graph_ent_entry *field, struct trace_seq *s,
469 608
470static enum print_line_t 609static enum print_line_t
471print_graph_return(struct ftrace_graph_ret *trace, struct trace_seq *s, 610print_graph_return(struct ftrace_graph_ret *trace, struct trace_seq *s,
472 struct trace_entry *ent, int cpu) 611 struct trace_entry *ent, struct trace_iterator *iter)
473{ 612{
474 int i; 613 int i;
475 int ret; 614 int ret;
615 int cpu = iter->cpu;
616 pid_t *last_pid = iter->private, pid = ent->pid;
476 unsigned long long duration = trace->rettime - trace->calltime; 617 unsigned long long duration = trace->rettime - trace->calltime;
477 618
478 /* Pid */ 619 /* Pid */
479 if (verif_pid(s, ent->pid, cpu) == TRACE_TYPE_PARTIAL_LINE) 620 if (verif_pid(s, pid, cpu, last_pid) == TRACE_TYPE_PARTIAL_LINE)
480 return TRACE_TYPE_PARTIAL_LINE; 621 return TRACE_TYPE_PARTIAL_LINE;
481 622
623 /* Absolute time */
624 if (tracer_flags.val & TRACE_GRAPH_PRINT_ABS_TIME) {
625 ret = print_graph_abs_time(iter->ts, s);
626 if (!ret)
627 return TRACE_TYPE_PARTIAL_LINE;
628 }
629
482 /* Cpu */ 630 /* Cpu */
483 if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU) { 631 if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU) {
484 ret = print_graph_cpu(s, cpu); 632 ret = print_graph_cpu(s, cpu);
@@ -498,16 +646,16 @@ print_graph_return(struct ftrace_graph_ret *trace, struct trace_seq *s,
498 } 646 }
499 647
500 /* Overhead */ 648 /* Overhead */
501 if (tracer_flags.val & TRACE_GRAPH_PRINT_OVERHEAD) { 649 ret = print_graph_overhead(duration, s);
502 ret = print_graph_overhead(duration, s); 650 if (!ret)
503 if (!ret) 651 return TRACE_TYPE_PARTIAL_LINE;
504 return TRACE_TYPE_PARTIAL_LINE;
505 }
506 652
507 /* Duration */ 653 /* Duration */
508 ret = print_graph_duration(duration, s); 654 if (tracer_flags.val & TRACE_GRAPH_PRINT_DURATION) {
509 if (ret == TRACE_TYPE_PARTIAL_LINE) 655 ret = print_graph_duration(duration, s);
510 return TRACE_TYPE_PARTIAL_LINE; 656 if (ret == TRACE_TYPE_PARTIAL_LINE)
657 return TRACE_TYPE_PARTIAL_LINE;
658 }
511 659
512 /* Closing brace */ 660 /* Closing brace */
513 for (i = 0; i < trace->depth * TRACE_GRAPH_INDENT; i++) { 661 for (i = 0; i < trace->depth * TRACE_GRAPH_INDENT; i++) {
@@ -528,7 +676,7 @@ print_graph_return(struct ftrace_graph_ret *trace, struct trace_seq *s,
528 return TRACE_TYPE_PARTIAL_LINE; 676 return TRACE_TYPE_PARTIAL_LINE;
529 } 677 }
530 678
531 ret = print_graph_irq(s, trace->func, TRACE_GRAPH_RET, cpu, ent->pid); 679 ret = print_graph_irq(iter, trace->func, TRACE_GRAPH_RET, cpu, pid);
532 if (ret == TRACE_TYPE_PARTIAL_LINE) 680 if (ret == TRACE_TYPE_PARTIAL_LINE)
533 return TRACE_TYPE_PARTIAL_LINE; 681 return TRACE_TYPE_PARTIAL_LINE;
534 682
@@ -541,14 +689,23 @@ print_graph_comment(struct print_entry *trace, struct trace_seq *s,
541{ 689{
542 int i; 690 int i;
543 int ret; 691 int ret;
692 int cpu = iter->cpu;
693 pid_t *last_pid = iter->private;
544 694
545 /* Pid */ 695 /* Pid */
546 if (verif_pid(s, ent->pid, iter->cpu) == TRACE_TYPE_PARTIAL_LINE) 696 if (verif_pid(s, ent->pid, cpu, last_pid) == TRACE_TYPE_PARTIAL_LINE)
547 return TRACE_TYPE_PARTIAL_LINE; 697 return TRACE_TYPE_PARTIAL_LINE;
548 698
699 /* Absolute time */
700 if (tracer_flags.val & TRACE_GRAPH_PRINT_ABS_TIME) {
701 ret = print_graph_abs_time(iter->ts, s);
702 if (!ret)
703 return TRACE_TYPE_PARTIAL_LINE;
704 }
705
549 /* Cpu */ 706 /* Cpu */
550 if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU) { 707 if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU) {
551 ret = print_graph_cpu(s, iter->cpu); 708 ret = print_graph_cpu(s, cpu);
552 if (ret == TRACE_TYPE_PARTIAL_LINE) 709 if (ret == TRACE_TYPE_PARTIAL_LINE)
553 return TRACE_TYPE_PARTIAL_LINE; 710 return TRACE_TYPE_PARTIAL_LINE;
554 } 711 }
@@ -565,17 +722,17 @@ print_graph_comment(struct print_entry *trace, struct trace_seq *s,
565 } 722 }
566 723
567 /* No overhead */ 724 /* No overhead */
568 if (tracer_flags.val & TRACE_GRAPH_PRINT_OVERHEAD) { 725 ret = print_graph_overhead(-1, s);
569 ret = trace_seq_printf(s, " "); 726 if (!ret)
727 return TRACE_TYPE_PARTIAL_LINE;
728
729 /* No time */
730 if (tracer_flags.val & TRACE_GRAPH_PRINT_DURATION) {
731 ret = trace_seq_printf(s, " | ");
570 if (!ret) 732 if (!ret)
571 return TRACE_TYPE_PARTIAL_LINE; 733 return TRACE_TYPE_PARTIAL_LINE;
572 } 734 }
573 735
574 /* No time */
575 ret = trace_seq_printf(s, " | ");
576 if (!ret)
577 return TRACE_TYPE_PARTIAL_LINE;
578
579 /* Indentation */ 736 /* Indentation */
580 if (trace->depth > 0) 737 if (trace->depth > 0)
581 for (i = 0; i < (trace->depth + 1) * TRACE_GRAPH_INDENT; i++) { 738 for (i = 0; i < (trace->depth + 1) * TRACE_GRAPH_INDENT; i++) {
@@ -589,8 +746,11 @@ print_graph_comment(struct print_entry *trace, struct trace_seq *s,
589 if (!ret) 746 if (!ret)
590 return TRACE_TYPE_PARTIAL_LINE; 747 return TRACE_TYPE_PARTIAL_LINE;
591 748
592 if (ent->flags & TRACE_FLAG_CONT) 749 /* Strip ending newline */
593 trace_seq_print_cont(s, iter); 750 if (s->buffer[s->len - 1] == '\n') {
751 s->buffer[s->len - 1] = '\0';
752 s->len--;
753 }
594 754
595 ret = trace_seq_printf(s, " */\n"); 755 ret = trace_seq_printf(s, " */\n");
596 if (!ret) 756 if (!ret)
@@ -610,13 +770,12 @@ print_graph_function(struct trace_iterator *iter)
610 case TRACE_GRAPH_ENT: { 770 case TRACE_GRAPH_ENT: {
611 struct ftrace_graph_ent_entry *field; 771 struct ftrace_graph_ent_entry *field;
612 trace_assign_type(field, entry); 772 trace_assign_type(field, entry);
613 return print_graph_entry(field, s, iter, 773 return print_graph_entry(field, s, iter);
614 iter->cpu);
615 } 774 }
616 case TRACE_GRAPH_RET: { 775 case TRACE_GRAPH_RET: {
617 struct ftrace_graph_ret_entry *field; 776 struct ftrace_graph_ret_entry *field;
618 trace_assign_type(field, entry); 777 trace_assign_type(field, entry);
619 return print_graph_return(&field->ret, s, entry, iter->cpu); 778 return print_graph_return(&field->ret, s, entry, iter);
620 } 779 }
621 case TRACE_PRINT: { 780 case TRACE_PRINT: {
622 struct print_entry *field; 781 struct print_entry *field;
@@ -632,33 +791,64 @@ static void print_graph_headers(struct seq_file *s)
632{ 791{
633 /* 1st line */ 792 /* 1st line */
634 seq_printf(s, "# "); 793 seq_printf(s, "# ");
794 if (tracer_flags.val & TRACE_GRAPH_PRINT_ABS_TIME)
795 seq_printf(s, " TIME ");
635 if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU) 796 if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU)
636 seq_printf(s, "CPU "); 797 seq_printf(s, "CPU");
637 if (tracer_flags.val & TRACE_GRAPH_PRINT_PROC) 798 if (tracer_flags.val & TRACE_GRAPH_PRINT_PROC)
638 seq_printf(s, "TASK/PID "); 799 seq_printf(s, " TASK/PID ");
639 if (tracer_flags.val & TRACE_GRAPH_PRINT_OVERHEAD) 800 if (tracer_flags.val & TRACE_GRAPH_PRINT_DURATION)
640 seq_printf(s, "OVERHEAD/"); 801 seq_printf(s, " DURATION ");
641 seq_printf(s, "DURATION FUNCTION CALLS\n"); 802 seq_printf(s, " FUNCTION CALLS\n");
642 803
643 /* 2nd line */ 804 /* 2nd line */
644 seq_printf(s, "# "); 805 seq_printf(s, "# ");
806 if (tracer_flags.val & TRACE_GRAPH_PRINT_ABS_TIME)
807 seq_printf(s, " | ");
645 if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU) 808 if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU)
646 seq_printf(s, "| "); 809 seq_printf(s, "| ");
647 if (tracer_flags.val & TRACE_GRAPH_PRINT_PROC) 810 if (tracer_flags.val & TRACE_GRAPH_PRINT_PROC)
648 seq_printf(s, "| | "); 811 seq_printf(s, " | | ");
649 if (tracer_flags.val & TRACE_GRAPH_PRINT_OVERHEAD) { 812 if (tracer_flags.val & TRACE_GRAPH_PRINT_DURATION)
650 seq_printf(s, "| "); 813 seq_printf(s, " | | ");
651 seq_printf(s, "| | | | |\n"); 814 seq_printf(s, " | | | |\n");
652 } else
653 seq_printf(s, " | | | | |\n");
654} 815}
816
817static void graph_trace_open(struct trace_iterator *iter)
818{
819 /* pid on the last trace processed */
820 pid_t *last_pid = alloc_percpu(pid_t);
821 int cpu;
822
823 if (!last_pid)
824 pr_warning("function graph tracer: not enough memory\n");
825 else
826 for_each_possible_cpu(cpu) {
827 pid_t *pid = per_cpu_ptr(last_pid, cpu);
828 *pid = -1;
829 }
830
831 iter->private = last_pid;
832}
833
834static void graph_trace_close(struct trace_iterator *iter)
835{
836 percpu_free(iter->private);
837}
838
655static struct tracer graph_trace __read_mostly = { 839static struct tracer graph_trace __read_mostly = {
656 .name = "function_graph", 840 .name = "function_graph",
841 .open = graph_trace_open,
842 .close = graph_trace_close,
843 .wait_pipe = poll_wait_pipe,
657 .init = graph_trace_init, 844 .init = graph_trace_init,
658 .reset = graph_trace_reset, 845 .reset = graph_trace_reset,
659 .print_line = print_graph_function, 846 .print_line = print_graph_function,
660 .print_header = print_graph_headers, 847 .print_header = print_graph_headers,
661 .flags = &tracer_flags, 848 .flags = &tracer_flags,
849#ifdef CONFIG_FTRACE_SELFTEST
850 .selftest = trace_selftest_startup_function_graph,
851#endif
662}; 852};
663 853
664static __init int init_graph_trace(void) 854static __init int init_graph_trace(void)
diff --git a/kernel/trace/trace_hw_branches.c b/kernel/trace/trace_hw_branches.c
index 649df22d435f..3561aace075c 100644
--- a/kernel/trace/trace_hw_branches.c
+++ b/kernel/trace/trace_hw_branches.c
@@ -1,7 +1,8 @@
1/* 1/*
2 * h/w branch tracer for x86 based on bts 2 * h/w branch tracer for x86 based on bts
3 * 3 *
4 * Copyright (C) 2008 Markus Metzger <markus.t.metzger@gmail.com> 4 * Copyright (C) 2008-2009 Intel Corporation.
5 * Markus Metzger <markus.t.metzger@gmail.com>, 2008-2009
5 * 6 *
6 */ 7 */
7 8
@@ -10,21 +11,44 @@
10#include <linux/debugfs.h> 11#include <linux/debugfs.h>
11#include <linux/ftrace.h> 12#include <linux/ftrace.h>
12#include <linux/kallsyms.h> 13#include <linux/kallsyms.h>
14#include <linux/mutex.h>
15#include <linux/cpu.h>
16#include <linux/smp.h>
13 17
14#include <asm/ds.h> 18#include <asm/ds.h>
15 19
16#include "trace.h" 20#include "trace.h"
21#include "trace_output.h"
17 22
18 23
19#define SIZEOF_BTS (1 << 13) 24#define SIZEOF_BTS (1 << 13)
20 25
26/* The tracer mutex protects the below per-cpu tracer array.
27 It needs to be held to:
28 - start tracing on all cpus
29 - stop tracing on all cpus
30 - start tracing on a single hotplug cpu
31 - stop tracing on a single hotplug cpu
32 - read the trace from all cpus
33 - read the trace from a single cpu
34*/
35static DEFINE_MUTEX(bts_tracer_mutex);
21static DEFINE_PER_CPU(struct bts_tracer *, tracer); 36static DEFINE_PER_CPU(struct bts_tracer *, tracer);
22static DEFINE_PER_CPU(unsigned char[SIZEOF_BTS], buffer); 37static DEFINE_PER_CPU(unsigned char[SIZEOF_BTS], buffer);
23 38
24#define this_tracer per_cpu(tracer, smp_processor_id()) 39#define this_tracer per_cpu(tracer, smp_processor_id())
25#define this_buffer per_cpu(buffer, smp_processor_id()) 40#define this_buffer per_cpu(buffer, smp_processor_id())
26 41
42static int __read_mostly trace_hw_branches_enabled;
43static struct trace_array *hw_branch_trace __read_mostly;
27 44
45
46/*
47 * Start tracing on the current cpu.
48 * The argument is ignored.
49 *
50 * pre: bts_tracer_mutex must be locked.
51 */
28static void bts_trace_start_cpu(void *arg) 52static void bts_trace_start_cpu(void *arg)
29{ 53{
30 if (this_tracer) 54 if (this_tracer)
@@ -42,14 +66,20 @@ static void bts_trace_start_cpu(void *arg)
42 66
43static void bts_trace_start(struct trace_array *tr) 67static void bts_trace_start(struct trace_array *tr)
44{ 68{
45 int cpu; 69 mutex_lock(&bts_tracer_mutex);
46 70
47 tracing_reset_online_cpus(tr); 71 on_each_cpu(bts_trace_start_cpu, NULL, 1);
72 trace_hw_branches_enabled = 1;
48 73
49 for_each_cpu(cpu, cpu_possible_mask) 74 mutex_unlock(&bts_tracer_mutex);
50 smp_call_function_single(cpu, bts_trace_start_cpu, NULL, 1);
51} 75}
52 76
77/*
78 * Stop tracing on the current cpu.
79 * The argument is ignored.
80 *
81 * pre: bts_tracer_mutex must be locked.
82 */
53static void bts_trace_stop_cpu(void *arg) 83static void bts_trace_stop_cpu(void *arg)
54{ 84{
55 if (this_tracer) { 85 if (this_tracer) {
@@ -60,26 +90,62 @@ static void bts_trace_stop_cpu(void *arg)
60 90
61static void bts_trace_stop(struct trace_array *tr) 91static void bts_trace_stop(struct trace_array *tr)
62{ 92{
63 int cpu; 93 mutex_lock(&bts_tracer_mutex);
94
95 trace_hw_branches_enabled = 0;
96 on_each_cpu(bts_trace_stop_cpu, NULL, 1);
97
98 mutex_unlock(&bts_tracer_mutex);
99}
100
101static int __cpuinit bts_hotcpu_handler(struct notifier_block *nfb,
102 unsigned long action, void *hcpu)
103{
104 unsigned int cpu = (unsigned long)hcpu;
105
106 mutex_lock(&bts_tracer_mutex);
107
108 if (!trace_hw_branches_enabled)
109 goto out;
64 110
65 for_each_cpu(cpu, cpu_possible_mask) 111 switch (action) {
112 case CPU_ONLINE:
113 case CPU_DOWN_FAILED:
114 smp_call_function_single(cpu, bts_trace_start_cpu, NULL, 1);
115 break;
116 case CPU_DOWN_PREPARE:
66 smp_call_function_single(cpu, bts_trace_stop_cpu, NULL, 1); 117 smp_call_function_single(cpu, bts_trace_stop_cpu, NULL, 1);
118 break;
119 }
120
121 out:
122 mutex_unlock(&bts_tracer_mutex);
123 return NOTIFY_DONE;
67} 124}
68 125
69static int bts_trace_init(struct trace_array *tr) 126static struct notifier_block bts_hotcpu_notifier __cpuinitdata = {
127 .notifier_call = bts_hotcpu_handler
128};
129
130static int __cpuinit bts_trace_init(struct trace_array *tr)
70{ 131{
71 tracing_reset_online_cpus(tr); 132 hw_branch_trace = tr;
133
134 register_hotcpu_notifier(&bts_hotcpu_notifier);
72 bts_trace_start(tr); 135 bts_trace_start(tr);
73 136
74 return 0; 137 return 0;
75} 138}
76 139
140static void __cpuinit bts_trace_reset(struct trace_array *tr)
141{
142 bts_trace_stop(tr);
143 unregister_hotcpu_notifier(&bts_hotcpu_notifier);
144}
145
77static void bts_trace_print_header(struct seq_file *m) 146static void bts_trace_print_header(struct seq_file *m)
78{ 147{
79 seq_puts(m, 148 seq_puts(m, "# CPU# TO <- FROM\n");
80 "# CPU# FROM TO FUNCTION\n");
81 seq_puts(m,
82 "# | | | |\n");
83} 149}
84 150
85static enum print_line_t bts_trace_print_line(struct trace_iterator *iter) 151static enum print_line_t bts_trace_print_line(struct trace_iterator *iter)
@@ -87,15 +153,15 @@ static enum print_line_t bts_trace_print_line(struct trace_iterator *iter)
87 struct trace_entry *entry = iter->ent; 153 struct trace_entry *entry = iter->ent;
88 struct trace_seq *seq = &iter->seq; 154 struct trace_seq *seq = &iter->seq;
89 struct hw_branch_entry *it; 155 struct hw_branch_entry *it;
156 unsigned long symflags = TRACE_ITER_SYM_OFFSET;
90 157
91 trace_assign_type(it, entry); 158 trace_assign_type(it, entry);
92 159
93 if (entry->type == TRACE_HW_BRANCHES) { 160 if (entry->type == TRACE_HW_BRANCHES) {
94 if (trace_seq_printf(seq, "%4d ", entry->cpu) && 161 if (trace_seq_printf(seq, "%4d ", iter->cpu) &&
95 trace_seq_printf(seq, "0x%016llx -> 0x%016llx ", 162 seq_print_ip_sym(seq, it->to, symflags) &&
96 it->from, it->to) && 163 trace_seq_printf(seq, "\t <- ") &&
97 (!it->from || 164 seq_print_ip_sym(seq, it->from, symflags) &&
98 seq_print_ip_sym(seq, it->from, /* sym_flags = */ 0)) &&
99 trace_seq_printf(seq, "\n")) 165 trace_seq_printf(seq, "\n"))
100 return TRACE_TYPE_HANDLED; 166 return TRACE_TYPE_HANDLED;
101 return TRACE_TYPE_PARTIAL_LINE;; 167 return TRACE_TYPE_PARTIAL_LINE;;
@@ -103,26 +169,42 @@ static enum print_line_t bts_trace_print_line(struct trace_iterator *iter)
103 return TRACE_TYPE_UNHANDLED; 169 return TRACE_TYPE_UNHANDLED;
104} 170}
105 171
106void trace_hw_branch(struct trace_array *tr, u64 from, u64 to) 172void trace_hw_branch(u64 from, u64 to)
107{ 173{
174 struct trace_array *tr = hw_branch_trace;
108 struct ring_buffer_event *event; 175 struct ring_buffer_event *event;
109 struct hw_branch_entry *entry; 176 struct hw_branch_entry *entry;
110 unsigned long irq; 177 unsigned long irq1;
178 int cpu;
111 179
112 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry), &irq); 180 if (unlikely(!tr))
113 if (!event)
114 return; 181 return;
182
183 if (unlikely(!trace_hw_branches_enabled))
184 return;
185
186 local_irq_save(irq1);
187 cpu = raw_smp_processor_id();
188 if (atomic_inc_return(&tr->data[cpu]->disabled) != 1)
189 goto out;
190
191 event = trace_buffer_lock_reserve(tr, TRACE_HW_BRANCHES,
192 sizeof(*entry), 0, 0);
193 if (!event)
194 goto out;
115 entry = ring_buffer_event_data(event); 195 entry = ring_buffer_event_data(event);
116 tracing_generic_entry_update(&entry->ent, 0, from); 196 tracing_generic_entry_update(&entry->ent, 0, from);
117 entry->ent.type = TRACE_HW_BRANCHES; 197 entry->ent.type = TRACE_HW_BRANCHES;
118 entry->ent.cpu = smp_processor_id();
119 entry->from = from; 198 entry->from = from;
120 entry->to = to; 199 entry->to = to;
121 ring_buffer_unlock_commit(tr->buffer, event, irq); 200 trace_buffer_unlock_commit(tr, event, 0, 0);
201
202 out:
203 atomic_dec(&tr->data[cpu]->disabled);
204 local_irq_restore(irq1);
122} 205}
123 206
124static void trace_bts_at(struct trace_array *tr, 207static void trace_bts_at(const struct bts_trace *trace, void *at)
125 const struct bts_trace *trace, void *at)
126{ 208{
127 struct bts_struct bts; 209 struct bts_struct bts;
128 int err = 0; 210 int err = 0;
@@ -137,18 +219,29 @@ static void trace_bts_at(struct trace_array *tr,
137 219
138 switch (bts.qualifier) { 220 switch (bts.qualifier) {
139 case BTS_BRANCH: 221 case BTS_BRANCH:
140 trace_hw_branch(tr, bts.variant.lbr.from, bts.variant.lbr.to); 222 trace_hw_branch(bts.variant.lbr.from, bts.variant.lbr.to);
141 break; 223 break;
142 } 224 }
143} 225}
144 226
227/*
228 * Collect the trace on the current cpu and write it into the ftrace buffer.
229 *
230 * pre: bts_tracer_mutex must be locked
231 */
145static void trace_bts_cpu(void *arg) 232static void trace_bts_cpu(void *arg)
146{ 233{
147 struct trace_array *tr = (struct trace_array *) arg; 234 struct trace_array *tr = (struct trace_array *) arg;
148 const struct bts_trace *trace; 235 const struct bts_trace *trace;
149 unsigned char *at; 236 unsigned char *at;
150 237
151 if (!this_tracer) 238 if (unlikely(!tr))
239 return;
240
241 if (unlikely(atomic_read(&tr->data[raw_smp_processor_id()]->disabled)))
242 return;
243
244 if (unlikely(!this_tracer))
152 return; 245 return;
153 246
154 ds_suspend_bts(this_tracer); 247 ds_suspend_bts(this_tracer);
@@ -158,11 +251,11 @@ static void trace_bts_cpu(void *arg)
158 251
159 for (at = trace->ds.top; (void *)at < trace->ds.end; 252 for (at = trace->ds.top; (void *)at < trace->ds.end;
160 at += trace->ds.size) 253 at += trace->ds.size)
161 trace_bts_at(tr, trace, at); 254 trace_bts_at(trace, at);
162 255
163 for (at = trace->ds.begin; (void *)at < trace->ds.top; 256 for (at = trace->ds.begin; (void *)at < trace->ds.top;
164 at += trace->ds.size) 257 at += trace->ds.size)
165 trace_bts_at(tr, trace, at); 258 trace_bts_at(trace, at);
166 259
167out: 260out:
168 ds_resume_bts(this_tracer); 261 ds_resume_bts(this_tracer);
@@ -170,22 +263,38 @@ out:
170 263
171static void trace_bts_prepare(struct trace_iterator *iter) 264static void trace_bts_prepare(struct trace_iterator *iter)
172{ 265{
173 int cpu; 266 mutex_lock(&bts_tracer_mutex);
267
268 on_each_cpu(trace_bts_cpu, iter->tr, 1);
269
270 mutex_unlock(&bts_tracer_mutex);
271}
272
273static void trace_bts_close(struct trace_iterator *iter)
274{
275 tracing_reset_online_cpus(iter->tr);
276}
277
278void trace_hw_branch_oops(void)
279{
280 mutex_lock(&bts_tracer_mutex);
281
282 trace_bts_cpu(hw_branch_trace);
174 283
175 for_each_cpu(cpu, cpu_possible_mask) 284 mutex_unlock(&bts_tracer_mutex);
176 smp_call_function_single(cpu, trace_bts_cpu, iter->tr, 1);
177} 285}
178 286
179struct tracer bts_tracer __read_mostly = 287struct tracer bts_tracer __read_mostly =
180{ 288{
181 .name = "hw-branch-tracer", 289 .name = "hw-branch-tracer",
182 .init = bts_trace_init, 290 .init = bts_trace_init,
183 .reset = bts_trace_stop, 291 .reset = bts_trace_reset,
184 .print_header = bts_trace_print_header, 292 .print_header = bts_trace_print_header,
185 .print_line = bts_trace_print_line, 293 .print_line = bts_trace_print_line,
186 .start = bts_trace_start, 294 .start = bts_trace_start,
187 .stop = bts_trace_stop, 295 .stop = bts_trace_stop,
188 .open = trace_bts_prepare 296 .open = trace_bts_prepare,
297 .close = trace_bts_close
189}; 298};
190 299
191__init static int init_bts_trace(void) 300__init static int init_bts_trace(void)
diff --git a/kernel/trace/trace_irqsoff.c b/kernel/trace/trace_irqsoff.c
index 62a78d943534..9e5ebd844158 100644
--- a/kernel/trace/trace_irqsoff.c
+++ b/kernel/trace/trace_irqsoff.c
@@ -1,5 +1,5 @@
1/* 1/*
2 * trace irqs off criticall timings 2 * trace irqs off critical timings
3 * 3 *
4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> 4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com> 5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
@@ -95,7 +95,7 @@ irqsoff_tracer_call(unsigned long ip, unsigned long parent_ip)
95 disabled = atomic_inc_return(&data->disabled); 95 disabled = atomic_inc_return(&data->disabled);
96 96
97 if (likely(disabled == 1)) 97 if (likely(disabled == 1))
98 trace_function(tr, data, ip, parent_ip, flags, preempt_count()); 98 trace_function(tr, ip, parent_ip, flags, preempt_count());
99 99
100 atomic_dec(&data->disabled); 100 atomic_dec(&data->disabled);
101} 101}
@@ -153,7 +153,7 @@ check_critical_timing(struct trace_array *tr,
153 if (!report_latency(delta)) 153 if (!report_latency(delta))
154 goto out_unlock; 154 goto out_unlock;
155 155
156 trace_function(tr, data, CALLER_ADDR0, parent_ip, flags, pc); 156 trace_function(tr, CALLER_ADDR0, parent_ip, flags, pc);
157 157
158 latency = nsecs_to_usecs(delta); 158 latency = nsecs_to_usecs(delta);
159 159
@@ -177,7 +177,7 @@ out:
177 data->critical_sequence = max_sequence; 177 data->critical_sequence = max_sequence;
178 data->preempt_timestamp = ftrace_now(cpu); 178 data->preempt_timestamp = ftrace_now(cpu);
179 tracing_reset(tr, cpu); 179 tracing_reset(tr, cpu);
180 trace_function(tr, data, CALLER_ADDR0, parent_ip, flags, pc); 180 trace_function(tr, CALLER_ADDR0, parent_ip, flags, pc);
181} 181}
182 182
183static inline void 183static inline void
@@ -210,7 +210,7 @@ start_critical_timing(unsigned long ip, unsigned long parent_ip)
210 210
211 local_save_flags(flags); 211 local_save_flags(flags);
212 212
213 trace_function(tr, data, ip, parent_ip, flags, preempt_count()); 213 trace_function(tr, ip, parent_ip, flags, preempt_count());
214 214
215 per_cpu(tracing_cpu, cpu) = 1; 215 per_cpu(tracing_cpu, cpu) = 1;
216 216
@@ -244,7 +244,7 @@ stop_critical_timing(unsigned long ip, unsigned long parent_ip)
244 atomic_inc(&data->disabled); 244 atomic_inc(&data->disabled);
245 245
246 local_save_flags(flags); 246 local_save_flags(flags);
247 trace_function(tr, data, ip, parent_ip, flags, preempt_count()); 247 trace_function(tr, ip, parent_ip, flags, preempt_count());
248 check_critical_timing(tr, data, parent_ip ? : ip, cpu); 248 check_critical_timing(tr, data, parent_ip ? : ip, cpu);
249 data->critical_start = 0; 249 data->critical_start = 0;
250 atomic_dec(&data->disabled); 250 atomic_dec(&data->disabled);
@@ -353,28 +353,18 @@ void trace_preempt_off(unsigned long a0, unsigned long a1)
353} 353}
354#endif /* CONFIG_PREEMPT_TRACER */ 354#endif /* CONFIG_PREEMPT_TRACER */
355 355
356/*
357 * save_tracer_enabled is used to save the state of the tracer_enabled
358 * variable when we disable it when we open a trace output file.
359 */
360static int save_tracer_enabled;
361
362static void start_irqsoff_tracer(struct trace_array *tr) 356static void start_irqsoff_tracer(struct trace_array *tr)
363{ 357{
364 register_ftrace_function(&trace_ops); 358 register_ftrace_function(&trace_ops);
365 if (tracing_is_enabled()) { 359 if (tracing_is_enabled())
366 tracer_enabled = 1; 360 tracer_enabled = 1;
367 save_tracer_enabled = 1; 361 else
368 } else {
369 tracer_enabled = 0; 362 tracer_enabled = 0;
370 save_tracer_enabled = 0;
371 }
372} 363}
373 364
374static void stop_irqsoff_tracer(struct trace_array *tr) 365static void stop_irqsoff_tracer(struct trace_array *tr)
375{ 366{
376 tracer_enabled = 0; 367 tracer_enabled = 0;
377 save_tracer_enabled = 0;
378 unregister_ftrace_function(&trace_ops); 368 unregister_ftrace_function(&trace_ops);
379} 369}
380 370
@@ -395,25 +385,11 @@ static void irqsoff_tracer_reset(struct trace_array *tr)
395static void irqsoff_tracer_start(struct trace_array *tr) 385static void irqsoff_tracer_start(struct trace_array *tr)
396{ 386{
397 tracer_enabled = 1; 387 tracer_enabled = 1;
398 save_tracer_enabled = 1;
399} 388}
400 389
401static void irqsoff_tracer_stop(struct trace_array *tr) 390static void irqsoff_tracer_stop(struct trace_array *tr)
402{ 391{
403 tracer_enabled = 0; 392 tracer_enabled = 0;
404 save_tracer_enabled = 0;
405}
406
407static void irqsoff_tracer_open(struct trace_iterator *iter)
408{
409 /* stop the trace while dumping */
410 tracer_enabled = 0;
411}
412
413static void irqsoff_tracer_close(struct trace_iterator *iter)
414{
415 /* restart tracing */
416 tracer_enabled = save_tracer_enabled;
417} 393}
418 394
419#ifdef CONFIG_IRQSOFF_TRACER 395#ifdef CONFIG_IRQSOFF_TRACER
@@ -431,8 +407,6 @@ static struct tracer irqsoff_tracer __read_mostly =
431 .reset = irqsoff_tracer_reset, 407 .reset = irqsoff_tracer_reset,
432 .start = irqsoff_tracer_start, 408 .start = irqsoff_tracer_start,
433 .stop = irqsoff_tracer_stop, 409 .stop = irqsoff_tracer_stop,
434 .open = irqsoff_tracer_open,
435 .close = irqsoff_tracer_close,
436 .print_max = 1, 410 .print_max = 1,
437#ifdef CONFIG_FTRACE_SELFTEST 411#ifdef CONFIG_FTRACE_SELFTEST
438 .selftest = trace_selftest_startup_irqsoff, 412 .selftest = trace_selftest_startup_irqsoff,
@@ -459,8 +433,6 @@ static struct tracer preemptoff_tracer __read_mostly =
459 .reset = irqsoff_tracer_reset, 433 .reset = irqsoff_tracer_reset,
460 .start = irqsoff_tracer_start, 434 .start = irqsoff_tracer_start,
461 .stop = irqsoff_tracer_stop, 435 .stop = irqsoff_tracer_stop,
462 .open = irqsoff_tracer_open,
463 .close = irqsoff_tracer_close,
464 .print_max = 1, 436 .print_max = 1,
465#ifdef CONFIG_FTRACE_SELFTEST 437#ifdef CONFIG_FTRACE_SELFTEST
466 .selftest = trace_selftest_startup_preemptoff, 438 .selftest = trace_selftest_startup_preemptoff,
@@ -489,8 +461,6 @@ static struct tracer preemptirqsoff_tracer __read_mostly =
489 .reset = irqsoff_tracer_reset, 461 .reset = irqsoff_tracer_reset,
490 .start = irqsoff_tracer_start, 462 .start = irqsoff_tracer_start,
491 .stop = irqsoff_tracer_stop, 463 .stop = irqsoff_tracer_stop,
492 .open = irqsoff_tracer_open,
493 .close = irqsoff_tracer_close,
494 .print_max = 1, 464 .print_max = 1,
495#ifdef CONFIG_FTRACE_SELFTEST 465#ifdef CONFIG_FTRACE_SELFTEST
496 .selftest = trace_selftest_startup_preemptirqsoff, 466 .selftest = trace_selftest_startup_preemptirqsoff,
diff --git a/kernel/trace/trace_mmiotrace.c b/kernel/trace/trace_mmiotrace.c
index 80e503ef6136..c401b908e805 100644
--- a/kernel/trace/trace_mmiotrace.c
+++ b/kernel/trace/trace_mmiotrace.c
@@ -12,6 +12,7 @@
12#include <asm/atomic.h> 12#include <asm/atomic.h>
13 13
14#include "trace.h" 14#include "trace.h"
15#include "trace_output.h"
15 16
16struct header_iter { 17struct header_iter {
17 struct pci_dev *dev; 18 struct pci_dev *dev;
@@ -183,21 +184,22 @@ static enum print_line_t mmio_print_rw(struct trace_iterator *iter)
183 switch (rw->opcode) { 184 switch (rw->opcode) {
184 case MMIO_READ: 185 case MMIO_READ:
185 ret = trace_seq_printf(s, 186 ret = trace_seq_printf(s,
186 "R %d %lu.%06lu %d 0x%llx 0x%lx 0x%lx %d\n", 187 "R %d %u.%06lu %d 0x%llx 0x%lx 0x%lx %d\n",
187 rw->width, secs, usec_rem, rw->map_id, 188 rw->width, secs, usec_rem, rw->map_id,
188 (unsigned long long)rw->phys, 189 (unsigned long long)rw->phys,
189 rw->value, rw->pc, 0); 190 rw->value, rw->pc, 0);
190 break; 191 break;
191 case MMIO_WRITE: 192 case MMIO_WRITE:
192 ret = trace_seq_printf(s, 193 ret = trace_seq_printf(s,
193 "W %d %lu.%06lu %d 0x%llx 0x%lx 0x%lx %d\n", 194 "W %d %u.%06lu %d 0x%llx 0x%lx 0x%lx %d\n",
194 rw->width, secs, usec_rem, rw->map_id, 195 rw->width, secs, usec_rem, rw->map_id,
195 (unsigned long long)rw->phys, 196 (unsigned long long)rw->phys,
196 rw->value, rw->pc, 0); 197 rw->value, rw->pc, 0);
197 break; 198 break;
198 case MMIO_UNKNOWN_OP: 199 case MMIO_UNKNOWN_OP:
199 ret = trace_seq_printf(s, 200 ret = trace_seq_printf(s,
200 "UNKNOWN %lu.%06lu %d 0x%llx %02x,%02x,%02x 0x%lx %d\n", 201 "UNKNOWN %u.%06lu %d 0x%llx %02lx,%02lx,"
202 "%02lx 0x%lx %d\n",
201 secs, usec_rem, rw->map_id, 203 secs, usec_rem, rw->map_id,
202 (unsigned long long)rw->phys, 204 (unsigned long long)rw->phys,
203 (rw->value >> 16) & 0xff, (rw->value >> 8) & 0xff, 205 (rw->value >> 16) & 0xff, (rw->value >> 8) & 0xff,
@@ -229,14 +231,14 @@ static enum print_line_t mmio_print_map(struct trace_iterator *iter)
229 switch (m->opcode) { 231 switch (m->opcode) {
230 case MMIO_PROBE: 232 case MMIO_PROBE:
231 ret = trace_seq_printf(s, 233 ret = trace_seq_printf(s,
232 "MAP %lu.%06lu %d 0x%llx 0x%lx 0x%lx 0x%lx %d\n", 234 "MAP %u.%06lu %d 0x%llx 0x%lx 0x%lx 0x%lx %d\n",
233 secs, usec_rem, m->map_id, 235 secs, usec_rem, m->map_id,
234 (unsigned long long)m->phys, m->virt, m->len, 236 (unsigned long long)m->phys, m->virt, m->len,
235 0UL, 0); 237 0UL, 0);
236 break; 238 break;
237 case MMIO_UNPROBE: 239 case MMIO_UNPROBE:
238 ret = trace_seq_printf(s, 240 ret = trace_seq_printf(s,
239 "UNMAP %lu.%06lu %d 0x%lx %d\n", 241 "UNMAP %u.%06lu %d 0x%lx %d\n",
240 secs, usec_rem, m->map_id, 0UL, 0); 242 secs, usec_rem, m->map_id, 0UL, 0);
241 break; 243 break;
242 default: 244 default:
@@ -260,13 +262,10 @@ static enum print_line_t mmio_print_mark(struct trace_iterator *iter)
260 int ret; 262 int ret;
261 263
262 /* The trailing newline must be in the message. */ 264 /* The trailing newline must be in the message. */
263 ret = trace_seq_printf(s, "MARK %lu.%06lu %s", secs, usec_rem, msg); 265 ret = trace_seq_printf(s, "MARK %u.%06lu %s", secs, usec_rem, msg);
264 if (!ret) 266 if (!ret)
265 return TRACE_TYPE_PARTIAL_LINE; 267 return TRACE_TYPE_PARTIAL_LINE;
266 268
267 if (entry->flags & TRACE_FLAG_CONT)
268 trace_seq_print_cont(s, iter);
269
270 return TRACE_TYPE_HANDLED; 269 return TRACE_TYPE_HANDLED;
271} 270}
272 271
@@ -308,21 +307,17 @@ static void __trace_mmiotrace_rw(struct trace_array *tr,
308{ 307{
309 struct ring_buffer_event *event; 308 struct ring_buffer_event *event;
310 struct trace_mmiotrace_rw *entry; 309 struct trace_mmiotrace_rw *entry;
311 unsigned long irq_flags; 310 int pc = preempt_count();
312 311
313 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry), 312 event = trace_buffer_lock_reserve(tr, TRACE_MMIO_RW,
314 &irq_flags); 313 sizeof(*entry), 0, pc);
315 if (!event) { 314 if (!event) {
316 atomic_inc(&dropped_count); 315 atomic_inc(&dropped_count);
317 return; 316 return;
318 } 317 }
319 entry = ring_buffer_event_data(event); 318 entry = ring_buffer_event_data(event);
320 tracing_generic_entry_update(&entry->ent, 0, preempt_count());
321 entry->ent.type = TRACE_MMIO_RW;
322 entry->rw = *rw; 319 entry->rw = *rw;
323 ring_buffer_unlock_commit(tr->buffer, event, irq_flags); 320 trace_buffer_unlock_commit(tr, event, 0, pc);
324
325 trace_wake_up();
326} 321}
327 322
328void mmio_trace_rw(struct mmiotrace_rw *rw) 323void mmio_trace_rw(struct mmiotrace_rw *rw)
@@ -338,21 +333,17 @@ static void __trace_mmiotrace_map(struct trace_array *tr,
338{ 333{
339 struct ring_buffer_event *event; 334 struct ring_buffer_event *event;
340 struct trace_mmiotrace_map *entry; 335 struct trace_mmiotrace_map *entry;
341 unsigned long irq_flags; 336 int pc = preempt_count();
342 337
343 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry), 338 event = trace_buffer_lock_reserve(tr, TRACE_MMIO_MAP,
344 &irq_flags); 339 sizeof(*entry), 0, pc);
345 if (!event) { 340 if (!event) {
346 atomic_inc(&dropped_count); 341 atomic_inc(&dropped_count);
347 return; 342 return;
348 } 343 }
349 entry = ring_buffer_event_data(event); 344 entry = ring_buffer_event_data(event);
350 tracing_generic_entry_update(&entry->ent, 0, preempt_count());
351 entry->ent.type = TRACE_MMIO_MAP;
352 entry->map = *map; 345 entry->map = *map;
353 ring_buffer_unlock_commit(tr->buffer, event, irq_flags); 346 trace_buffer_unlock_commit(tr, event, 0, pc);
354
355 trace_wake_up();
356} 347}
357 348
358void mmio_trace_mapping(struct mmiotrace_map *map) 349void mmio_trace_mapping(struct mmiotrace_map *map)
diff --git a/kernel/trace/trace_nop.c b/kernel/trace/trace_nop.c
index b9767acd30ac..9aa84bde23cd 100644
--- a/kernel/trace/trace_nop.c
+++ b/kernel/trace/trace_nop.c
@@ -47,12 +47,7 @@ static void stop_nop_trace(struct trace_array *tr)
47 47
48static int nop_trace_init(struct trace_array *tr) 48static int nop_trace_init(struct trace_array *tr)
49{ 49{
50 int cpu;
51 ctx_trace = tr; 50 ctx_trace = tr;
52
53 for_each_online_cpu(cpu)
54 tracing_reset(tr, cpu);
55
56 start_nop_trace(tr); 51 start_nop_trace(tr);
57 return 0; 52 return 0;
58} 53}
diff --git a/kernel/trace/trace_output.c b/kernel/trace/trace_output.c
new file mode 100644
index 000000000000..9fc815031b09
--- /dev/null
+++ b/kernel/trace/trace_output.c
@@ -0,0 +1,919 @@
1/*
2 * trace_output.c
3 *
4 * Copyright (C) 2008 Red Hat Inc, Steven Rostedt <srostedt@redhat.com>
5 *
6 */
7
8#include <linux/module.h>
9#include <linux/mutex.h>
10#include <linux/ftrace.h>
11
12#include "trace_output.h"
13
14/* must be a power of 2 */
15#define EVENT_HASHSIZE 128
16
17static DEFINE_MUTEX(trace_event_mutex);
18static struct hlist_head event_hash[EVENT_HASHSIZE] __read_mostly;
19
20static int next_event_type = __TRACE_LAST_TYPE + 1;
21
22/**
23 * trace_seq_printf - sequence printing of trace information
24 * @s: trace sequence descriptor
25 * @fmt: printf format string
26 *
27 * The tracer may use either sequence operations or its own
28 * copy to user routines. To simplify formating of a trace
29 * trace_seq_printf is used to store strings into a special
30 * buffer (@s). Then the output may be either used by
31 * the sequencer or pulled into another buffer.
32 */
33int
34trace_seq_printf(struct trace_seq *s, const char *fmt, ...)
35{
36 int len = (PAGE_SIZE - 1) - s->len;
37 va_list ap;
38 int ret;
39
40 if (!len)
41 return 0;
42
43 va_start(ap, fmt);
44 ret = vsnprintf(s->buffer + s->len, len, fmt, ap);
45 va_end(ap);
46
47 /* If we can't write it all, don't bother writing anything */
48 if (ret >= len)
49 return 0;
50
51 s->len += ret;
52
53 return len;
54}
55
56/**
57 * trace_seq_puts - trace sequence printing of simple string
58 * @s: trace sequence descriptor
59 * @str: simple string to record
60 *
61 * The tracer may use either the sequence operations or its own
62 * copy to user routines. This function records a simple string
63 * into a special buffer (@s) for later retrieval by a sequencer
64 * or other mechanism.
65 */
66int trace_seq_puts(struct trace_seq *s, const char *str)
67{
68 int len = strlen(str);
69
70 if (len > ((PAGE_SIZE - 1) - s->len))
71 return 0;
72
73 memcpy(s->buffer + s->len, str, len);
74 s->len += len;
75
76 return len;
77}
78
79int trace_seq_putc(struct trace_seq *s, unsigned char c)
80{
81 if (s->len >= (PAGE_SIZE - 1))
82 return 0;
83
84 s->buffer[s->len++] = c;
85
86 return 1;
87}
88
89int trace_seq_putmem(struct trace_seq *s, void *mem, size_t len)
90{
91 if (len > ((PAGE_SIZE - 1) - s->len))
92 return 0;
93
94 memcpy(s->buffer + s->len, mem, len);
95 s->len += len;
96
97 return len;
98}
99
100int trace_seq_putmem_hex(struct trace_seq *s, void *mem, size_t len)
101{
102 unsigned char hex[HEX_CHARS];
103 unsigned char *data = mem;
104 int i, j;
105
106#ifdef __BIG_ENDIAN
107 for (i = 0, j = 0; i < len; i++) {
108#else
109 for (i = len-1, j = 0; i >= 0; i--) {
110#endif
111 hex[j++] = hex_asc_hi(data[i]);
112 hex[j++] = hex_asc_lo(data[i]);
113 }
114 hex[j++] = ' ';
115
116 return trace_seq_putmem(s, hex, j);
117}
118
119int trace_seq_path(struct trace_seq *s, struct path *path)
120{
121 unsigned char *p;
122
123 if (s->len >= (PAGE_SIZE - 1))
124 return 0;
125 p = d_path(path, s->buffer + s->len, PAGE_SIZE - s->len);
126 if (!IS_ERR(p)) {
127 p = mangle_path(s->buffer + s->len, p, "\n");
128 if (p) {
129 s->len = p - s->buffer;
130 return 1;
131 }
132 } else {
133 s->buffer[s->len++] = '?';
134 return 1;
135 }
136
137 return 0;
138}
139
140#ifdef CONFIG_KRETPROBES
141static inline const char *kretprobed(const char *name)
142{
143 static const char tramp_name[] = "kretprobe_trampoline";
144 int size = sizeof(tramp_name);
145
146 if (strncmp(tramp_name, name, size) == 0)
147 return "[unknown/kretprobe'd]";
148 return name;
149}
150#else
151static inline const char *kretprobed(const char *name)
152{
153 return name;
154}
155#endif /* CONFIG_KRETPROBES */
156
157static int
158seq_print_sym_short(struct trace_seq *s, const char *fmt, unsigned long address)
159{
160#ifdef CONFIG_KALLSYMS
161 char str[KSYM_SYMBOL_LEN];
162 const char *name;
163
164 kallsyms_lookup(address, NULL, NULL, NULL, str);
165
166 name = kretprobed(str);
167
168 return trace_seq_printf(s, fmt, name);
169#endif
170 return 1;
171}
172
173static int
174seq_print_sym_offset(struct trace_seq *s, const char *fmt,
175 unsigned long address)
176{
177#ifdef CONFIG_KALLSYMS
178 char str[KSYM_SYMBOL_LEN];
179 const char *name;
180
181 sprint_symbol(str, address);
182 name = kretprobed(str);
183
184 return trace_seq_printf(s, fmt, name);
185#endif
186 return 1;
187}
188
189#ifndef CONFIG_64BIT
190# define IP_FMT "%08lx"
191#else
192# define IP_FMT "%016lx"
193#endif
194
195int seq_print_user_ip(struct trace_seq *s, struct mm_struct *mm,
196 unsigned long ip, unsigned long sym_flags)
197{
198 struct file *file = NULL;
199 unsigned long vmstart = 0;
200 int ret = 1;
201
202 if (mm) {
203 const struct vm_area_struct *vma;
204
205 down_read(&mm->mmap_sem);
206 vma = find_vma(mm, ip);
207 if (vma) {
208 file = vma->vm_file;
209 vmstart = vma->vm_start;
210 }
211 if (file) {
212 ret = trace_seq_path(s, &file->f_path);
213 if (ret)
214 ret = trace_seq_printf(s, "[+0x%lx]",
215 ip - vmstart);
216 }
217 up_read(&mm->mmap_sem);
218 }
219 if (ret && ((sym_flags & TRACE_ITER_SYM_ADDR) || !file))
220 ret = trace_seq_printf(s, " <" IP_FMT ">", ip);
221 return ret;
222}
223
224int
225seq_print_userip_objs(const struct userstack_entry *entry, struct trace_seq *s,
226 unsigned long sym_flags)
227{
228 struct mm_struct *mm = NULL;
229 int ret = 1;
230 unsigned int i;
231
232 if (trace_flags & TRACE_ITER_SYM_USEROBJ) {
233 struct task_struct *task;
234 /*
235 * we do the lookup on the thread group leader,
236 * since individual threads might have already quit!
237 */
238 rcu_read_lock();
239 task = find_task_by_vpid(entry->ent.tgid);
240 if (task)
241 mm = get_task_mm(task);
242 rcu_read_unlock();
243 }
244
245 for (i = 0; i < FTRACE_STACK_ENTRIES; i++) {
246 unsigned long ip = entry->caller[i];
247
248 if (ip == ULONG_MAX || !ret)
249 break;
250 if (i && ret)
251 ret = trace_seq_puts(s, " <- ");
252 if (!ip) {
253 if (ret)
254 ret = trace_seq_puts(s, "??");
255 continue;
256 }
257 if (!ret)
258 break;
259 if (ret)
260 ret = seq_print_user_ip(s, mm, ip, sym_flags);
261 }
262
263 if (mm)
264 mmput(mm);
265 return ret;
266}
267
268int
269seq_print_ip_sym(struct trace_seq *s, unsigned long ip, unsigned long sym_flags)
270{
271 int ret;
272
273 if (!ip)
274 return trace_seq_printf(s, "0");
275
276 if (sym_flags & TRACE_ITER_SYM_OFFSET)
277 ret = seq_print_sym_offset(s, "%s", ip);
278 else
279 ret = seq_print_sym_short(s, "%s", ip);
280
281 if (!ret)
282 return 0;
283
284 if (sym_flags & TRACE_ITER_SYM_ADDR)
285 ret = trace_seq_printf(s, " <" IP_FMT ">", ip);
286 return ret;
287}
288
289static int
290lat_print_generic(struct trace_seq *s, struct trace_entry *entry, int cpu)
291{
292 int hardirq, softirq;
293 char *comm;
294
295 comm = trace_find_cmdline(entry->pid);
296 hardirq = entry->flags & TRACE_FLAG_HARDIRQ;
297 softirq = entry->flags & TRACE_FLAG_SOFTIRQ;
298
299 if (!trace_seq_printf(s, "%8.8s-%-5d %3d%c%c%c",
300 comm, entry->pid, cpu,
301 (entry->flags & TRACE_FLAG_IRQS_OFF) ? 'd' :
302 (entry->flags & TRACE_FLAG_IRQS_NOSUPPORT) ?
303 'X' : '.',
304 (entry->flags & TRACE_FLAG_NEED_RESCHED) ?
305 'N' : '.',
306 (hardirq && softirq) ? 'H' :
307 hardirq ? 'h' : softirq ? 's' : '.'))
308 return 0;
309
310 if (entry->preempt_count)
311 return trace_seq_printf(s, "%x", entry->preempt_count);
312 return trace_seq_puts(s, ".");
313}
314
315static unsigned long preempt_mark_thresh = 100;
316
317static int
318lat_print_timestamp(struct trace_seq *s, u64 abs_usecs,
319 unsigned long rel_usecs)
320{
321 return trace_seq_printf(s, " %4lldus%c: ", abs_usecs,
322 rel_usecs > preempt_mark_thresh ? '!' :
323 rel_usecs > 1 ? '+' : ' ');
324}
325
326int trace_print_context(struct trace_iterator *iter)
327{
328 struct trace_seq *s = &iter->seq;
329 struct trace_entry *entry = iter->ent;
330 char *comm = trace_find_cmdline(entry->pid);
331 unsigned long long t = ns2usecs(iter->ts);
332 unsigned long usec_rem = do_div(t, USEC_PER_SEC);
333 unsigned long secs = (unsigned long)t;
334
335 return trace_seq_printf(s, "%16s-%-5d [%03d] %5lu.%06lu: ",
336 comm, entry->pid, iter->cpu, secs, usec_rem);
337}
338
339int trace_print_lat_context(struct trace_iterator *iter)
340{
341 u64 next_ts;
342 int ret;
343 struct trace_seq *s = &iter->seq;
344 struct trace_entry *entry = iter->ent,
345 *next_entry = trace_find_next_entry(iter, NULL,
346 &next_ts);
347 unsigned long verbose = (trace_flags & TRACE_ITER_VERBOSE);
348 unsigned long abs_usecs = ns2usecs(iter->ts - iter->tr->time_start);
349 unsigned long rel_usecs;
350
351 if (!next_entry)
352 next_ts = iter->ts;
353 rel_usecs = ns2usecs(next_ts - iter->ts);
354
355 if (verbose) {
356 char *comm = trace_find_cmdline(entry->pid);
357 ret = trace_seq_printf(s, "%16s %5d %3d %d %08x %08lx [%08lx]"
358 " %ld.%03ldms (+%ld.%03ldms): ", comm,
359 entry->pid, iter->cpu, entry->flags,
360 entry->preempt_count, iter->idx,
361 ns2usecs(iter->ts),
362 abs_usecs / USEC_PER_MSEC,
363 abs_usecs % USEC_PER_MSEC,
364 rel_usecs / USEC_PER_MSEC,
365 rel_usecs % USEC_PER_MSEC);
366 } else {
367 ret = lat_print_generic(s, entry, iter->cpu);
368 if (ret)
369 ret = lat_print_timestamp(s, abs_usecs, rel_usecs);
370 }
371
372 return ret;
373}
374
375static const char state_to_char[] = TASK_STATE_TO_CHAR_STR;
376
377static int task_state_char(unsigned long state)
378{
379 int bit = state ? __ffs(state) + 1 : 0;
380
381 return bit < sizeof(state_to_char) - 1 ? state_to_char[bit] : '?';
382}
383
384/**
385 * ftrace_find_event - find a registered event
386 * @type: the type of event to look for
387 *
388 * Returns an event of type @type otherwise NULL
389 */
390struct trace_event *ftrace_find_event(int type)
391{
392 struct trace_event *event;
393 struct hlist_node *n;
394 unsigned key;
395
396 key = type & (EVENT_HASHSIZE - 1);
397
398 hlist_for_each_entry_rcu(event, n, &event_hash[key], node) {
399 if (event->type == type)
400 return event;
401 }
402
403 return NULL;
404}
405
406/**
407 * register_ftrace_event - register output for an event type
408 * @event: the event type to register
409 *
410 * Event types are stored in a hash and this hash is used to
411 * find a way to print an event. If the @event->type is set
412 * then it will use that type, otherwise it will assign a
413 * type to use.
414 *
415 * If you assign your own type, please make sure it is added
416 * to the trace_type enum in trace.h, to avoid collisions
417 * with the dynamic types.
418 *
419 * Returns the event type number or zero on error.
420 */
421int register_ftrace_event(struct trace_event *event)
422{
423 unsigned key;
424 int ret = 0;
425
426 mutex_lock(&trace_event_mutex);
427
428 if (!event->type)
429 event->type = next_event_type++;
430 else if (event->type > __TRACE_LAST_TYPE) {
431 printk(KERN_WARNING "Need to add type to trace.h\n");
432 WARN_ON(1);
433 }
434
435 if (ftrace_find_event(event->type))
436 goto out;
437
438 if (event->trace == NULL)
439 event->trace = trace_nop_print;
440 if (event->latency_trace == NULL)
441 event->latency_trace = trace_nop_print;
442 if (event->raw == NULL)
443 event->raw = trace_nop_print;
444 if (event->hex == NULL)
445 event->hex = trace_nop_print;
446 if (event->binary == NULL)
447 event->binary = trace_nop_print;
448
449 key = event->type & (EVENT_HASHSIZE - 1);
450
451 hlist_add_head_rcu(&event->node, &event_hash[key]);
452
453 ret = event->type;
454 out:
455 mutex_unlock(&trace_event_mutex);
456
457 return ret;
458}
459
460/**
461 * unregister_ftrace_event - remove a no longer used event
462 * @event: the event to remove
463 */
464int unregister_ftrace_event(struct trace_event *event)
465{
466 mutex_lock(&trace_event_mutex);
467 hlist_del(&event->node);
468 mutex_unlock(&trace_event_mutex);
469
470 return 0;
471}
472
473/*
474 * Standard events
475 */
476
477enum print_line_t trace_nop_print(struct trace_iterator *iter, int flags)
478{
479 return TRACE_TYPE_HANDLED;
480}
481
482/* TRACE_FN */
483static enum print_line_t trace_fn_latency(struct trace_iterator *iter,
484 int flags)
485{
486 struct ftrace_entry *field;
487 struct trace_seq *s = &iter->seq;
488
489 trace_assign_type(field, iter->ent);
490
491 if (!seq_print_ip_sym(s, field->ip, flags))
492 goto partial;
493 if (!trace_seq_puts(s, " ("))
494 goto partial;
495 if (!seq_print_ip_sym(s, field->parent_ip, flags))
496 goto partial;
497 if (!trace_seq_puts(s, ")\n"))
498 goto partial;
499
500 return TRACE_TYPE_HANDLED;
501
502 partial:
503 return TRACE_TYPE_PARTIAL_LINE;
504}
505
506static enum print_line_t trace_fn_trace(struct trace_iterator *iter, int flags)
507{
508 struct ftrace_entry *field;
509 struct trace_seq *s = &iter->seq;
510
511 trace_assign_type(field, iter->ent);
512
513 if (!seq_print_ip_sym(s, field->ip, flags))
514 goto partial;
515
516 if ((flags & TRACE_ITER_PRINT_PARENT) && field->parent_ip) {
517 if (!trace_seq_printf(s, " <-"))
518 goto partial;
519 if (!seq_print_ip_sym(s,
520 field->parent_ip,
521 flags))
522 goto partial;
523 }
524 if (!trace_seq_printf(s, "\n"))
525 goto partial;
526
527 return TRACE_TYPE_HANDLED;
528
529 partial:
530 return TRACE_TYPE_PARTIAL_LINE;
531}
532
533static enum print_line_t trace_fn_raw(struct trace_iterator *iter, int flags)
534{
535 struct ftrace_entry *field;
536
537 trace_assign_type(field, iter->ent);
538
539 if (!trace_seq_printf(&iter->seq, "%lx %lx\n",
540 field->ip,
541 field->parent_ip))
542 return TRACE_TYPE_PARTIAL_LINE;
543
544 return TRACE_TYPE_HANDLED;
545}
546
547static enum print_line_t trace_fn_hex(struct trace_iterator *iter, int flags)
548{
549 struct ftrace_entry *field;
550 struct trace_seq *s = &iter->seq;
551
552 trace_assign_type(field, iter->ent);
553
554 SEQ_PUT_HEX_FIELD_RET(s, field->ip);
555 SEQ_PUT_HEX_FIELD_RET(s, field->parent_ip);
556
557 return TRACE_TYPE_HANDLED;
558}
559
560static enum print_line_t trace_fn_bin(struct trace_iterator *iter, int flags)
561{
562 struct ftrace_entry *field;
563 struct trace_seq *s = &iter->seq;
564
565 trace_assign_type(field, iter->ent);
566
567 SEQ_PUT_FIELD_RET(s, field->ip);
568 SEQ_PUT_FIELD_RET(s, field->parent_ip);
569
570 return TRACE_TYPE_HANDLED;
571}
572
573static struct trace_event trace_fn_event = {
574 .type = TRACE_FN,
575 .trace = trace_fn_trace,
576 .latency_trace = trace_fn_latency,
577 .raw = trace_fn_raw,
578 .hex = trace_fn_hex,
579 .binary = trace_fn_bin,
580};
581
582/* TRACE_CTX an TRACE_WAKE */
583static enum print_line_t trace_ctxwake_print(struct trace_iterator *iter,
584 char *delim)
585{
586 struct ctx_switch_entry *field;
587 char *comm;
588 int S, T;
589
590 trace_assign_type(field, iter->ent);
591
592 T = task_state_char(field->next_state);
593 S = task_state_char(field->prev_state);
594 comm = trace_find_cmdline(field->next_pid);
595 if (!trace_seq_printf(&iter->seq,
596 " %5d:%3d:%c %s [%03d] %5d:%3d:%c %s\n",
597 field->prev_pid,
598 field->prev_prio,
599 S, delim,
600 field->next_cpu,
601 field->next_pid,
602 field->next_prio,
603 T, comm))
604 return TRACE_TYPE_PARTIAL_LINE;
605
606 return TRACE_TYPE_HANDLED;
607}
608
609static enum print_line_t trace_ctx_print(struct trace_iterator *iter, int flags)
610{
611 return trace_ctxwake_print(iter, "==>");
612}
613
614static enum print_line_t trace_wake_print(struct trace_iterator *iter,
615 int flags)
616{
617 return trace_ctxwake_print(iter, " +");
618}
619
620static int trace_ctxwake_raw(struct trace_iterator *iter, char S)
621{
622 struct ctx_switch_entry *field;
623 int T;
624
625 trace_assign_type(field, iter->ent);
626
627 if (!S)
628 task_state_char(field->prev_state);
629 T = task_state_char(field->next_state);
630 if (!trace_seq_printf(&iter->seq, "%d %d %c %d %d %d %c\n",
631 field->prev_pid,
632 field->prev_prio,
633 S,
634 field->next_cpu,
635 field->next_pid,
636 field->next_prio,
637 T))
638 return TRACE_TYPE_PARTIAL_LINE;
639
640 return TRACE_TYPE_HANDLED;
641}
642
643static enum print_line_t trace_ctx_raw(struct trace_iterator *iter, int flags)
644{
645 return trace_ctxwake_raw(iter, 0);
646}
647
648static enum print_line_t trace_wake_raw(struct trace_iterator *iter, int flags)
649{
650 return trace_ctxwake_raw(iter, '+');
651}
652
653
654static int trace_ctxwake_hex(struct trace_iterator *iter, char S)
655{
656 struct ctx_switch_entry *field;
657 struct trace_seq *s = &iter->seq;
658 int T;
659
660 trace_assign_type(field, iter->ent);
661
662 if (!S)
663 task_state_char(field->prev_state);
664 T = task_state_char(field->next_state);
665
666 SEQ_PUT_HEX_FIELD_RET(s, field->prev_pid);
667 SEQ_PUT_HEX_FIELD_RET(s, field->prev_prio);
668 SEQ_PUT_HEX_FIELD_RET(s, S);
669 SEQ_PUT_HEX_FIELD_RET(s, field->next_cpu);
670 SEQ_PUT_HEX_FIELD_RET(s, field->next_pid);
671 SEQ_PUT_HEX_FIELD_RET(s, field->next_prio);
672 SEQ_PUT_HEX_FIELD_RET(s, T);
673
674 return TRACE_TYPE_HANDLED;
675}
676
677static enum print_line_t trace_ctx_hex(struct trace_iterator *iter, int flags)
678{
679 return trace_ctxwake_hex(iter, 0);
680}
681
682static enum print_line_t trace_wake_hex(struct trace_iterator *iter, int flags)
683{
684 return trace_ctxwake_hex(iter, '+');
685}
686
687static enum print_line_t trace_ctxwake_bin(struct trace_iterator *iter,
688 int flags)
689{
690 struct ctx_switch_entry *field;
691 struct trace_seq *s = &iter->seq;
692
693 trace_assign_type(field, iter->ent);
694
695 SEQ_PUT_FIELD_RET(s, field->prev_pid);
696 SEQ_PUT_FIELD_RET(s, field->prev_prio);
697 SEQ_PUT_FIELD_RET(s, field->prev_state);
698 SEQ_PUT_FIELD_RET(s, field->next_pid);
699 SEQ_PUT_FIELD_RET(s, field->next_prio);
700 SEQ_PUT_FIELD_RET(s, field->next_state);
701
702 return TRACE_TYPE_HANDLED;
703}
704
705static struct trace_event trace_ctx_event = {
706 .type = TRACE_CTX,
707 .trace = trace_ctx_print,
708 .latency_trace = trace_ctx_print,
709 .raw = trace_ctx_raw,
710 .hex = trace_ctx_hex,
711 .binary = trace_ctxwake_bin,
712};
713
714static struct trace_event trace_wake_event = {
715 .type = TRACE_WAKE,
716 .trace = trace_wake_print,
717 .latency_trace = trace_wake_print,
718 .raw = trace_wake_raw,
719 .hex = trace_wake_hex,
720 .binary = trace_ctxwake_bin,
721};
722
723/* TRACE_SPECIAL */
724static enum print_line_t trace_special_print(struct trace_iterator *iter,
725 int flags)
726{
727 struct special_entry *field;
728
729 trace_assign_type(field, iter->ent);
730
731 if (!trace_seq_printf(&iter->seq, "# %ld %ld %ld\n",
732 field->arg1,
733 field->arg2,
734 field->arg3))
735 return TRACE_TYPE_PARTIAL_LINE;
736
737 return TRACE_TYPE_HANDLED;
738}
739
740static enum print_line_t trace_special_hex(struct trace_iterator *iter,
741 int flags)
742{
743 struct special_entry *field;
744 struct trace_seq *s = &iter->seq;
745
746 trace_assign_type(field, iter->ent);
747
748 SEQ_PUT_HEX_FIELD_RET(s, field->arg1);
749 SEQ_PUT_HEX_FIELD_RET(s, field->arg2);
750 SEQ_PUT_HEX_FIELD_RET(s, field->arg3);
751
752 return TRACE_TYPE_HANDLED;
753}
754
755static enum print_line_t trace_special_bin(struct trace_iterator *iter,
756 int flags)
757{
758 struct special_entry *field;
759 struct trace_seq *s = &iter->seq;
760
761 trace_assign_type(field, iter->ent);
762
763 SEQ_PUT_FIELD_RET(s, field->arg1);
764 SEQ_PUT_FIELD_RET(s, field->arg2);
765 SEQ_PUT_FIELD_RET(s, field->arg3);
766
767 return TRACE_TYPE_HANDLED;
768}
769
770static struct trace_event trace_special_event = {
771 .type = TRACE_SPECIAL,
772 .trace = trace_special_print,
773 .latency_trace = trace_special_print,
774 .raw = trace_special_print,
775 .hex = trace_special_hex,
776 .binary = trace_special_bin,
777};
778
779/* TRACE_STACK */
780
781static enum print_line_t trace_stack_print(struct trace_iterator *iter,
782 int flags)
783{
784 struct stack_entry *field;
785 struct trace_seq *s = &iter->seq;
786 int i;
787
788 trace_assign_type(field, iter->ent);
789
790 for (i = 0; i < FTRACE_STACK_ENTRIES; i++) {
791 if (i) {
792 if (!trace_seq_puts(s, " <= "))
793 goto partial;
794
795 if (!seq_print_ip_sym(s, field->caller[i], flags))
796 goto partial;
797 }
798 if (!trace_seq_puts(s, "\n"))
799 goto partial;
800 }
801
802 return TRACE_TYPE_HANDLED;
803
804 partial:
805 return TRACE_TYPE_PARTIAL_LINE;
806}
807
808static struct trace_event trace_stack_event = {
809 .type = TRACE_STACK,
810 .trace = trace_stack_print,
811 .latency_trace = trace_stack_print,
812 .raw = trace_special_print,
813 .hex = trace_special_hex,
814 .binary = trace_special_bin,
815};
816
817/* TRACE_USER_STACK */
818static enum print_line_t trace_user_stack_print(struct trace_iterator *iter,
819 int flags)
820{
821 struct userstack_entry *field;
822 struct trace_seq *s = &iter->seq;
823
824 trace_assign_type(field, iter->ent);
825
826 if (!seq_print_userip_objs(field, s, flags))
827 goto partial;
828
829 if (!trace_seq_putc(s, '\n'))
830 goto partial;
831
832 return TRACE_TYPE_HANDLED;
833
834 partial:
835 return TRACE_TYPE_PARTIAL_LINE;
836}
837
838static struct trace_event trace_user_stack_event = {
839 .type = TRACE_USER_STACK,
840 .trace = trace_user_stack_print,
841 .latency_trace = trace_user_stack_print,
842 .raw = trace_special_print,
843 .hex = trace_special_hex,
844 .binary = trace_special_bin,
845};
846
847/* TRACE_PRINT */
848static enum print_line_t trace_print_print(struct trace_iterator *iter,
849 int flags)
850{
851 struct print_entry *field;
852 struct trace_seq *s = &iter->seq;
853
854 trace_assign_type(field, iter->ent);
855
856 if (!seq_print_ip_sym(s, field->ip, flags))
857 goto partial;
858
859 if (!trace_seq_printf(s, ": %s", field->buf))
860 goto partial;
861
862 return TRACE_TYPE_HANDLED;
863
864 partial:
865 return TRACE_TYPE_PARTIAL_LINE;
866}
867
868static enum print_line_t trace_print_raw(struct trace_iterator *iter, int flags)
869{
870 struct print_entry *field;
871
872 trace_assign_type(field, iter->ent);
873
874 if (!trace_seq_printf(&iter->seq, "# %lx %s", field->ip, field->buf))
875 goto partial;
876
877 return TRACE_TYPE_HANDLED;
878
879 partial:
880 return TRACE_TYPE_PARTIAL_LINE;
881}
882
883static struct trace_event trace_print_event = {
884 .type = TRACE_PRINT,
885 .trace = trace_print_print,
886 .latency_trace = trace_print_print,
887 .raw = trace_print_raw,
888};
889
890static struct trace_event *events[] __initdata = {
891 &trace_fn_event,
892 &trace_ctx_event,
893 &trace_wake_event,
894 &trace_special_event,
895 &trace_stack_event,
896 &trace_user_stack_event,
897 &trace_print_event,
898 NULL
899};
900
901__init static int init_events(void)
902{
903 struct trace_event *event;
904 int i, ret;
905
906 for (i = 0; events[i]; i++) {
907 event = events[i];
908
909 ret = register_ftrace_event(event);
910 if (!ret) {
911 printk(KERN_WARNING "event %d failed to register\n",
912 event->type);
913 WARN_ON_ONCE(1);
914 }
915 }
916
917 return 0;
918}
919device_initcall(init_events);
diff --git a/kernel/trace/trace_output.h b/kernel/trace/trace_output.h
new file mode 100644
index 000000000000..551a25a72217
--- /dev/null
+++ b/kernel/trace/trace_output.h
@@ -0,0 +1,62 @@
1#ifndef __TRACE_EVENTS_H
2#define __TRACE_EVENTS_H
3
4#include "trace.h"
5
6typedef enum print_line_t (*trace_print_func)(struct trace_iterator *iter,
7 int flags);
8
9struct trace_event {
10 struct hlist_node node;
11 int type;
12 trace_print_func trace;
13 trace_print_func latency_trace;
14 trace_print_func raw;
15 trace_print_func hex;
16 trace_print_func binary;
17};
18
19extern int trace_seq_printf(struct trace_seq *s, const char *fmt, ...)
20 __attribute__ ((format (printf, 2, 3)));
21extern int
22seq_print_ip_sym(struct trace_seq *s, unsigned long ip,
23 unsigned long sym_flags);
24extern ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf,
25 size_t cnt);
26int trace_seq_puts(struct trace_seq *s, const char *str);
27int trace_seq_putc(struct trace_seq *s, unsigned char c);
28int trace_seq_putmem(struct trace_seq *s, void *mem, size_t len);
29int trace_seq_putmem_hex(struct trace_seq *s, void *mem, size_t len);
30int trace_seq_path(struct trace_seq *s, struct path *path);
31int seq_print_userip_objs(const struct userstack_entry *entry,
32 struct trace_seq *s, unsigned long sym_flags);
33int seq_print_user_ip(struct trace_seq *s, struct mm_struct *mm,
34 unsigned long ip, unsigned long sym_flags);
35
36int trace_print_context(struct trace_iterator *iter);
37int trace_print_lat_context(struct trace_iterator *iter);
38
39struct trace_event *ftrace_find_event(int type);
40int register_ftrace_event(struct trace_event *event);
41int unregister_ftrace_event(struct trace_event *event);
42
43enum print_line_t trace_nop_print(struct trace_iterator *iter, int flags);
44
45#define MAX_MEMHEX_BYTES 8
46#define HEX_CHARS (MAX_MEMHEX_BYTES*2 + 1)
47
48#define SEQ_PUT_FIELD_RET(s, x) \
49do { \
50 if (!trace_seq_putmem(s, &(x), sizeof(x))) \
51 return TRACE_TYPE_PARTIAL_LINE; \
52} while (0)
53
54#define SEQ_PUT_HEX_FIELD_RET(s, x) \
55do { \
56 BUILD_BUG_ON(sizeof(x) > MAX_MEMHEX_BYTES); \
57 if (!trace_seq_putmem_hex(s, &(x), sizeof(x))) \
58 return TRACE_TYPE_PARTIAL_LINE; \
59} while (0)
60
61#endif
62
diff --git a/kernel/trace/trace_power.c b/kernel/trace/trace_power.c
index 7bda248daf55..91ce672fb037 100644
--- a/kernel/trace/trace_power.c
+++ b/kernel/trace/trace_power.c
@@ -11,24 +11,126 @@
11 11
12#include <linux/init.h> 12#include <linux/init.h>
13#include <linux/debugfs.h> 13#include <linux/debugfs.h>
14#include <linux/ftrace.h> 14#include <trace/power.h>
15#include <linux/kallsyms.h> 15#include <linux/kallsyms.h>
16#include <linux/module.h> 16#include <linux/module.h>
17 17
18#include "trace.h" 18#include "trace.h"
19#include "trace_output.h"
19 20
20static struct trace_array *power_trace; 21static struct trace_array *power_trace;
21static int __read_mostly trace_power_enabled; 22static int __read_mostly trace_power_enabled;
22 23
24static void probe_power_start(struct power_trace *it, unsigned int type,
25 unsigned int level)
26{
27 if (!trace_power_enabled)
28 return;
29
30 memset(it, 0, sizeof(struct power_trace));
31 it->state = level;
32 it->type = type;
33 it->stamp = ktime_get();
34}
35
36
37static void probe_power_end(struct power_trace *it)
38{
39 struct ring_buffer_event *event;
40 struct trace_power *entry;
41 struct trace_array_cpu *data;
42 struct trace_array *tr = power_trace;
43
44 if (!trace_power_enabled)
45 return;
46
47 preempt_disable();
48 it->end = ktime_get();
49 data = tr->data[smp_processor_id()];
50
51 event = trace_buffer_lock_reserve(tr, TRACE_POWER,
52 sizeof(*entry), 0, 0);
53 if (!event)
54 goto out;
55 entry = ring_buffer_event_data(event);
56 entry->state_data = *it;
57 trace_buffer_unlock_commit(tr, event, 0, 0);
58 out:
59 preempt_enable();
60}
61
62static void probe_power_mark(struct power_trace *it, unsigned int type,
63 unsigned int level)
64{
65 struct ring_buffer_event *event;
66 struct trace_power *entry;
67 struct trace_array_cpu *data;
68 struct trace_array *tr = power_trace;
69
70 if (!trace_power_enabled)
71 return;
72
73 memset(it, 0, sizeof(struct power_trace));
74 it->state = level;
75 it->type = type;
76 it->stamp = ktime_get();
77 preempt_disable();
78 it->end = it->stamp;
79 data = tr->data[smp_processor_id()];
80
81 event = trace_buffer_lock_reserve(tr, TRACE_POWER,
82 sizeof(*entry), 0, 0);
83 if (!event)
84 goto out;
85 entry = ring_buffer_event_data(event);
86 entry->state_data = *it;
87 trace_buffer_unlock_commit(tr, event, 0, 0);
88 out:
89 preempt_enable();
90}
91
92static int tracing_power_register(void)
93{
94 int ret;
95
96 ret = register_trace_power_start(probe_power_start);
97 if (ret) {
98 pr_info("power trace: Couldn't activate tracepoint"
99 " probe to trace_power_start\n");
100 return ret;
101 }
102 ret = register_trace_power_end(probe_power_end);
103 if (ret) {
104 pr_info("power trace: Couldn't activate tracepoint"
105 " probe to trace_power_end\n");
106 goto fail_start;
107 }
108 ret = register_trace_power_mark(probe_power_mark);
109 if (ret) {
110 pr_info("power trace: Couldn't activate tracepoint"
111 " probe to trace_power_mark\n");
112 goto fail_end;
113 }
114 return ret;
115fail_end:
116 unregister_trace_power_end(probe_power_end);
117fail_start:
118 unregister_trace_power_start(probe_power_start);
119 return ret;
120}
23 121
24static void start_power_trace(struct trace_array *tr) 122static void start_power_trace(struct trace_array *tr)
25{ 123{
26 trace_power_enabled = 1; 124 trace_power_enabled = 1;
125 tracing_power_register();
27} 126}
28 127
29static void stop_power_trace(struct trace_array *tr) 128static void stop_power_trace(struct trace_array *tr)
30{ 129{
31 trace_power_enabled = 0; 130 trace_power_enabled = 0;
131 unregister_trace_power_start(probe_power_start);
132 unregister_trace_power_end(probe_power_end);
133 unregister_trace_power_mark(probe_power_mark);
32} 134}
33 135
34 136
@@ -38,6 +140,7 @@ static int power_trace_init(struct trace_array *tr)
38 power_trace = tr; 140 power_trace = tr;
39 141
40 trace_power_enabled = 1; 142 trace_power_enabled = 1;
143 tracing_power_register();
41 144
42 for_each_cpu(cpu, cpu_possible_mask) 145 for_each_cpu(cpu, cpu_possible_mask)
43 tracing_reset(tr, cpu); 146 tracing_reset(tr, cpu);
@@ -94,86 +197,3 @@ static int init_power_trace(void)
94 return register_tracer(&power_tracer); 197 return register_tracer(&power_tracer);
95} 198}
96device_initcall(init_power_trace); 199device_initcall(init_power_trace);
97
98void trace_power_start(struct power_trace *it, unsigned int type,
99 unsigned int level)
100{
101 if (!trace_power_enabled)
102 return;
103
104 memset(it, 0, sizeof(struct power_trace));
105 it->state = level;
106 it->type = type;
107 it->stamp = ktime_get();
108}
109EXPORT_SYMBOL_GPL(trace_power_start);
110
111
112void trace_power_end(struct power_trace *it)
113{
114 struct ring_buffer_event *event;
115 struct trace_power *entry;
116 struct trace_array_cpu *data;
117 unsigned long irq_flags;
118 struct trace_array *tr = power_trace;
119
120 if (!trace_power_enabled)
121 return;
122
123 preempt_disable();
124 it->end = ktime_get();
125 data = tr->data[smp_processor_id()];
126
127 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
128 &irq_flags);
129 if (!event)
130 goto out;
131 entry = ring_buffer_event_data(event);
132 tracing_generic_entry_update(&entry->ent, 0, 0);
133 entry->ent.type = TRACE_POWER;
134 entry->state_data = *it;
135 ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
136
137 trace_wake_up();
138
139 out:
140 preempt_enable();
141}
142EXPORT_SYMBOL_GPL(trace_power_end);
143
144void trace_power_mark(struct power_trace *it, unsigned int type,
145 unsigned int level)
146{
147 struct ring_buffer_event *event;
148 struct trace_power *entry;
149 struct trace_array_cpu *data;
150 unsigned long irq_flags;
151 struct trace_array *tr = power_trace;
152
153 if (!trace_power_enabled)
154 return;
155
156 memset(it, 0, sizeof(struct power_trace));
157 it->state = level;
158 it->type = type;
159 it->stamp = ktime_get();
160 preempt_disable();
161 it->end = it->stamp;
162 data = tr->data[smp_processor_id()];
163
164 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
165 &irq_flags);
166 if (!event)
167 goto out;
168 entry = ring_buffer_event_data(event);
169 tracing_generic_entry_update(&entry->ent, 0, 0);
170 entry->ent.type = TRACE_POWER;
171 entry->state_data = *it;
172 ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
173
174 trace_wake_up();
175
176 out:
177 preempt_enable();
178}
179EXPORT_SYMBOL_GPL(trace_power_mark);
diff --git a/kernel/trace/trace_sched_switch.c b/kernel/trace/trace_sched_switch.c
index df175cb4564f..77132c2cf3d9 100644
--- a/kernel/trace/trace_sched_switch.c
+++ b/kernel/trace/trace_sched_switch.c
@@ -43,7 +43,7 @@ probe_sched_switch(struct rq *__rq, struct task_struct *prev,
43 data = ctx_trace->data[cpu]; 43 data = ctx_trace->data[cpu];
44 44
45 if (likely(!atomic_read(&data->disabled))) 45 if (likely(!atomic_read(&data->disabled)))
46 tracing_sched_switch_trace(ctx_trace, data, prev, next, flags, pc); 46 tracing_sched_switch_trace(ctx_trace, prev, next, flags, pc);
47 47
48 local_irq_restore(flags); 48 local_irq_restore(flags);
49} 49}
@@ -66,7 +66,7 @@ probe_sched_wakeup(struct rq *__rq, struct task_struct *wakee, int success)
66 data = ctx_trace->data[cpu]; 66 data = ctx_trace->data[cpu];
67 67
68 if (likely(!atomic_read(&data->disabled))) 68 if (likely(!atomic_read(&data->disabled)))
69 tracing_sched_wakeup_trace(ctx_trace, data, wakee, current, 69 tracing_sched_wakeup_trace(ctx_trace, wakee, current,
70 flags, pc); 70 flags, pc);
71 71
72 local_irq_restore(flags); 72 local_irq_restore(flags);
@@ -93,7 +93,7 @@ static int tracing_sched_register(void)
93 ret = register_trace_sched_switch(probe_sched_switch); 93 ret = register_trace_sched_switch(probe_sched_switch);
94 if (ret) { 94 if (ret) {
95 pr_info("sched trace: Couldn't activate tracepoint" 95 pr_info("sched trace: Couldn't activate tracepoint"
96 " probe to kernel_sched_schedule\n"); 96 " probe to kernel_sched_switch\n");
97 goto fail_deprobe_wake_new; 97 goto fail_deprobe_wake_new;
98 } 98 }
99 99
@@ -185,12 +185,6 @@ void tracing_sched_switch_assign_trace(struct trace_array *tr)
185 ctx_trace = tr; 185 ctx_trace = tr;
186} 186}
187 187
188static void start_sched_trace(struct trace_array *tr)
189{
190 tracing_reset_online_cpus(tr);
191 tracing_start_sched_switch_record();
192}
193
194static void stop_sched_trace(struct trace_array *tr) 188static void stop_sched_trace(struct trace_array *tr)
195{ 189{
196 tracing_stop_sched_switch_record(); 190 tracing_stop_sched_switch_record();
@@ -199,7 +193,7 @@ static void stop_sched_trace(struct trace_array *tr)
199static int sched_switch_trace_init(struct trace_array *tr) 193static int sched_switch_trace_init(struct trace_array *tr)
200{ 194{
201 ctx_trace = tr; 195 ctx_trace = tr;
202 start_sched_trace(tr); 196 tracing_start_sched_switch_record();
203 return 0; 197 return 0;
204} 198}
205 199
@@ -227,6 +221,7 @@ static struct tracer sched_switch_trace __read_mostly =
227 .reset = sched_switch_trace_reset, 221 .reset = sched_switch_trace_reset,
228 .start = sched_switch_trace_start, 222 .start = sched_switch_trace_start,
229 .stop = sched_switch_trace_stop, 223 .stop = sched_switch_trace_stop,
224 .wait_pipe = poll_wait_pipe,
230#ifdef CONFIG_FTRACE_SELFTEST 225#ifdef CONFIG_FTRACE_SELFTEST
231 .selftest = trace_selftest_startup_sched_switch, 226 .selftest = trace_selftest_startup_sched_switch,
232#endif 227#endif
diff --git a/kernel/trace/trace_sched_wakeup.c b/kernel/trace/trace_sched_wakeup.c
index 42ae1e77b6b3..db55f7aaa640 100644
--- a/kernel/trace/trace_sched_wakeup.c
+++ b/kernel/trace/trace_sched_wakeup.c
@@ -25,6 +25,7 @@ static int __read_mostly tracer_enabled;
25static struct task_struct *wakeup_task; 25static struct task_struct *wakeup_task;
26static int wakeup_cpu; 26static int wakeup_cpu;
27static unsigned wakeup_prio = -1; 27static unsigned wakeup_prio = -1;
28static int wakeup_rt;
28 29
29static raw_spinlock_t wakeup_lock = 30static raw_spinlock_t wakeup_lock =
30 (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED; 31 (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
@@ -71,7 +72,7 @@ wakeup_tracer_call(unsigned long ip, unsigned long parent_ip)
71 if (task_cpu(wakeup_task) != cpu) 72 if (task_cpu(wakeup_task) != cpu)
72 goto unlock; 73 goto unlock;
73 74
74 trace_function(tr, data, ip, parent_ip, flags, pc); 75 trace_function(tr, ip, parent_ip, flags, pc);
75 76
76 unlock: 77 unlock:
77 __raw_spin_unlock(&wakeup_lock); 78 __raw_spin_unlock(&wakeup_lock);
@@ -151,7 +152,8 @@ probe_wakeup_sched_switch(struct rq *rq, struct task_struct *prev,
151 if (unlikely(!tracer_enabled || next != wakeup_task)) 152 if (unlikely(!tracer_enabled || next != wakeup_task))
152 goto out_unlock; 153 goto out_unlock;
153 154
154 trace_function(wakeup_trace, data, CALLER_ADDR1, CALLER_ADDR2, flags, pc); 155 trace_function(wakeup_trace, CALLER_ADDR1, CALLER_ADDR2, flags, pc);
156 tracing_sched_switch_trace(wakeup_trace, prev, next, flags, pc);
155 157
156 /* 158 /*
157 * usecs conversion is slow so we try to delay the conversion 159 * usecs conversion is slow so we try to delay the conversion
@@ -182,13 +184,10 @@ out:
182 184
183static void __wakeup_reset(struct trace_array *tr) 185static void __wakeup_reset(struct trace_array *tr)
184{ 186{
185 struct trace_array_cpu *data;
186 int cpu; 187 int cpu;
187 188
188 for_each_possible_cpu(cpu) { 189 for_each_possible_cpu(cpu)
189 data = tr->data[cpu];
190 tracing_reset(tr, cpu); 190 tracing_reset(tr, cpu);
191 }
192 191
193 wakeup_cpu = -1; 192 wakeup_cpu = -1;
194 wakeup_prio = -1; 193 wakeup_prio = -1;
@@ -213,6 +212,7 @@ static void wakeup_reset(struct trace_array *tr)
213static void 212static void
214probe_wakeup(struct rq *rq, struct task_struct *p, int success) 213probe_wakeup(struct rq *rq, struct task_struct *p, int success)
215{ 214{
215 struct trace_array_cpu *data;
216 int cpu = smp_processor_id(); 216 int cpu = smp_processor_id();
217 unsigned long flags; 217 unsigned long flags;
218 long disabled; 218 long disabled;
@@ -224,7 +224,7 @@ probe_wakeup(struct rq *rq, struct task_struct *p, int success)
224 tracing_record_cmdline(p); 224 tracing_record_cmdline(p);
225 tracing_record_cmdline(current); 225 tracing_record_cmdline(current);
226 226
227 if (likely(!rt_task(p)) || 227 if ((wakeup_rt && !rt_task(p)) ||
228 p->prio >= wakeup_prio || 228 p->prio >= wakeup_prio ||
229 p->prio >= current->prio) 229 p->prio >= current->prio)
230 return; 230 return;
@@ -252,9 +252,10 @@ probe_wakeup(struct rq *rq, struct task_struct *p, int success)
252 252
253 local_save_flags(flags); 253 local_save_flags(flags);
254 254
255 wakeup_trace->data[wakeup_cpu]->preempt_timestamp = ftrace_now(cpu); 255 data = wakeup_trace->data[wakeup_cpu];
256 trace_function(wakeup_trace, wakeup_trace->data[wakeup_cpu], 256 data->preempt_timestamp = ftrace_now(cpu);
257 CALLER_ADDR1, CALLER_ADDR2, flags, pc); 257 tracing_sched_wakeup_trace(wakeup_trace, p, current, flags, pc);
258 trace_function(wakeup_trace, CALLER_ADDR1, CALLER_ADDR2, flags, pc);
258 259
259out_locked: 260out_locked:
260 __raw_spin_unlock(&wakeup_lock); 261 __raw_spin_unlock(&wakeup_lock);
@@ -262,12 +263,6 @@ out:
262 atomic_dec(&wakeup_trace->data[cpu]->disabled); 263 atomic_dec(&wakeup_trace->data[cpu]->disabled);
263} 264}
264 265
265/*
266 * save_tracer_enabled is used to save the state of the tracer_enabled
267 * variable when we disable it when we open a trace output file.
268 */
269static int save_tracer_enabled;
270
271static void start_wakeup_tracer(struct trace_array *tr) 266static void start_wakeup_tracer(struct trace_array *tr)
272{ 267{
273 int ret; 268 int ret;
@@ -289,7 +284,7 @@ static void start_wakeup_tracer(struct trace_array *tr)
289 ret = register_trace_sched_switch(probe_wakeup_sched_switch); 284 ret = register_trace_sched_switch(probe_wakeup_sched_switch);
290 if (ret) { 285 if (ret) {
291 pr_info("sched trace: Couldn't activate tracepoint" 286 pr_info("sched trace: Couldn't activate tracepoint"
292 " probe to kernel_sched_schedule\n"); 287 " probe to kernel_sched_switch\n");
293 goto fail_deprobe_wake_new; 288 goto fail_deprobe_wake_new;
294 } 289 }
295 290
@@ -306,13 +301,10 @@ static void start_wakeup_tracer(struct trace_array *tr)
306 301
307 register_ftrace_function(&trace_ops); 302 register_ftrace_function(&trace_ops);
308 303
309 if (tracing_is_enabled()) { 304 if (tracing_is_enabled())
310 tracer_enabled = 1; 305 tracer_enabled = 1;
311 save_tracer_enabled = 1; 306 else
312 } else {
313 tracer_enabled = 0; 307 tracer_enabled = 0;
314 save_tracer_enabled = 0;
315 }
316 308
317 return; 309 return;
318fail_deprobe_wake_new: 310fail_deprobe_wake_new:
@@ -324,14 +316,13 @@ fail_deprobe:
324static void stop_wakeup_tracer(struct trace_array *tr) 316static void stop_wakeup_tracer(struct trace_array *tr)
325{ 317{
326 tracer_enabled = 0; 318 tracer_enabled = 0;
327 save_tracer_enabled = 0;
328 unregister_ftrace_function(&trace_ops); 319 unregister_ftrace_function(&trace_ops);
329 unregister_trace_sched_switch(probe_wakeup_sched_switch); 320 unregister_trace_sched_switch(probe_wakeup_sched_switch);
330 unregister_trace_sched_wakeup_new(probe_wakeup); 321 unregister_trace_sched_wakeup_new(probe_wakeup);
331 unregister_trace_sched_wakeup(probe_wakeup); 322 unregister_trace_sched_wakeup(probe_wakeup);
332} 323}
333 324
334static int wakeup_tracer_init(struct trace_array *tr) 325static int __wakeup_tracer_init(struct trace_array *tr)
335{ 326{
336 tracing_max_latency = 0; 327 tracing_max_latency = 0;
337 wakeup_trace = tr; 328 wakeup_trace = tr;
@@ -339,6 +330,18 @@ static int wakeup_tracer_init(struct trace_array *tr)
339 return 0; 330 return 0;
340} 331}
341 332
333static int wakeup_tracer_init(struct trace_array *tr)
334{
335 wakeup_rt = 0;
336 return __wakeup_tracer_init(tr);
337}
338
339static int wakeup_rt_tracer_init(struct trace_array *tr)
340{
341 wakeup_rt = 1;
342 return __wakeup_tracer_init(tr);
343}
344
342static void wakeup_tracer_reset(struct trace_array *tr) 345static void wakeup_tracer_reset(struct trace_array *tr)
343{ 346{
344 stop_wakeup_tracer(tr); 347 stop_wakeup_tracer(tr);
@@ -350,28 +353,11 @@ static void wakeup_tracer_start(struct trace_array *tr)
350{ 353{
351 wakeup_reset(tr); 354 wakeup_reset(tr);
352 tracer_enabled = 1; 355 tracer_enabled = 1;
353 save_tracer_enabled = 1;
354} 356}
355 357
356static void wakeup_tracer_stop(struct trace_array *tr) 358static void wakeup_tracer_stop(struct trace_array *tr)
357{ 359{
358 tracer_enabled = 0; 360 tracer_enabled = 0;
359 save_tracer_enabled = 0;
360}
361
362static void wakeup_tracer_open(struct trace_iterator *iter)
363{
364 /* stop the trace while dumping */
365 tracer_enabled = 0;
366}
367
368static void wakeup_tracer_close(struct trace_iterator *iter)
369{
370 /* forget about any processes we were recording */
371 if (save_tracer_enabled) {
372 wakeup_reset(iter->tr);
373 tracer_enabled = 1;
374 }
375} 361}
376 362
377static struct tracer wakeup_tracer __read_mostly = 363static struct tracer wakeup_tracer __read_mostly =
@@ -381,8 +367,20 @@ static struct tracer wakeup_tracer __read_mostly =
381 .reset = wakeup_tracer_reset, 367 .reset = wakeup_tracer_reset,
382 .start = wakeup_tracer_start, 368 .start = wakeup_tracer_start,
383 .stop = wakeup_tracer_stop, 369 .stop = wakeup_tracer_stop,
384 .open = wakeup_tracer_open, 370 .print_max = 1,
385 .close = wakeup_tracer_close, 371#ifdef CONFIG_FTRACE_SELFTEST
372 .selftest = trace_selftest_startup_wakeup,
373#endif
374};
375
376static struct tracer wakeup_rt_tracer __read_mostly =
377{
378 .name = "wakeup_rt",
379 .init = wakeup_rt_tracer_init,
380 .reset = wakeup_tracer_reset,
381 .start = wakeup_tracer_start,
382 .stop = wakeup_tracer_stop,
383 .wait_pipe = poll_wait_pipe,
386 .print_max = 1, 384 .print_max = 1,
387#ifdef CONFIG_FTRACE_SELFTEST 385#ifdef CONFIG_FTRACE_SELFTEST
388 .selftest = trace_selftest_startup_wakeup, 386 .selftest = trace_selftest_startup_wakeup,
@@ -397,6 +395,10 @@ __init static int init_wakeup_tracer(void)
397 if (ret) 395 if (ret)
398 return ret; 396 return ret;
399 397
398 ret = register_tracer(&wakeup_rt_tracer);
399 if (ret)
400 return ret;
401
400 return 0; 402 return 0;
401} 403}
402device_initcall(init_wakeup_tracer); 404device_initcall(init_wakeup_tracer);
diff --git a/kernel/trace/trace_selftest.c b/kernel/trace/trace_selftest.c
index bc8e80a86bca..7238646b8723 100644
--- a/kernel/trace/trace_selftest.c
+++ b/kernel/trace/trace_selftest.c
@@ -9,11 +9,12 @@ static inline int trace_valid_entry(struct trace_entry *entry)
9 case TRACE_FN: 9 case TRACE_FN:
10 case TRACE_CTX: 10 case TRACE_CTX:
11 case TRACE_WAKE: 11 case TRACE_WAKE:
12 case TRACE_CONT:
13 case TRACE_STACK: 12 case TRACE_STACK:
14 case TRACE_PRINT: 13 case TRACE_PRINT:
15 case TRACE_SPECIAL: 14 case TRACE_SPECIAL:
16 case TRACE_BRANCH: 15 case TRACE_BRANCH:
16 case TRACE_GRAPH_ENT:
17 case TRACE_GRAPH_RET:
17 return 1; 18 return 1;
18 } 19 }
19 return 0; 20 return 0;
@@ -125,9 +126,9 @@ int trace_selftest_startup_dynamic_tracing(struct tracer *trace,
125 func(); 126 func();
126 127
127 /* 128 /*
128 * Some archs *cough*PowerPC*cough* add charachters to the 129 * Some archs *cough*PowerPC*cough* add characters to the
129 * start of the function names. We simply put a '*' to 130 * start of the function names. We simply put a '*' to
130 * accomodate them. 131 * accommodate them.
131 */ 132 */
132 func_name = "*" STR(DYN_FTRACE_TEST_NAME); 133 func_name = "*" STR(DYN_FTRACE_TEST_NAME);
133 134
@@ -135,7 +136,7 @@ int trace_selftest_startup_dynamic_tracing(struct tracer *trace,
135 ftrace_set_filter(func_name, strlen(func_name), 1); 136 ftrace_set_filter(func_name, strlen(func_name), 1);
136 137
137 /* enable tracing */ 138 /* enable tracing */
138 ret = trace->init(tr); 139 ret = tracer_init(trace, tr);
139 if (ret) { 140 if (ret) {
140 warn_failed_init_tracer(trace, ret); 141 warn_failed_init_tracer(trace, ret);
141 goto out; 142 goto out;
@@ -209,7 +210,7 @@ trace_selftest_startup_function(struct tracer *trace, struct trace_array *tr)
209 ftrace_enabled = 1; 210 ftrace_enabled = 1;
210 tracer_enabled = 1; 211 tracer_enabled = 1;
211 212
212 ret = trace->init(tr); 213 ret = tracer_init(trace, tr);
213 if (ret) { 214 if (ret) {
214 warn_failed_init_tracer(trace, ret); 215 warn_failed_init_tracer(trace, ret);
215 goto out; 216 goto out;
@@ -247,6 +248,54 @@ trace_selftest_startup_function(struct tracer *trace, struct trace_array *tr)
247} 248}
248#endif /* CONFIG_FUNCTION_TRACER */ 249#endif /* CONFIG_FUNCTION_TRACER */
249 250
251
252#ifdef CONFIG_FUNCTION_GRAPH_TRACER
253/*
254 * Pretty much the same than for the function tracer from which the selftest
255 * has been borrowed.
256 */
257int
258trace_selftest_startup_function_graph(struct tracer *trace,
259 struct trace_array *tr)
260{
261 int ret;
262 unsigned long count;
263
264 ret = tracer_init(trace, tr);
265 if (ret) {
266 warn_failed_init_tracer(trace, ret);
267 goto out;
268 }
269
270 /* Sleep for a 1/10 of a second */
271 msleep(100);
272
273 tracing_stop();
274
275 /* check the trace buffer */
276 ret = trace_test_buffer(tr, &count);
277
278 trace->reset(tr);
279 tracing_start();
280
281 if (!ret && !count) {
282 printk(KERN_CONT ".. no entries found ..");
283 ret = -1;
284 goto out;
285 }
286
287 /* Don't test dynamic tracing, the function tracer already did */
288
289out:
290 /* Stop it if we failed */
291 if (ret)
292 ftrace_graph_stop();
293
294 return ret;
295}
296#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
297
298
250#ifdef CONFIG_IRQSOFF_TRACER 299#ifdef CONFIG_IRQSOFF_TRACER
251int 300int
252trace_selftest_startup_irqsoff(struct tracer *trace, struct trace_array *tr) 301trace_selftest_startup_irqsoff(struct tracer *trace, struct trace_array *tr)
@@ -256,7 +305,7 @@ trace_selftest_startup_irqsoff(struct tracer *trace, struct trace_array *tr)
256 int ret; 305 int ret;
257 306
258 /* start the tracing */ 307 /* start the tracing */
259 ret = trace->init(tr); 308 ret = tracer_init(trace, tr);
260 if (ret) { 309 if (ret) {
261 warn_failed_init_tracer(trace, ret); 310 warn_failed_init_tracer(trace, ret);
262 return ret; 311 return ret;
@@ -310,7 +359,7 @@ trace_selftest_startup_preemptoff(struct tracer *trace, struct trace_array *tr)
310 } 359 }
311 360
312 /* start the tracing */ 361 /* start the tracing */
313 ret = trace->init(tr); 362 ret = tracer_init(trace, tr);
314 if (ret) { 363 if (ret) {
315 warn_failed_init_tracer(trace, ret); 364 warn_failed_init_tracer(trace, ret);
316 return ret; 365 return ret;
@@ -364,7 +413,7 @@ trace_selftest_startup_preemptirqsoff(struct tracer *trace, struct trace_array *
364 } 413 }
365 414
366 /* start the tracing */ 415 /* start the tracing */
367 ret = trace->init(tr); 416 ret = tracer_init(trace, tr);
368 if (ret) { 417 if (ret) {
369 warn_failed_init_tracer(trace, ret); 418 warn_failed_init_tracer(trace, ret);
370 goto out; 419 goto out;
@@ -496,7 +545,7 @@ trace_selftest_startup_wakeup(struct tracer *trace, struct trace_array *tr)
496 wait_for_completion(&isrt); 545 wait_for_completion(&isrt);
497 546
498 /* start the tracing */ 547 /* start the tracing */
499 ret = trace->init(tr); 548 ret = tracer_init(trace, tr);
500 if (ret) { 549 if (ret) {
501 warn_failed_init_tracer(trace, ret); 550 warn_failed_init_tracer(trace, ret);
502 return ret; 551 return ret;
@@ -557,7 +606,7 @@ trace_selftest_startup_sched_switch(struct tracer *trace, struct trace_array *tr
557 int ret; 606 int ret;
558 607
559 /* start the tracing */ 608 /* start the tracing */
560 ret = trace->init(tr); 609 ret = tracer_init(trace, tr);
561 if (ret) { 610 if (ret) {
562 warn_failed_init_tracer(trace, ret); 611 warn_failed_init_tracer(trace, ret);
563 return ret; 612 return ret;
@@ -589,10 +638,10 @@ trace_selftest_startup_sysprof(struct tracer *trace, struct trace_array *tr)
589 int ret; 638 int ret;
590 639
591 /* start the tracing */ 640 /* start the tracing */
592 ret = trace->init(tr); 641 ret = tracer_init(trace, tr);
593 if (ret) { 642 if (ret) {
594 warn_failed_init_tracer(trace, ret); 643 warn_failed_init_tracer(trace, ret);
595 return 0; 644 return ret;
596 } 645 }
597 646
598 /* Sleep for a 1/10 of a second */ 647 /* Sleep for a 1/10 of a second */
@@ -604,6 +653,11 @@ trace_selftest_startup_sysprof(struct tracer *trace, struct trace_array *tr)
604 trace->reset(tr); 653 trace->reset(tr);
605 tracing_start(); 654 tracing_start();
606 655
656 if (!ret && !count) {
657 printk(KERN_CONT ".. no entries found ..");
658 ret = -1;
659 }
660
607 return ret; 661 return ret;
608} 662}
609#endif /* CONFIG_SYSPROF_TRACER */ 663#endif /* CONFIG_SYSPROF_TRACER */
@@ -616,7 +670,7 @@ trace_selftest_startup_branch(struct tracer *trace, struct trace_array *tr)
616 int ret; 670 int ret;
617 671
618 /* start the tracing */ 672 /* start the tracing */
619 ret = trace->init(tr); 673 ret = tracer_init(trace, tr);
620 if (ret) { 674 if (ret) {
621 warn_failed_init_tracer(trace, ret); 675 warn_failed_init_tracer(trace, ret);
622 return ret; 676 return ret;
@@ -631,6 +685,11 @@ trace_selftest_startup_branch(struct tracer *trace, struct trace_array *tr)
631 trace->reset(tr); 685 trace->reset(tr);
632 tracing_start(); 686 tracing_start();
633 687
688 if (!ret && !count) {
689 printk(KERN_CONT ".. no entries found ..");
690 ret = -1;
691 }
692
634 return ret; 693 return ret;
635} 694}
636#endif /* CONFIG_BRANCH_TRACER */ 695#endif /* CONFIG_BRANCH_TRACER */
diff --git a/kernel/trace/trace_stat.c b/kernel/trace/trace_stat.c
new file mode 100644
index 000000000000..39310e3434ee
--- /dev/null
+++ b/kernel/trace/trace_stat.c
@@ -0,0 +1,319 @@
1/*
2 * Infrastructure for statistic tracing (histogram output).
3 *
4 * Copyright (C) 2008 Frederic Weisbecker <fweisbec@gmail.com>
5 *
6 * Based on the code from trace_branch.c which is
7 * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com>
8 *
9 */
10
11
12#include <linux/list.h>
13#include <linux/debugfs.h>
14#include "trace_stat.h"
15#include "trace.h"
16
17
18/* List of stat entries from a tracer */
19struct trace_stat_list {
20 struct list_head list;
21 void *stat;
22};
23
24/* A stat session is the stats output in one file */
25struct tracer_stat_session {
26 struct list_head session_list;
27 struct tracer_stat *ts;
28 struct list_head stat_list;
29 struct mutex stat_mutex;
30 struct dentry *file;
31};
32
33/* All of the sessions currently in use. Each stat file embed one session */
34static LIST_HEAD(all_stat_sessions);
35static DEFINE_MUTEX(all_stat_sessions_mutex);
36
37/* The root directory for all stat files */
38static struct dentry *stat_dir;
39
40
41static void reset_stat_session(struct tracer_stat_session *session)
42{
43 struct trace_stat_list *node, *next;
44
45 list_for_each_entry_safe(node, next, &session->stat_list, list)
46 kfree(node);
47
48 INIT_LIST_HEAD(&session->stat_list);
49}
50
51static void destroy_session(struct tracer_stat_session *session)
52{
53 debugfs_remove(session->file);
54 reset_stat_session(session);
55 mutex_destroy(&session->stat_mutex);
56 kfree(session);
57}
58
59/*
60 * For tracers that don't provide a stat_cmp callback.
61 * This one will force an immediate insertion on tail of
62 * the list.
63 */
64static int dummy_cmp(void *p1, void *p2)
65{
66 return 1;
67}
68
69/*
70 * Initialize the stat list at each trace_stat file opening.
71 * All of these copies and sorting are required on all opening
72 * since the stats could have changed between two file sessions.
73 */
74static int stat_seq_init(struct tracer_stat_session *session)
75{
76 struct trace_stat_list *iter_entry, *new_entry;
77 struct tracer_stat *ts = session->ts;
78 void *prev_stat;
79 int ret = 0;
80 int i;
81
82 mutex_lock(&session->stat_mutex);
83 reset_stat_session(session);
84
85 if (!ts->stat_cmp)
86 ts->stat_cmp = dummy_cmp;
87
88 /*
89 * The first entry. Actually this is the second, but the first
90 * one (the stat_list head) is pointless.
91 */
92 new_entry = kmalloc(sizeof(struct trace_stat_list), GFP_KERNEL);
93 if (!new_entry) {
94 ret = -ENOMEM;
95 goto exit;
96 }
97
98 INIT_LIST_HEAD(&new_entry->list);
99
100 list_add(&new_entry->list, &session->stat_list);
101
102 new_entry->stat = ts->stat_start();
103 prev_stat = new_entry->stat;
104
105 /*
106 * Iterate over the tracer stat entries and store them in a sorted
107 * list.
108 */
109 for (i = 1; ; i++) {
110 new_entry = kmalloc(sizeof(struct trace_stat_list), GFP_KERNEL);
111 if (!new_entry) {
112 ret = -ENOMEM;
113 goto exit_free_list;
114 }
115
116 INIT_LIST_HEAD(&new_entry->list);
117 new_entry->stat = ts->stat_next(prev_stat, i);
118
119 /* End of insertion */
120 if (!new_entry->stat)
121 break;
122
123 list_for_each_entry(iter_entry, &session->stat_list, list) {
124
125 /* Insertion with a descendent sorting */
126 if (ts->stat_cmp(new_entry->stat,
127 iter_entry->stat) > 0) {
128
129 list_add_tail(&new_entry->list,
130 &iter_entry->list);
131 break;
132
133 /* The current smaller value */
134 } else if (list_is_last(&iter_entry->list,
135 &session->stat_list)) {
136 list_add(&new_entry->list, &iter_entry->list);
137 break;
138 }
139 }
140
141 prev_stat = new_entry->stat;
142 }
143exit:
144 mutex_unlock(&session->stat_mutex);
145 return ret;
146
147exit_free_list:
148 reset_stat_session(session);
149 mutex_unlock(&session->stat_mutex);
150 return ret;
151}
152
153
154static void *stat_seq_start(struct seq_file *s, loff_t *pos)
155{
156 struct tracer_stat_session *session = s->private;
157
158 /* Prevent from tracer switch or stat_list modification */
159 mutex_lock(&session->stat_mutex);
160
161 /* If we are in the beginning of the file, print the headers */
162 if (!*pos && session->ts->stat_headers)
163 session->ts->stat_headers(s);
164
165 return seq_list_start(&session->stat_list, *pos);
166}
167
168static void *stat_seq_next(struct seq_file *s, void *p, loff_t *pos)
169{
170 struct tracer_stat_session *session = s->private;
171
172 return seq_list_next(p, &session->stat_list, pos);
173}
174
175static void stat_seq_stop(struct seq_file *s, void *p)
176{
177 struct tracer_stat_session *session = s->private;
178 mutex_unlock(&session->stat_mutex);
179}
180
181static int stat_seq_show(struct seq_file *s, void *v)
182{
183 struct tracer_stat_session *session = s->private;
184 struct trace_stat_list *l = list_entry(v, struct trace_stat_list, list);
185
186 return session->ts->stat_show(s, l->stat);
187}
188
189static const struct seq_operations trace_stat_seq_ops = {
190 .start = stat_seq_start,
191 .next = stat_seq_next,
192 .stop = stat_seq_stop,
193 .show = stat_seq_show
194};
195
196/* The session stat is refilled and resorted at each stat file opening */
197static int tracing_stat_open(struct inode *inode, struct file *file)
198{
199 int ret;
200
201 struct tracer_stat_session *session = inode->i_private;
202
203 ret = seq_open(file, &trace_stat_seq_ops);
204 if (!ret) {
205 struct seq_file *m = file->private_data;
206 m->private = session;
207 ret = stat_seq_init(session);
208 }
209
210 return ret;
211}
212
213/*
214 * Avoid consuming memory with our now useless list.
215 */
216static int tracing_stat_release(struct inode *i, struct file *f)
217{
218 struct tracer_stat_session *session = i->i_private;
219
220 mutex_lock(&session->stat_mutex);
221 reset_stat_session(session);
222 mutex_unlock(&session->stat_mutex);
223
224 return 0;
225}
226
227static const struct file_operations tracing_stat_fops = {
228 .open = tracing_stat_open,
229 .read = seq_read,
230 .llseek = seq_lseek,
231 .release = tracing_stat_release
232};
233
234static int tracing_stat_init(void)
235{
236 struct dentry *d_tracing;
237
238 d_tracing = tracing_init_dentry();
239
240 stat_dir = debugfs_create_dir("trace_stat", d_tracing);
241 if (!stat_dir)
242 pr_warning("Could not create debugfs "
243 "'trace_stat' entry\n");
244 return 0;
245}
246
247static int init_stat_file(struct tracer_stat_session *session)
248{
249 if (!stat_dir && tracing_stat_init())
250 return -ENODEV;
251
252 session->file = debugfs_create_file(session->ts->name, 0644,
253 stat_dir,
254 session, &tracing_stat_fops);
255 if (!session->file)
256 return -ENOMEM;
257 return 0;
258}
259
260int register_stat_tracer(struct tracer_stat *trace)
261{
262 struct tracer_stat_session *session, *node, *tmp;
263 int ret;
264
265 if (!trace)
266 return -EINVAL;
267
268 if (!trace->stat_start || !trace->stat_next || !trace->stat_show)
269 return -EINVAL;
270
271 /* Already registered? */
272 mutex_lock(&all_stat_sessions_mutex);
273 list_for_each_entry_safe(node, tmp, &all_stat_sessions, session_list) {
274 if (node->ts == trace) {
275 mutex_unlock(&all_stat_sessions_mutex);
276 return -EINVAL;
277 }
278 }
279 mutex_unlock(&all_stat_sessions_mutex);
280
281 /* Init the session */
282 session = kmalloc(sizeof(struct tracer_stat_session), GFP_KERNEL);
283 if (!session)
284 return -ENOMEM;
285
286 session->ts = trace;
287 INIT_LIST_HEAD(&session->session_list);
288 INIT_LIST_HEAD(&session->stat_list);
289 mutex_init(&session->stat_mutex);
290 session->file = NULL;
291
292 ret = init_stat_file(session);
293 if (ret) {
294 destroy_session(session);
295 return ret;
296 }
297
298 /* Register */
299 mutex_lock(&all_stat_sessions_mutex);
300 list_add_tail(&session->session_list, &all_stat_sessions);
301 mutex_unlock(&all_stat_sessions_mutex);
302
303 return 0;
304}
305
306void unregister_stat_tracer(struct tracer_stat *trace)
307{
308 struct tracer_stat_session *node, *tmp;
309
310 mutex_lock(&all_stat_sessions_mutex);
311 list_for_each_entry_safe(node, tmp, &all_stat_sessions, session_list) {
312 if (node->ts == trace) {
313 list_del(&node->session_list);
314 destroy_session(node);
315 break;
316 }
317 }
318 mutex_unlock(&all_stat_sessions_mutex);
319}
diff --git a/kernel/trace/trace_stat.h b/kernel/trace/trace_stat.h
new file mode 100644
index 000000000000..202274cf7f3d
--- /dev/null
+++ b/kernel/trace/trace_stat.h
@@ -0,0 +1,31 @@
1#ifndef __TRACE_STAT_H
2#define __TRACE_STAT_H
3
4#include <linux/seq_file.h>
5
6/*
7 * If you want to provide a stat file (one-shot statistics), fill
8 * an iterator with stat_start/stat_next and a stat_show callbacks.
9 * The others callbacks are optional.
10 */
11struct tracer_stat {
12 /* The name of your stat file */
13 const char *name;
14 /* Iteration over statistic entries */
15 void *(*stat_start)(void);
16 void *(*stat_next)(void *prev, int idx);
17 /* Compare two entries for stats sorting */
18 int (*stat_cmp)(void *p1, void *p2);
19 /* Print a stat entry */
20 int (*stat_show)(struct seq_file *s, void *p);
21 /* Print the headers of your stat entries */
22 int (*stat_headers)(struct seq_file *s);
23};
24
25/*
26 * Destroy or create a stat file
27 */
28extern int register_stat_tracer(struct tracer_stat *trace);
29extern void unregister_stat_tracer(struct tracer_stat *trace);
30
31#endif /* __TRACE_STAT_H */
diff --git a/kernel/trace/trace_sysprof.c b/kernel/trace/trace_sysprof.c
index eaca5ad803ff..c771af4e8f1a 100644
--- a/kernel/trace/trace_sysprof.c
+++ b/kernel/trace/trace_sysprof.c
@@ -88,7 +88,7 @@ static void backtrace_address(void *data, unsigned long addr, int reliable)
88 } 88 }
89} 89}
90 90
91const static struct stacktrace_ops backtrace_ops = { 91static const struct stacktrace_ops backtrace_ops = {
92 .warning = backtrace_warning, 92 .warning = backtrace_warning,
93 .warning_symbol = backtrace_warning_symbol, 93 .warning_symbol = backtrace_warning_symbol,
94 .stack = backtrace_stack, 94 .stack = backtrace_stack,
@@ -226,15 +226,6 @@ static void stop_stack_timers(void)
226 stop_stack_timer(cpu); 226 stop_stack_timer(cpu);
227} 227}
228 228
229static void start_stack_trace(struct trace_array *tr)
230{
231 mutex_lock(&sample_timer_lock);
232 tracing_reset_online_cpus(tr);
233 start_stack_timers();
234 tracer_enabled = 1;
235 mutex_unlock(&sample_timer_lock);
236}
237
238static void stop_stack_trace(struct trace_array *tr) 229static void stop_stack_trace(struct trace_array *tr)
239{ 230{
240 mutex_lock(&sample_timer_lock); 231 mutex_lock(&sample_timer_lock);
@@ -247,12 +238,18 @@ static int stack_trace_init(struct trace_array *tr)
247{ 238{
248 sysprof_trace = tr; 239 sysprof_trace = tr;
249 240
250 start_stack_trace(tr); 241 tracing_start_cmdline_record();
242
243 mutex_lock(&sample_timer_lock);
244 start_stack_timers();
245 tracer_enabled = 1;
246 mutex_unlock(&sample_timer_lock);
251 return 0; 247 return 0;
252} 248}
253 249
254static void stack_trace_reset(struct trace_array *tr) 250static void stack_trace_reset(struct trace_array *tr)
255{ 251{
252 tracing_stop_cmdline_record();
256 stop_stack_trace(tr); 253 stop_stack_trace(tr);
257} 254}
258 255
@@ -330,5 +327,5 @@ void init_tracer_sysprof_debugfs(struct dentry *d_tracer)
330 d_tracer, NULL, &sysprof_sample_fops); 327 d_tracer, NULL, &sysprof_sample_fops);
331 if (entry) 328 if (entry)
332 return; 329 return;
333 pr_warning("Could not create debugfs 'dyn_ftrace_total_info' entry\n"); 330 pr_warning("Could not create debugfs 'sysprof_sample_period' entry\n");
334} 331}
diff --git a/kernel/trace/trace_workqueue.c b/kernel/trace/trace_workqueue.c
new file mode 100644
index 000000000000..4664990fe9c5
--- /dev/null
+++ b/kernel/trace/trace_workqueue.c
@@ -0,0 +1,281 @@
1/*
2 * Workqueue statistical tracer.
3 *
4 * Copyright (C) 2008 Frederic Weisbecker <fweisbec@gmail.com>
5 *
6 */
7
8
9#include <trace/workqueue.h>
10#include <linux/list.h>
11#include <linux/percpu.h>
12#include "trace_stat.h"
13#include "trace.h"
14
15
16/* A cpu workqueue thread */
17struct cpu_workqueue_stats {
18 struct list_head list;
19/* Useful to know if we print the cpu headers */
20 bool first_entry;
21 int cpu;
22 pid_t pid;
23/* Can be inserted from interrupt or user context, need to be atomic */
24 atomic_t inserted;
25/*
26 * Don't need to be atomic, works are serialized in a single workqueue thread
27 * on a single CPU.
28 */
29 unsigned int executed;
30};
31
32/* List of workqueue threads on one cpu */
33struct workqueue_global_stats {
34 struct list_head list;
35 spinlock_t lock;
36};
37
38/* Don't need a global lock because allocated before the workqueues, and
39 * never freed.
40 */
41static DEFINE_PER_CPU(struct workqueue_global_stats, all_workqueue_stat);
42#define workqueue_cpu_stat(cpu) (&per_cpu(all_workqueue_stat, cpu))
43
44/* Insertion of a work */
45static void
46probe_workqueue_insertion(struct task_struct *wq_thread,
47 struct work_struct *work)
48{
49 int cpu = cpumask_first(&wq_thread->cpus_allowed);
50 struct cpu_workqueue_stats *node, *next;
51 unsigned long flags;
52
53 spin_lock_irqsave(&workqueue_cpu_stat(cpu)->lock, flags);
54 list_for_each_entry_safe(node, next, &workqueue_cpu_stat(cpu)->list,
55 list) {
56 if (node->pid == wq_thread->pid) {
57 atomic_inc(&node->inserted);
58 goto found;
59 }
60 }
61 pr_debug("trace_workqueue: entry not found\n");
62found:
63 spin_unlock_irqrestore(&workqueue_cpu_stat(cpu)->lock, flags);
64}
65
66/* Execution of a work */
67static void
68probe_workqueue_execution(struct task_struct *wq_thread,
69 struct work_struct *work)
70{
71 int cpu = cpumask_first(&wq_thread->cpus_allowed);
72 struct cpu_workqueue_stats *node, *next;
73 unsigned long flags;
74
75 spin_lock_irqsave(&workqueue_cpu_stat(cpu)->lock, flags);
76 list_for_each_entry_safe(node, next, &workqueue_cpu_stat(cpu)->list,
77 list) {
78 if (node->pid == wq_thread->pid) {
79 node->executed++;
80 goto found;
81 }
82 }
83 pr_debug("trace_workqueue: entry not found\n");
84found:
85 spin_unlock_irqrestore(&workqueue_cpu_stat(cpu)->lock, flags);
86}
87
88/* Creation of a cpu workqueue thread */
89static void probe_workqueue_creation(struct task_struct *wq_thread, int cpu)
90{
91 struct cpu_workqueue_stats *cws;
92 unsigned long flags;
93
94 WARN_ON(cpu < 0 || cpu >= num_possible_cpus());
95
96 /* Workqueues are sometimes created in atomic context */
97 cws = kzalloc(sizeof(struct cpu_workqueue_stats), GFP_ATOMIC);
98 if (!cws) {
99 pr_warning("trace_workqueue: not enough memory\n");
100 return;
101 }
102 tracing_record_cmdline(wq_thread);
103
104 INIT_LIST_HEAD(&cws->list);
105 cws->cpu = cpu;
106
107 cws->pid = wq_thread->pid;
108
109 spin_lock_irqsave(&workqueue_cpu_stat(cpu)->lock, flags);
110 if (list_empty(&workqueue_cpu_stat(cpu)->list))
111 cws->first_entry = true;
112 list_add_tail(&cws->list, &workqueue_cpu_stat(cpu)->list);
113 spin_unlock_irqrestore(&workqueue_cpu_stat(cpu)->lock, flags);
114}
115
116/* Destruction of a cpu workqueue thread */
117static void probe_workqueue_destruction(struct task_struct *wq_thread)
118{
119 /* Workqueue only execute on one cpu */
120 int cpu = cpumask_first(&wq_thread->cpus_allowed);
121 struct cpu_workqueue_stats *node, *next;
122 unsigned long flags;
123
124 spin_lock_irqsave(&workqueue_cpu_stat(cpu)->lock, flags);
125 list_for_each_entry_safe(node, next, &workqueue_cpu_stat(cpu)->list,
126 list) {
127 if (node->pid == wq_thread->pid) {
128 list_del(&node->list);
129 kfree(node);
130 goto found;
131 }
132 }
133
134 pr_debug("trace_workqueue: don't find workqueue to destroy\n");
135found:
136 spin_unlock_irqrestore(&workqueue_cpu_stat(cpu)->lock, flags);
137
138}
139
140static struct cpu_workqueue_stats *workqueue_stat_start_cpu(int cpu)
141{
142 unsigned long flags;
143 struct cpu_workqueue_stats *ret = NULL;
144
145
146 spin_lock_irqsave(&workqueue_cpu_stat(cpu)->lock, flags);
147
148 if (!list_empty(&workqueue_cpu_stat(cpu)->list))
149 ret = list_entry(workqueue_cpu_stat(cpu)->list.next,
150 struct cpu_workqueue_stats, list);
151
152 spin_unlock_irqrestore(&workqueue_cpu_stat(cpu)->lock, flags);
153
154 return ret;
155}
156
157static void *workqueue_stat_start(void)
158{
159 int cpu;
160 void *ret = NULL;
161
162 for_each_possible_cpu(cpu) {
163 ret = workqueue_stat_start_cpu(cpu);
164 if (ret)
165 return ret;
166 }
167 return NULL;
168}
169
170static void *workqueue_stat_next(void *prev, int idx)
171{
172 struct cpu_workqueue_stats *prev_cws = prev;
173 int cpu = prev_cws->cpu;
174 unsigned long flags;
175 void *ret = NULL;
176
177 spin_lock_irqsave(&workqueue_cpu_stat(cpu)->lock, flags);
178 if (list_is_last(&prev_cws->list, &workqueue_cpu_stat(cpu)->list)) {
179 spin_unlock_irqrestore(&workqueue_cpu_stat(cpu)->lock, flags);
180 for (++cpu ; cpu < num_possible_cpus(); cpu++) {
181 ret = workqueue_stat_start_cpu(cpu);
182 if (ret)
183 return ret;
184 }
185 return NULL;
186 }
187 spin_unlock_irqrestore(&workqueue_cpu_stat(cpu)->lock, flags);
188
189 return list_entry(prev_cws->list.next, struct cpu_workqueue_stats,
190 list);
191}
192
193static int workqueue_stat_show(struct seq_file *s, void *p)
194{
195 struct cpu_workqueue_stats *cws = p;
196 unsigned long flags;
197 int cpu = cws->cpu;
198
199 seq_printf(s, "%3d %6d %6u %s\n", cws->cpu,
200 atomic_read(&cws->inserted),
201 cws->executed,
202 trace_find_cmdline(cws->pid));
203
204 spin_lock_irqsave(&workqueue_cpu_stat(cpu)->lock, flags);
205 if (&cws->list == workqueue_cpu_stat(cpu)->list.next)
206 seq_printf(s, "\n");
207 spin_unlock_irqrestore(&workqueue_cpu_stat(cpu)->lock, flags);
208
209 return 0;
210}
211
212static int workqueue_stat_headers(struct seq_file *s)
213{
214 seq_printf(s, "# CPU INSERTED EXECUTED NAME\n");
215 seq_printf(s, "# | | | |\n\n");
216 return 0;
217}
218
219struct tracer_stat workqueue_stats __read_mostly = {
220 .name = "workqueues",
221 .stat_start = workqueue_stat_start,
222 .stat_next = workqueue_stat_next,
223 .stat_show = workqueue_stat_show,
224 .stat_headers = workqueue_stat_headers
225};
226
227
228int __init stat_workqueue_init(void)
229{
230 if (register_stat_tracer(&workqueue_stats)) {
231 pr_warning("Unable to register workqueue stat tracer\n");
232 return 1;
233 }
234
235 return 0;
236}
237fs_initcall(stat_workqueue_init);
238
239/*
240 * Workqueues are created very early, just after pre-smp initcalls.
241 * So we must register our tracepoints at this stage.
242 */
243int __init trace_workqueue_early_init(void)
244{
245 int ret, cpu;
246
247 ret = register_trace_workqueue_insertion(probe_workqueue_insertion);
248 if (ret)
249 goto out;
250
251 ret = register_trace_workqueue_execution(probe_workqueue_execution);
252 if (ret)
253 goto no_insertion;
254
255 ret = register_trace_workqueue_creation(probe_workqueue_creation);
256 if (ret)
257 goto no_execution;
258
259 ret = register_trace_workqueue_destruction(probe_workqueue_destruction);
260 if (ret)
261 goto no_creation;
262
263 for_each_possible_cpu(cpu) {
264 spin_lock_init(&workqueue_cpu_stat(cpu)->lock);
265 INIT_LIST_HEAD(&workqueue_cpu_stat(cpu)->list);
266 }
267
268 return 0;
269
270no_creation:
271 unregister_trace_workqueue_creation(probe_workqueue_creation);
272no_execution:
273 unregister_trace_workqueue_execution(probe_workqueue_execution);
274no_insertion:
275 unregister_trace_workqueue_insertion(probe_workqueue_insertion);
276out:
277 pr_warning("trace_workqueue: unable to trace workqueues\n");
278
279 return 1;
280}
281early_initcall(trace_workqueue_early_init);
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index 1f0c509b40d3..e53ee18ef431 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -33,6 +33,7 @@
33#include <linux/kallsyms.h> 33#include <linux/kallsyms.h>
34#include <linux/debug_locks.h> 34#include <linux/debug_locks.h>
35#include <linux/lockdep.h> 35#include <linux/lockdep.h>
36#include <trace/workqueue.h>
36 37
37/* 38/*
38 * The per-CPU workqueue (if single thread, we always use the first 39 * The per-CPU workqueue (if single thread, we always use the first
@@ -125,9 +126,13 @@ struct cpu_workqueue_struct *get_wq_data(struct work_struct *work)
125 return (void *) (atomic_long_read(&work->data) & WORK_STRUCT_WQ_DATA_MASK); 126 return (void *) (atomic_long_read(&work->data) & WORK_STRUCT_WQ_DATA_MASK);
126} 127}
127 128
129DEFINE_TRACE(workqueue_insertion);
130
128static void insert_work(struct cpu_workqueue_struct *cwq, 131static void insert_work(struct cpu_workqueue_struct *cwq,
129 struct work_struct *work, struct list_head *head) 132 struct work_struct *work, struct list_head *head)
130{ 133{
134 trace_workqueue_insertion(cwq->thread, work);
135
131 set_wq_data(work, cwq); 136 set_wq_data(work, cwq);
132 /* 137 /*
133 * Ensure that we get the right work->data if we see the 138 * Ensure that we get the right work->data if we see the
@@ -259,6 +264,8 @@ int queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
259} 264}
260EXPORT_SYMBOL_GPL(queue_delayed_work_on); 265EXPORT_SYMBOL_GPL(queue_delayed_work_on);
261 266
267DEFINE_TRACE(workqueue_execution);
268
262static void run_workqueue(struct cpu_workqueue_struct *cwq) 269static void run_workqueue(struct cpu_workqueue_struct *cwq)
263{ 270{
264 spin_lock_irq(&cwq->lock); 271 spin_lock_irq(&cwq->lock);
@@ -284,7 +291,7 @@ static void run_workqueue(struct cpu_workqueue_struct *cwq)
284 */ 291 */
285 struct lockdep_map lockdep_map = work->lockdep_map; 292 struct lockdep_map lockdep_map = work->lockdep_map;
286#endif 293#endif
287 294 trace_workqueue_execution(cwq->thread, work);
288 cwq->current_work = work; 295 cwq->current_work = work;
289 list_del_init(cwq->worklist.next); 296 list_del_init(cwq->worklist.next);
290 spin_unlock_irq(&cwq->lock); 297 spin_unlock_irq(&cwq->lock);
@@ -765,6 +772,8 @@ init_cpu_workqueue(struct workqueue_struct *wq, int cpu)
765 return cwq; 772 return cwq;
766} 773}
767 774
775DEFINE_TRACE(workqueue_creation);
776
768static int create_workqueue_thread(struct cpu_workqueue_struct *cwq, int cpu) 777static int create_workqueue_thread(struct cpu_workqueue_struct *cwq, int cpu)
769{ 778{
770 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 }; 779 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
@@ -787,6 +796,8 @@ static int create_workqueue_thread(struct cpu_workqueue_struct *cwq, int cpu)
787 sched_setscheduler_nocheck(p, SCHED_FIFO, &param); 796 sched_setscheduler_nocheck(p, SCHED_FIFO, &param);
788 cwq->thread = p; 797 cwq->thread = p;
789 798
799 trace_workqueue_creation(cwq->thread, cpu);
800
790 return 0; 801 return 0;
791} 802}
792 803
@@ -868,6 +879,8 @@ struct workqueue_struct *__create_workqueue_key(const char *name,
868} 879}
869EXPORT_SYMBOL_GPL(__create_workqueue_key); 880EXPORT_SYMBOL_GPL(__create_workqueue_key);
870 881
882DEFINE_TRACE(workqueue_destruction);
883
871static void cleanup_workqueue_thread(struct cpu_workqueue_struct *cwq) 884static void cleanup_workqueue_thread(struct cpu_workqueue_struct *cwq)
872{ 885{
873 /* 886 /*
@@ -891,6 +904,7 @@ static void cleanup_workqueue_thread(struct cpu_workqueue_struct *cwq)
891 * checks list_empty(), and a "normal" queue_work() can't use 904 * checks list_empty(), and a "normal" queue_work() can't use
892 * a dead CPU. 905 * a dead CPU.
893 */ 906 */
907 trace_workqueue_destruction(cwq->thread);
894 kthread_stop(cwq->thread); 908 kthread_stop(cwq->thread);
895 cwq->thread = NULL; 909 cwq->thread = NULL;
896} 910}
generated by cgit v1.2.2 (git 2.25.0) at 2025-01-15 17:55:35 -0500