aboutsummaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/exit.c5
-rw-r--r--kernel/extable.c4
-rw-r--r--kernel/fork.c5
-rw-r--r--kernel/irq/chip.c5
-rw-r--r--kernel/irq/handle.c63
-rw-r--r--kernel/irq/internals.h7
-rw-r--r--kernel/irq/manage.c12
-rw-r--r--kernel/irq/migration.c12
-rw-r--r--kernel/irq/numa_migrate.c19
-rw-r--r--kernel/irq/proc.c4
-rw-r--r--kernel/kexec.c2
-rw-r--r--kernel/kprobes.c15
-rw-r--r--kernel/lockdep.c531
-rw-r--r--kernel/lockdep_internals.h45
-rw-r--r--kernel/lockdep_proc.c22
-rw-r--r--kernel/lockdep_states.h9
-rw-r--r--kernel/module.c66
-rw-r--r--kernel/mutex-debug.c9
-rw-r--r--kernel/mutex-debug.h18
-rw-r--r--kernel/mutex.c121
-rw-r--r--kernel/mutex.h22
-rw-r--r--kernel/panic.c8
-rw-r--r--kernel/relay.c4
-rw-r--r--kernel/sched.c92
-rw-r--r--kernel/sched_clock.c53
-rw-r--r--kernel/sched_features.h1
-rw-r--r--kernel/sched_rt.c32
-rw-r--r--kernel/softirq.c18
-rw-r--r--kernel/stop_machine.c2
-rw-r--r--kernel/timer.c68
-rw-r--r--kernel/trace/Kconfig108
-rw-r--r--kernel/trace/Makefile10
-rw-r--r--kernel/trace/blktrace.c1537
-rw-r--r--kernel/trace/events.c15
-rw-r--r--kernel/trace/ftrace.c1029
-rw-r--r--kernel/trace/kmemtrace.c339
-rw-r--r--kernel/trace/ring_buffer.c281
-rw-r--r--kernel/trace/trace.c2633
-rw-r--r--kernel/trace/trace.h189
-rw-r--r--kernel/trace/trace_boot.c36
-rw-r--r--kernel/trace/trace_branch.c278
-rw-r--r--kernel/trace/trace_clock.c108
-rw-r--r--kernel/trace/trace_event_types.h166
-rw-r--r--kernel/trace/trace_events.c581
-rw-r--r--kernel/trace/trace_events_stage_1.h37
-rw-r--r--kernel/trace/trace_events_stage_2.h132
-rw-r--r--kernel/trace/trace_events_stage_3.h217
-rw-r--r--kernel/trace/trace_export.c102
-rw-r--r--kernel/trace/trace_functions.c369
-rw-r--r--kernel/trace/trace_functions_graph.c480
-rw-r--r--kernel/trace/trace_hw_branches.c185
-rw-r--r--kernel/trace/trace_irqsoff.c54
-rw-r--r--kernel/trace/trace_mmiotrace.c46
-rw-r--r--kernel/trace/trace_nop.c5
-rw-r--r--kernel/trace/trace_output.c916
-rw-r--r--kernel/trace/trace_output.h63
-rw-r--r--kernel/trace/trace_power.c188
-rw-r--r--kernel/trace/trace_printk.c138
-rw-r--r--kernel/trace/trace_sched_switch.c15
-rw-r--r--kernel/trace/trace_sched_wakeup.c96
-rw-r--r--kernel/trace/trace_selftest.c91
-rw-r--r--kernel/trace/trace_stat.c319
-rw-r--r--kernel/trace/trace_stat.h31
-rw-r--r--kernel/trace/trace_sysprof.c23
-rw-r--r--kernel/trace/trace_workqueue.c280
-rw-r--r--kernel/workqueue.c16
66 files changed, 9731 insertions, 2656 deletions
diff --git a/kernel/exit.c b/kernel/exit.c
index efd30ccf3858..167e1e3ad7c6 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -980,12 +980,9 @@ static void check_stack_usage(void)
980{ 980{
981 static DEFINE_SPINLOCK(low_water_lock); 981 static DEFINE_SPINLOCK(low_water_lock);
982 static int lowest_to_date = THREAD_SIZE; 982 static int lowest_to_date = THREAD_SIZE;
983 unsigned long *n = end_of_stack(current);
984 unsigned long free; 983 unsigned long free;
985 984
986 while (*n == 0) 985 free = stack_not_used(current);
987 n++;
988 free = (unsigned long)n - (unsigned long)end_of_stack(current);
989 986
990 if (free >= lowest_to_date) 987 if (free >= lowest_to_date)
991 return; 988 return;
diff --git a/kernel/extable.c b/kernel/extable.c
index e136ed8d82ba..0df6253730be 100644
--- a/kernel/extable.c
+++ b/kernel/extable.c
@@ -41,7 +41,7 @@ const struct exception_table_entry *search_exception_tables(unsigned long addr)
41 return e; 41 return e;
42} 42}
43 43
44__notrace_funcgraph int core_kernel_text(unsigned long addr) 44int core_kernel_text(unsigned long addr)
45{ 45{
46 if (addr >= (unsigned long)_stext && 46 if (addr >= (unsigned long)_stext &&
47 addr <= (unsigned long)_etext) 47 addr <= (unsigned long)_etext)
@@ -54,7 +54,7 @@ __notrace_funcgraph int core_kernel_text(unsigned long addr)
54 return 0; 54 return 0;
55} 55}
56 56
57__notrace_funcgraph int __kernel_text_address(unsigned long addr) 57int __kernel_text_address(unsigned long addr)
58{ 58{
59 if (core_kernel_text(addr)) 59 if (core_kernel_text(addr))
60 return 1; 60 return 1;
diff --git a/kernel/fork.c b/kernel/fork.c
index 4854c2c4a82e..6715ebc3761d 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -61,6 +61,7 @@
61#include <linux/proc_fs.h> 61#include <linux/proc_fs.h>
62#include <linux/blkdev.h> 62#include <linux/blkdev.h>
63#include <trace/sched.h> 63#include <trace/sched.h>
64#include <linux/magic.h>
64 65
65#include <asm/pgtable.h> 66#include <asm/pgtable.h>
66#include <asm/pgalloc.h> 67#include <asm/pgalloc.h>
@@ -212,6 +213,8 @@ static struct task_struct *dup_task_struct(struct task_struct *orig)
212{ 213{
213 struct task_struct *tsk; 214 struct task_struct *tsk;
214 struct thread_info *ti; 215 struct thread_info *ti;
216 unsigned long *stackend;
217
215 int err; 218 int err;
216 219
217 prepare_to_copy(orig); 220 prepare_to_copy(orig);
@@ -237,6 +240,8 @@ static struct task_struct *dup_task_struct(struct task_struct *orig)
237 goto out; 240 goto out;
238 241
239 setup_thread_stack(tsk, orig); 242 setup_thread_stack(tsk, orig);
243 stackend = end_of_stack(tsk);
244 *stackend = STACK_END_MAGIC; /* for overflow detection */
240 245
241#ifdef CONFIG_CC_STACKPROTECTOR 246#ifdef CONFIG_CC_STACKPROTECTOR
242 tsk->stack_canary = get_random_int(); 247 tsk->stack_canary = get_random_int();
diff --git a/kernel/irq/chip.c b/kernel/irq/chip.c
index 7de11bd64dfe..122fef4b0bd3 100644
--- a/kernel/irq/chip.c
+++ b/kernel/irq/chip.c
@@ -46,7 +46,10 @@ void dynamic_irq_init(unsigned int irq)
46 desc->irq_count = 0; 46 desc->irq_count = 0;
47 desc->irqs_unhandled = 0; 47 desc->irqs_unhandled = 0;
48#ifdef CONFIG_SMP 48#ifdef CONFIG_SMP
49 cpumask_setall(&desc->affinity); 49 cpumask_setall(desc->affinity);
50#ifdef CONFIG_GENERIC_PENDING_IRQ
51 cpumask_clear(desc->pending_mask);
52#endif
50#endif 53#endif
51 spin_unlock_irqrestore(&desc->lock, flags); 54 spin_unlock_irqrestore(&desc->lock, flags);
52} 55}
diff --git a/kernel/irq/handle.c b/kernel/irq/handle.c
index 3aba8d12f328..412370ab9a34 100644
--- a/kernel/irq/handle.c
+++ b/kernel/irq/handle.c
@@ -17,6 +17,8 @@
17#include <linux/kernel_stat.h> 17#include <linux/kernel_stat.h>
18#include <linux/rculist.h> 18#include <linux/rculist.h>
19#include <linux/hash.h> 19#include <linux/hash.h>
20#include <trace/irq.h>
21#include <linux/bootmem.h>
20 22
21#include "internals.h" 23#include "internals.h"
22 24
@@ -69,6 +71,7 @@ int nr_irqs = NR_IRQS;
69EXPORT_SYMBOL_GPL(nr_irqs); 71EXPORT_SYMBOL_GPL(nr_irqs);
70 72
71#ifdef CONFIG_SPARSE_IRQ 73#ifdef CONFIG_SPARSE_IRQ
74
72static struct irq_desc irq_desc_init = { 75static struct irq_desc irq_desc_init = {
73 .irq = -1, 76 .irq = -1,
74 .status = IRQ_DISABLED, 77 .status = IRQ_DISABLED,
@@ -76,9 +79,6 @@ static struct irq_desc irq_desc_init = {
76 .handle_irq = handle_bad_irq, 79 .handle_irq = handle_bad_irq,
77 .depth = 1, 80 .depth = 1,
78 .lock = __SPIN_LOCK_UNLOCKED(irq_desc_init.lock), 81 .lock = __SPIN_LOCK_UNLOCKED(irq_desc_init.lock),
79#ifdef CONFIG_SMP
80 .affinity = CPU_MASK_ALL
81#endif
82}; 82};
83 83
84void init_kstat_irqs(struct irq_desc *desc, int cpu, int nr) 84void init_kstat_irqs(struct irq_desc *desc, int cpu, int nr)
@@ -113,6 +113,10 @@ static void init_one_irq_desc(int irq, struct irq_desc *desc, int cpu)
113 printk(KERN_ERR "can not alloc kstat_irqs\n"); 113 printk(KERN_ERR "can not alloc kstat_irqs\n");
114 BUG_ON(1); 114 BUG_ON(1);
115 } 115 }
116 if (!init_alloc_desc_masks(desc, cpu, false)) {
117 printk(KERN_ERR "can not alloc irq_desc cpumasks\n");
118 BUG_ON(1);
119 }
116 arch_init_chip_data(desc, cpu); 120 arch_init_chip_data(desc, cpu);
117} 121}
118 122
@@ -121,7 +125,7 @@ static void init_one_irq_desc(int irq, struct irq_desc *desc, int cpu)
121 */ 125 */
122DEFINE_SPINLOCK(sparse_irq_lock); 126DEFINE_SPINLOCK(sparse_irq_lock);
123 127
124struct irq_desc *irq_desc_ptrs[NR_IRQS] __read_mostly; 128struct irq_desc **irq_desc_ptrs __read_mostly;
125 129
126static struct irq_desc irq_desc_legacy[NR_IRQS_LEGACY] __cacheline_aligned_in_smp = { 130static struct irq_desc irq_desc_legacy[NR_IRQS_LEGACY] __cacheline_aligned_in_smp = {
127 [0 ... NR_IRQS_LEGACY-1] = { 131 [0 ... NR_IRQS_LEGACY-1] = {
@@ -131,14 +135,10 @@ static struct irq_desc irq_desc_legacy[NR_IRQS_LEGACY] __cacheline_aligned_in_sm
131 .handle_irq = handle_bad_irq, 135 .handle_irq = handle_bad_irq,
132 .depth = 1, 136 .depth = 1,
133 .lock = __SPIN_LOCK_UNLOCKED(irq_desc_init.lock), 137 .lock = __SPIN_LOCK_UNLOCKED(irq_desc_init.lock),
134#ifdef CONFIG_SMP
135 .affinity = CPU_MASK_ALL
136#endif
137 } 138 }
138}; 139};
139 140
140/* FIXME: use bootmem alloc ...*/ 141static unsigned int *kstat_irqs_legacy;
141static unsigned int kstat_irqs_legacy[NR_IRQS_LEGACY][NR_CPUS];
142 142
143int __init early_irq_init(void) 143int __init early_irq_init(void)
144{ 144{
@@ -148,18 +148,30 @@ int __init early_irq_init(void)
148 148
149 init_irq_default_affinity(); 149 init_irq_default_affinity();
150 150
151 /* initialize nr_irqs based on nr_cpu_ids */
152 arch_probe_nr_irqs();
153 printk(KERN_INFO "NR_IRQS:%d nr_irqs:%d\n", NR_IRQS, nr_irqs);
154
151 desc = irq_desc_legacy; 155 desc = irq_desc_legacy;
152 legacy_count = ARRAY_SIZE(irq_desc_legacy); 156 legacy_count = ARRAY_SIZE(irq_desc_legacy);
153 157
158 /* allocate irq_desc_ptrs array based on nr_irqs */
159 irq_desc_ptrs = alloc_bootmem(nr_irqs * sizeof(void *));
160
161 /* allocate based on nr_cpu_ids */
162 /* FIXME: invert kstat_irgs, and it'd be a per_cpu_alloc'd thing */
163 kstat_irqs_legacy = alloc_bootmem(NR_IRQS_LEGACY * nr_cpu_ids *
164 sizeof(int));
165
154 for (i = 0; i < legacy_count; i++) { 166 for (i = 0; i < legacy_count; i++) {
155 desc[i].irq = i; 167 desc[i].irq = i;
156 desc[i].kstat_irqs = kstat_irqs_legacy[i]; 168 desc[i].kstat_irqs = kstat_irqs_legacy + i * nr_cpu_ids;
157 lockdep_set_class(&desc[i].lock, &irq_desc_lock_class); 169 lockdep_set_class(&desc[i].lock, &irq_desc_lock_class);
158 170 init_alloc_desc_masks(&desc[i], 0, true);
159 irq_desc_ptrs[i] = desc + i; 171 irq_desc_ptrs[i] = desc + i;
160 } 172 }
161 173
162 for (i = legacy_count; i < NR_IRQS; i++) 174 for (i = legacy_count; i < nr_irqs; i++)
163 irq_desc_ptrs[i] = NULL; 175 irq_desc_ptrs[i] = NULL;
164 176
165 return arch_early_irq_init(); 177 return arch_early_irq_init();
@@ -167,7 +179,10 @@ int __init early_irq_init(void)
167 179
168struct irq_desc *irq_to_desc(unsigned int irq) 180struct irq_desc *irq_to_desc(unsigned int irq)
169{ 181{
170 return (irq < NR_IRQS) ? irq_desc_ptrs[irq] : NULL; 182 if (irq_desc_ptrs && irq < nr_irqs)
183 return irq_desc_ptrs[irq];
184
185 return NULL;
171} 186}
172 187
173struct irq_desc *irq_to_desc_alloc_cpu(unsigned int irq, int cpu) 188struct irq_desc *irq_to_desc_alloc_cpu(unsigned int irq, int cpu)
@@ -176,10 +191,9 @@ struct irq_desc *irq_to_desc_alloc_cpu(unsigned int irq, int cpu)
176 unsigned long flags; 191 unsigned long flags;
177 int node; 192 int node;
178 193
179 if (irq >= NR_IRQS) { 194 if (irq >= nr_irqs) {
180 printk(KERN_WARNING "irq >= NR_IRQS in irq_to_desc_alloc: %d %d\n", 195 WARN(1, "irq (%d) >= nr_irqs (%d) in irq_to_desc_alloc\n",
181 irq, NR_IRQS); 196 irq, nr_irqs);
182 WARN_ON(1);
183 return NULL; 197 return NULL;
184 } 198 }
185 199
@@ -221,9 +235,6 @@ struct irq_desc irq_desc[NR_IRQS] __cacheline_aligned_in_smp = {
221 .handle_irq = handle_bad_irq, 235 .handle_irq = handle_bad_irq,
222 .depth = 1, 236 .depth = 1,
223 .lock = __SPIN_LOCK_UNLOCKED(irq_desc->lock), 237 .lock = __SPIN_LOCK_UNLOCKED(irq_desc->lock),
224#ifdef CONFIG_SMP
225 .affinity = CPU_MASK_ALL
226#endif
227 } 238 }
228}; 239};
229 240
@@ -235,12 +246,15 @@ int __init early_irq_init(void)
235 246
236 init_irq_default_affinity(); 247 init_irq_default_affinity();
237 248
249 printk(KERN_INFO "NR_IRQS:%d\n", NR_IRQS);
250
238 desc = irq_desc; 251 desc = irq_desc;
239 count = ARRAY_SIZE(irq_desc); 252 count = ARRAY_SIZE(irq_desc);
240 253
241 for (i = 0; i < count; i++) 254 for (i = 0; i < count; i++) {
242 desc[i].irq = i; 255 desc[i].irq = i;
243 256 init_alloc_desc_masks(&desc[i], 0, true);
257 }
244 return arch_early_irq_init(); 258 return arch_early_irq_init();
245} 259}
246 260
@@ -316,6 +330,9 @@ irqreturn_t no_action(int cpl, void *dev_id)
316 return IRQ_NONE; 330 return IRQ_NONE;
317} 331}
318 332
333DEFINE_TRACE(irq_handler_entry);
334DEFINE_TRACE(irq_handler_exit);
335
319/** 336/**
320 * handle_IRQ_event - irq action chain handler 337 * handle_IRQ_event - irq action chain handler
321 * @irq: the interrupt number 338 * @irq: the interrupt number
@@ -332,7 +349,9 @@ irqreturn_t handle_IRQ_event(unsigned int irq, struct irqaction *action)
332 local_irq_enable_in_hardirq(); 349 local_irq_enable_in_hardirq();
333 350
334 do { 351 do {
352 trace_irq_handler_entry(irq, action);
335 ret = action->handler(irq, action->dev_id); 353 ret = action->handler(irq, action->dev_id);
354 trace_irq_handler_exit(irq, action, ret);
336 if (ret == IRQ_HANDLED) 355 if (ret == IRQ_HANDLED)
337 status |= action->flags; 356 status |= action->flags;
338 retval |= ret; 357 retval |= ret;
diff --git a/kernel/irq/internals.h b/kernel/irq/internals.h
index e6d0a43cc125..40416a81a0f5 100644
--- a/kernel/irq/internals.h
+++ b/kernel/irq/internals.h
@@ -16,7 +16,14 @@ extern int __irq_set_trigger(struct irq_desc *desc, unsigned int irq,
16extern struct lock_class_key irq_desc_lock_class; 16extern struct lock_class_key irq_desc_lock_class;
17extern void init_kstat_irqs(struct irq_desc *desc, int cpu, int nr); 17extern void init_kstat_irqs(struct irq_desc *desc, int cpu, int nr);
18extern spinlock_t sparse_irq_lock; 18extern spinlock_t sparse_irq_lock;
19
20#ifdef CONFIG_SPARSE_IRQ
21/* irq_desc_ptrs allocated at boot time */
22extern struct irq_desc **irq_desc_ptrs;
23#else
24/* irq_desc_ptrs is a fixed size array */
19extern struct irq_desc *irq_desc_ptrs[NR_IRQS]; 25extern struct irq_desc *irq_desc_ptrs[NR_IRQS];
26#endif
20 27
21#ifdef CONFIG_PROC_FS 28#ifdef CONFIG_PROC_FS
22extern void register_irq_proc(unsigned int irq, struct irq_desc *desc); 29extern void register_irq_proc(unsigned int irq, struct irq_desc *desc);
diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c
index 291f03664552..a3a5dc9ef346 100644
--- a/kernel/irq/manage.c
+++ b/kernel/irq/manage.c
@@ -90,14 +90,14 @@ int irq_set_affinity(unsigned int irq, const struct cpumask *cpumask)
90 90
91#ifdef CONFIG_GENERIC_PENDING_IRQ 91#ifdef CONFIG_GENERIC_PENDING_IRQ
92 if (desc->status & IRQ_MOVE_PCNTXT || desc->status & IRQ_DISABLED) { 92 if (desc->status & IRQ_MOVE_PCNTXT || desc->status & IRQ_DISABLED) {
93 cpumask_copy(&desc->affinity, cpumask); 93 cpumask_copy(desc->affinity, cpumask);
94 desc->chip->set_affinity(irq, cpumask); 94 desc->chip->set_affinity(irq, cpumask);
95 } else { 95 } else {
96 desc->status |= IRQ_MOVE_PENDING; 96 desc->status |= IRQ_MOVE_PENDING;
97 cpumask_copy(&desc->pending_mask, cpumask); 97 cpumask_copy(desc->pending_mask, cpumask);
98 } 98 }
99#else 99#else
100 cpumask_copy(&desc->affinity, cpumask); 100 cpumask_copy(desc->affinity, cpumask);
101 desc->chip->set_affinity(irq, cpumask); 101 desc->chip->set_affinity(irq, cpumask);
102#endif 102#endif
103 desc->status |= IRQ_AFFINITY_SET; 103 desc->status |= IRQ_AFFINITY_SET;
@@ -119,16 +119,16 @@ int do_irq_select_affinity(unsigned int irq, struct irq_desc *desc)
119 * one of the targets is online. 119 * one of the targets is online.
120 */ 120 */
121 if (desc->status & (IRQ_AFFINITY_SET | IRQ_NO_BALANCING)) { 121 if (desc->status & (IRQ_AFFINITY_SET | IRQ_NO_BALANCING)) {
122 if (cpumask_any_and(&desc->affinity, cpu_online_mask) 122 if (cpumask_any_and(desc->affinity, cpu_online_mask)
123 < nr_cpu_ids) 123 < nr_cpu_ids)
124 goto set_affinity; 124 goto set_affinity;
125 else 125 else
126 desc->status &= ~IRQ_AFFINITY_SET; 126 desc->status &= ~IRQ_AFFINITY_SET;
127 } 127 }
128 128
129 cpumask_and(&desc->affinity, cpu_online_mask, irq_default_affinity); 129 cpumask_and(desc->affinity, cpu_online_mask, irq_default_affinity);
130set_affinity: 130set_affinity:
131 desc->chip->set_affinity(irq, &desc->affinity); 131 desc->chip->set_affinity(irq, desc->affinity);
132 132
133 return 0; 133 return 0;
134} 134}
diff --git a/kernel/irq/migration.c b/kernel/irq/migration.c
index bd72329e630c..e05ad9be43b7 100644
--- a/kernel/irq/migration.c
+++ b/kernel/irq/migration.c
@@ -18,7 +18,7 @@ void move_masked_irq(int irq)
18 18
19 desc->status &= ~IRQ_MOVE_PENDING; 19 desc->status &= ~IRQ_MOVE_PENDING;
20 20
21 if (unlikely(cpumask_empty(&desc->pending_mask))) 21 if (unlikely(cpumask_empty(desc->pending_mask)))
22 return; 22 return;
23 23
24 if (!desc->chip->set_affinity) 24 if (!desc->chip->set_affinity)
@@ -38,13 +38,13 @@ void move_masked_irq(int irq)
38 * For correct operation this depends on the caller 38 * For correct operation this depends on the caller
39 * masking the irqs. 39 * masking the irqs.
40 */ 40 */
41 if (likely(cpumask_any_and(&desc->pending_mask, cpu_online_mask) 41 if (likely(cpumask_any_and(desc->pending_mask, cpu_online_mask)
42 < nr_cpu_ids)) { 42 < nr_cpu_ids)) {
43 cpumask_and(&desc->affinity, 43 cpumask_and(desc->affinity,
44 &desc->pending_mask, cpu_online_mask); 44 desc->pending_mask, cpu_online_mask);
45 desc->chip->set_affinity(irq, &desc->affinity); 45 desc->chip->set_affinity(irq, desc->affinity);
46 } 46 }
47 cpumask_clear(&desc->pending_mask); 47 cpumask_clear(desc->pending_mask);
48} 48}
49 49
50void move_native_irq(int irq) 50void move_native_irq(int irq)
diff --git a/kernel/irq/numa_migrate.c b/kernel/irq/numa_migrate.c
index acd88356ac76..7f9b80434e32 100644
--- a/kernel/irq/numa_migrate.c
+++ b/kernel/irq/numa_migrate.c
@@ -38,15 +38,22 @@ static void free_kstat_irqs(struct irq_desc *old_desc, struct irq_desc *desc)
38 old_desc->kstat_irqs = NULL; 38 old_desc->kstat_irqs = NULL;
39} 39}
40 40
41static void init_copy_one_irq_desc(int irq, struct irq_desc *old_desc, 41static bool init_copy_one_irq_desc(int irq, struct irq_desc *old_desc,
42 struct irq_desc *desc, int cpu) 42 struct irq_desc *desc, int cpu)
43{ 43{
44 memcpy(desc, old_desc, sizeof(struct irq_desc)); 44 memcpy(desc, old_desc, sizeof(struct irq_desc));
45 if (!init_alloc_desc_masks(desc, cpu, false)) {
46 printk(KERN_ERR "irq %d: can not get new irq_desc cpumask "
47 "for migration.\n", irq);
48 return false;
49 }
45 spin_lock_init(&desc->lock); 50 spin_lock_init(&desc->lock);
46 desc->cpu = cpu; 51 desc->cpu = cpu;
47 lockdep_set_class(&desc->lock, &irq_desc_lock_class); 52 lockdep_set_class(&desc->lock, &irq_desc_lock_class);
48 init_copy_kstat_irqs(old_desc, desc, cpu, nr_cpu_ids); 53 init_copy_kstat_irqs(old_desc, desc, cpu, nr_cpu_ids);
54 init_copy_desc_masks(old_desc, desc);
49 arch_init_copy_chip_data(old_desc, desc, cpu); 55 arch_init_copy_chip_data(old_desc, desc, cpu);
56 return true;
50} 57}
51 58
52static void free_one_irq_desc(struct irq_desc *old_desc, struct irq_desc *desc) 59static void free_one_irq_desc(struct irq_desc *old_desc, struct irq_desc *desc)
@@ -76,12 +83,18 @@ static struct irq_desc *__real_move_irq_desc(struct irq_desc *old_desc,
76 node = cpu_to_node(cpu); 83 node = cpu_to_node(cpu);
77 desc = kzalloc_node(sizeof(*desc), GFP_ATOMIC, node); 84 desc = kzalloc_node(sizeof(*desc), GFP_ATOMIC, node);
78 if (!desc) { 85 if (!desc) {
79 printk(KERN_ERR "irq %d: can not get new irq_desc for migration.\n", irq); 86 printk(KERN_ERR "irq %d: can not get new irq_desc "
87 "for migration.\n", irq);
88 /* still use old one */
89 desc = old_desc;
90 goto out_unlock;
91 }
92 if (!init_copy_one_irq_desc(irq, old_desc, desc, cpu)) {
80 /* still use old one */ 93 /* still use old one */
94 kfree(desc);
81 desc = old_desc; 95 desc = old_desc;
82 goto out_unlock; 96 goto out_unlock;
83 } 97 }
84 init_copy_one_irq_desc(irq, old_desc, desc, cpu);
85 98
86 irq_desc_ptrs[irq] = desc; 99 irq_desc_ptrs[irq] = desc;
87 spin_unlock_irqrestore(&sparse_irq_lock, flags); 100 spin_unlock_irqrestore(&sparse_irq_lock, flags);
diff --git a/kernel/irq/proc.c b/kernel/irq/proc.c
index aae3f742bcec..692363dd591f 100644
--- a/kernel/irq/proc.c
+++ b/kernel/irq/proc.c
@@ -20,11 +20,11 @@ static struct proc_dir_entry *root_irq_dir;
20static int irq_affinity_proc_show(struct seq_file *m, void *v) 20static int irq_affinity_proc_show(struct seq_file *m, void *v)
21{ 21{
22 struct irq_desc *desc = irq_to_desc((long)m->private); 22 struct irq_desc *desc = irq_to_desc((long)m->private);
23 const struct cpumask *mask = &desc->affinity; 23 const struct cpumask *mask = desc->affinity;
24 24
25#ifdef CONFIG_GENERIC_PENDING_IRQ 25#ifdef CONFIG_GENERIC_PENDING_IRQ
26 if (desc->status & IRQ_MOVE_PENDING) 26 if (desc->status & IRQ_MOVE_PENDING)
27 mask = &desc->pending_mask; 27 mask = desc->pending_mask;
28#endif 28#endif
29 seq_cpumask(m, mask); 29 seq_cpumask(m, mask);
30 seq_putc(m, '\n'); 30 seq_putc(m, '\n');
diff --git a/kernel/kexec.c b/kernel/kexec.c
index 483899578259..c7fd6692939d 100644
--- a/kernel/kexec.c
+++ b/kernel/kexec.c
@@ -1130,7 +1130,7 @@ void crash_save_cpu(struct pt_regs *regs, int cpu)
1130 return; 1130 return;
1131 memset(&prstatus, 0, sizeof(prstatus)); 1131 memset(&prstatus, 0, sizeof(prstatus));
1132 prstatus.pr_pid = current->pid; 1132 prstatus.pr_pid = current->pid;
1133 elf_core_copy_regs(&prstatus.pr_reg, regs); 1133 elf_core_copy_kernel_regs(&prstatus.pr_reg, regs);
1134 buf = append_elf_note(buf, KEXEC_CORE_NOTE_NAME, NT_PRSTATUS, 1134 buf = append_elf_note(buf, KEXEC_CORE_NOTE_NAME, NT_PRSTATUS,
1135 &prstatus, sizeof(prstatus)); 1135 &prstatus, sizeof(prstatus));
1136 final_note(buf); 1136 final_note(buf);
diff --git a/kernel/kprobes.c b/kernel/kprobes.c
index 7ba8cd9845cb..479d4d5672f9 100644
--- a/kernel/kprobes.c
+++ b/kernel/kprobes.c
@@ -43,6 +43,7 @@
43#include <linux/seq_file.h> 43#include <linux/seq_file.h>
44#include <linux/debugfs.h> 44#include <linux/debugfs.h>
45#include <linux/kdebug.h> 45#include <linux/kdebug.h>
46#include <linux/memory.h>
46 47
47#include <asm-generic/sections.h> 48#include <asm-generic/sections.h>
48#include <asm/cacheflush.h> 49#include <asm/cacheflush.h>
@@ -699,9 +700,10 @@ int __kprobes register_kprobe(struct kprobe *p)
699 goto out; 700 goto out;
700 } 701 }
701 702
703 mutex_lock(&text_mutex);
702 ret = arch_prepare_kprobe(p); 704 ret = arch_prepare_kprobe(p);
703 if (ret) 705 if (ret)
704 goto out; 706 goto out_unlock_text;
705 707
706 INIT_HLIST_NODE(&p->hlist); 708 INIT_HLIST_NODE(&p->hlist);
707 hlist_add_head_rcu(&p->hlist, 709 hlist_add_head_rcu(&p->hlist,
@@ -710,6 +712,8 @@ int __kprobes register_kprobe(struct kprobe *p)
710 if (kprobe_enabled) 712 if (kprobe_enabled)
711 arch_arm_kprobe(p); 713 arch_arm_kprobe(p);
712 714
715out_unlock_text:
716 mutex_unlock(&text_mutex);
713out: 717out:
714 mutex_unlock(&kprobe_mutex); 718 mutex_unlock(&kprobe_mutex);
715 719
@@ -746,8 +750,11 @@ valid_p:
746 * enabled and not gone - otherwise, the breakpoint would 750 * enabled and not gone - otherwise, the breakpoint would
747 * already have been removed. We save on flushing icache. 751 * already have been removed. We save on flushing icache.
748 */ 752 */
749 if (kprobe_enabled && !kprobe_gone(old_p)) 753 if (kprobe_enabled && !kprobe_gone(old_p)) {
754 mutex_lock(&text_mutex);
750 arch_disarm_kprobe(p); 755 arch_disarm_kprobe(p);
756 mutex_unlock(&text_mutex);
757 }
751 hlist_del_rcu(&old_p->hlist); 758 hlist_del_rcu(&old_p->hlist);
752 } else { 759 } else {
753 if (p->break_handler && !kprobe_gone(p)) 760 if (p->break_handler && !kprobe_gone(p))
@@ -1280,12 +1287,14 @@ static void __kprobes enable_all_kprobes(void)
1280 if (kprobe_enabled) 1287 if (kprobe_enabled)
1281 goto already_enabled; 1288 goto already_enabled;
1282 1289
1290 mutex_lock(&text_mutex);
1283 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 1291 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
1284 head = &kprobe_table[i]; 1292 head = &kprobe_table[i];
1285 hlist_for_each_entry_rcu(p, node, head, hlist) 1293 hlist_for_each_entry_rcu(p, node, head, hlist)
1286 if (!kprobe_gone(p)) 1294 if (!kprobe_gone(p))
1287 arch_arm_kprobe(p); 1295 arch_arm_kprobe(p);
1288 } 1296 }
1297 mutex_unlock(&text_mutex);
1289 1298
1290 kprobe_enabled = true; 1299 kprobe_enabled = true;
1291 printk(KERN_INFO "Kprobes globally enabled\n"); 1300 printk(KERN_INFO "Kprobes globally enabled\n");
@@ -1310,6 +1319,7 @@ static void __kprobes disable_all_kprobes(void)
1310 1319
1311 kprobe_enabled = false; 1320 kprobe_enabled = false;
1312 printk(KERN_INFO "Kprobes globally disabled\n"); 1321 printk(KERN_INFO "Kprobes globally disabled\n");
1322 mutex_lock(&text_mutex);
1313 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 1323 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
1314 head = &kprobe_table[i]; 1324 head = &kprobe_table[i];
1315 hlist_for_each_entry_rcu(p, node, head, hlist) { 1325 hlist_for_each_entry_rcu(p, node, head, hlist) {
@@ -1318,6 +1328,7 @@ static void __kprobes disable_all_kprobes(void)
1318 } 1328 }
1319 } 1329 }
1320 1330
1331 mutex_unlock(&text_mutex);
1321 mutex_unlock(&kprobe_mutex); 1332 mutex_unlock(&kprobe_mutex);
1322 /* Allow all currently running kprobes to complete */ 1333 /* Allow all currently running kprobes to complete */
1323 synchronize_sched(); 1334 synchronize_sched();
diff --git a/kernel/lockdep.c b/kernel/lockdep.c
index 06b0c3568f0b..cb70c1db85d0 100644
--- a/kernel/lockdep.c
+++ b/kernel/lockdep.c
@@ -41,6 +41,8 @@
41#include <linux/utsname.h> 41#include <linux/utsname.h>
42#include <linux/hash.h> 42#include <linux/hash.h>
43#include <linux/ftrace.h> 43#include <linux/ftrace.h>
44#include <linux/stringify.h>
45#include <trace/lockdep.h>
44 46
45#include <asm/sections.h> 47#include <asm/sections.h>
46 48
@@ -310,12 +312,14 @@ EXPORT_SYMBOL(lockdep_on);
310#if VERBOSE 312#if VERBOSE
311# define HARDIRQ_VERBOSE 1 313# define HARDIRQ_VERBOSE 1
312# define SOFTIRQ_VERBOSE 1 314# define SOFTIRQ_VERBOSE 1
315# define RECLAIM_VERBOSE 1
313#else 316#else
314# define HARDIRQ_VERBOSE 0 317# define HARDIRQ_VERBOSE 0
315# define SOFTIRQ_VERBOSE 0 318# define SOFTIRQ_VERBOSE 0
319# define RECLAIM_VERBOSE 0
316#endif 320#endif
317 321
318#if VERBOSE || HARDIRQ_VERBOSE || SOFTIRQ_VERBOSE 322#if VERBOSE || HARDIRQ_VERBOSE || SOFTIRQ_VERBOSE || RECLAIM_VERBOSE
319/* 323/*
320 * Quick filtering for interesting events: 324 * Quick filtering for interesting events:
321 */ 325 */
@@ -443,17 +447,18 @@ atomic_t nr_find_usage_backwards_recursions;
443 * Locking printouts: 447 * Locking printouts:
444 */ 448 */
445 449
450#define __USAGE(__STATE) \
451 [LOCK_USED_IN_##__STATE] = "IN-"__stringify(__STATE)"-W", \
452 [LOCK_ENABLED_##__STATE] = __stringify(__STATE)"-ON-W", \
453 [LOCK_USED_IN_##__STATE##_READ] = "IN-"__stringify(__STATE)"-R",\
454 [LOCK_ENABLED_##__STATE##_READ] = __stringify(__STATE)"-ON-R",
455
446static const char *usage_str[] = 456static const char *usage_str[] =
447{ 457{
448 [LOCK_USED] = "initial-use ", 458#define LOCKDEP_STATE(__STATE) __USAGE(__STATE)
449 [LOCK_USED_IN_HARDIRQ] = "in-hardirq-W", 459#include "lockdep_states.h"
450 [LOCK_USED_IN_SOFTIRQ] = "in-softirq-W", 460#undef LOCKDEP_STATE
451 [LOCK_ENABLED_SOFTIRQS] = "softirq-on-W", 461 [LOCK_USED] = "INITIAL USE",
452 [LOCK_ENABLED_HARDIRQS] = "hardirq-on-W",
453 [LOCK_USED_IN_HARDIRQ_READ] = "in-hardirq-R",
454 [LOCK_USED_IN_SOFTIRQ_READ] = "in-softirq-R",
455 [LOCK_ENABLED_SOFTIRQS_READ] = "softirq-on-R",
456 [LOCK_ENABLED_HARDIRQS_READ] = "hardirq-on-R",
457}; 462};
458 463
459const char * __get_key_name(struct lockdep_subclass_key *key, char *str) 464const char * __get_key_name(struct lockdep_subclass_key *key, char *str)
@@ -461,46 +466,45 @@ const char * __get_key_name(struct lockdep_subclass_key *key, char *str)
461 return kallsyms_lookup((unsigned long)key, NULL, NULL, NULL, str); 466 return kallsyms_lookup((unsigned long)key, NULL, NULL, NULL, str);
462} 467}
463 468
464void 469static inline unsigned long lock_flag(enum lock_usage_bit bit)
465get_usage_chars(struct lock_class *class, char *c1, char *c2, char *c3, char *c4)
466{ 470{
467 *c1 = '.', *c2 = '.', *c3 = '.', *c4 = '.'; 471 return 1UL << bit;
468 472}
469 if (class->usage_mask & LOCKF_USED_IN_HARDIRQ)
470 *c1 = '+';
471 else
472 if (class->usage_mask & LOCKF_ENABLED_HARDIRQS)
473 *c1 = '-';
474 473
475 if (class->usage_mask & LOCKF_USED_IN_SOFTIRQ) 474static char get_usage_char(struct lock_class *class, enum lock_usage_bit bit)
476 *c2 = '+'; 475{
477 else 476 char c = '.';
478 if (class->usage_mask & LOCKF_ENABLED_SOFTIRQS)
479 *c2 = '-';
480 477
481 if (class->usage_mask & LOCKF_ENABLED_HARDIRQS_READ) 478 if (class->usage_mask & lock_flag(bit + 2))
482 *c3 = '-'; 479 c = '+';
483 if (class->usage_mask & LOCKF_USED_IN_HARDIRQ_READ) { 480 if (class->usage_mask & lock_flag(bit)) {
484 *c3 = '+'; 481 c = '-';
485 if (class->usage_mask & LOCKF_ENABLED_HARDIRQS_READ) 482 if (class->usage_mask & lock_flag(bit + 2))
486 *c3 = '?'; 483 c = '?';
487 } 484 }
488 485
489 if (class->usage_mask & LOCKF_ENABLED_SOFTIRQS_READ) 486 return c;
490 *c4 = '-'; 487}
491 if (class->usage_mask & LOCKF_USED_IN_SOFTIRQ_READ) { 488
492 *c4 = '+'; 489void get_usage_chars(struct lock_class *class, char usage[LOCK_USAGE_CHARS])
493 if (class->usage_mask & LOCKF_ENABLED_SOFTIRQS_READ) 490{
494 *c4 = '?'; 491 int i = 0;
495 } 492
493#define LOCKDEP_STATE(__STATE) \
494 usage[i++] = get_usage_char(class, LOCK_USED_IN_##__STATE); \
495 usage[i++] = get_usage_char(class, LOCK_USED_IN_##__STATE##_READ);
496#include "lockdep_states.h"
497#undef LOCKDEP_STATE
498
499 usage[i] = '\0';
496} 500}
497 501
498static void print_lock_name(struct lock_class *class) 502static void print_lock_name(struct lock_class *class)
499{ 503{
500 char str[KSYM_NAME_LEN], c1, c2, c3, c4; 504 char str[KSYM_NAME_LEN], usage[LOCK_USAGE_CHARS];
501 const char *name; 505 const char *name;
502 506
503 get_usage_chars(class, &c1, &c2, &c3, &c4); 507 get_usage_chars(class, usage);
504 508
505 name = class->name; 509 name = class->name;
506 if (!name) { 510 if (!name) {
@@ -513,7 +517,7 @@ static void print_lock_name(struct lock_class *class)
513 if (class->subclass) 517 if (class->subclass)
514 printk("/%d", class->subclass); 518 printk("/%d", class->subclass);
515 } 519 }
516 printk("){%c%c%c%c}", c1, c2, c3, c4); 520 printk("){%s}", usage);
517} 521}
518 522
519static void print_lockdep_cache(struct lockdep_map *lock) 523static void print_lockdep_cache(struct lockdep_map *lock)
@@ -1263,9 +1267,49 @@ check_usage(struct task_struct *curr, struct held_lock *prev,
1263 bit_backwards, bit_forwards, irqclass); 1267 bit_backwards, bit_forwards, irqclass);
1264} 1268}
1265 1269
1266static int 1270static const char *state_names[] = {
1267check_prev_add_irq(struct task_struct *curr, struct held_lock *prev, 1271#define LOCKDEP_STATE(__STATE) \
1268 struct held_lock *next) 1272 __stringify(__STATE),
1273#include "lockdep_states.h"
1274#undef LOCKDEP_STATE
1275};
1276
1277static const char *state_rnames[] = {
1278#define LOCKDEP_STATE(__STATE) \
1279 __stringify(__STATE)"-READ",
1280#include "lockdep_states.h"
1281#undef LOCKDEP_STATE
1282};
1283
1284static inline const char *state_name(enum lock_usage_bit bit)
1285{
1286 return (bit & 1) ? state_rnames[bit >> 2] : state_names[bit >> 2];
1287}
1288
1289static int exclusive_bit(int new_bit)
1290{
1291 /*
1292 * USED_IN
1293 * USED_IN_READ
1294 * ENABLED
1295 * ENABLED_READ
1296 *
1297 * bit 0 - write/read
1298 * bit 1 - used_in/enabled
1299 * bit 2+ state
1300 */
1301
1302 int state = new_bit & ~3;
1303 int dir = new_bit & 2;
1304
1305 /*
1306 * keep state, bit flip the direction and strip read.
1307 */
1308 return state | (dir ^ 2);
1309}
1310
1311static int check_irq_usage(struct task_struct *curr, struct held_lock *prev,
1312 struct held_lock *next, enum lock_usage_bit bit)
1269{ 1313{
1270 /* 1314 /*
1271 * Prove that the new dependency does not connect a hardirq-safe 1315 * Prove that the new dependency does not connect a hardirq-safe
@@ -1273,38 +1317,34 @@ check_prev_add_irq(struct task_struct *curr, struct held_lock *prev,
1273 * the backwards-subgraph starting at <prev>, and the 1317 * the backwards-subgraph starting at <prev>, and the
1274 * forwards-subgraph starting at <next>: 1318 * forwards-subgraph starting at <next>:
1275 */ 1319 */
1276 if (!check_usage(curr, prev, next, LOCK_USED_IN_HARDIRQ, 1320 if (!check_usage(curr, prev, next, bit,
1277 LOCK_ENABLED_HARDIRQS, "hard")) 1321 exclusive_bit(bit), state_name(bit)))
1278 return 0; 1322 return 0;
1279 1323
1324 bit++; /* _READ */
1325
1280 /* 1326 /*
1281 * Prove that the new dependency does not connect a hardirq-safe-read 1327 * Prove that the new dependency does not connect a hardirq-safe-read
1282 * lock with a hardirq-unsafe lock - to achieve this we search 1328 * lock with a hardirq-unsafe lock - to achieve this we search
1283 * the backwards-subgraph starting at <prev>, and the 1329 * the backwards-subgraph starting at <prev>, and the
1284 * forwards-subgraph starting at <next>: 1330 * forwards-subgraph starting at <next>:
1285 */ 1331 */
1286 if (!check_usage(curr, prev, next, LOCK_USED_IN_HARDIRQ_READ, 1332 if (!check_usage(curr, prev, next, bit,
1287 LOCK_ENABLED_HARDIRQS, "hard-read")) 1333 exclusive_bit(bit), state_name(bit)))
1288 return 0; 1334 return 0;
1289 1335
1290 /* 1336 return 1;
1291 * Prove that the new dependency does not connect a softirq-safe 1337}
1292 * lock with a softirq-unsafe lock - to achieve this we search 1338
1293 * the backwards-subgraph starting at <prev>, and the 1339static int
1294 * forwards-subgraph starting at <next>: 1340check_prev_add_irq(struct task_struct *curr, struct held_lock *prev,
1295 */ 1341 struct held_lock *next)
1296 if (!check_usage(curr, prev, next, LOCK_USED_IN_SOFTIRQ, 1342{
1297 LOCK_ENABLED_SOFTIRQS, "soft")) 1343#define LOCKDEP_STATE(__STATE) \
1298 return 0; 1344 if (!check_irq_usage(curr, prev, next, LOCK_USED_IN_##__STATE)) \
1299 /*
1300 * Prove that the new dependency does not connect a softirq-safe-read
1301 * lock with a softirq-unsafe lock - to achieve this we search
1302 * the backwards-subgraph starting at <prev>, and the
1303 * forwards-subgraph starting at <next>:
1304 */
1305 if (!check_usage(curr, prev, next, LOCK_USED_IN_SOFTIRQ_READ,
1306 LOCK_ENABLED_SOFTIRQS, "soft"))
1307 return 0; 1345 return 0;
1346#include "lockdep_states.h"
1347#undef LOCKDEP_STATE
1308 1348
1309 return 1; 1349 return 1;
1310} 1350}
@@ -1861,9 +1901,9 @@ print_irq_inversion_bug(struct task_struct *curr, struct lock_class *other,
1861 curr->comm, task_pid_nr(curr)); 1901 curr->comm, task_pid_nr(curr));
1862 print_lock(this); 1902 print_lock(this);
1863 if (forwards) 1903 if (forwards)
1864 printk("but this lock took another, %s-irq-unsafe lock in the past:\n", irqclass); 1904 printk("but this lock took another, %s-unsafe lock in the past:\n", irqclass);
1865 else 1905 else
1866 printk("but this lock was taken by another, %s-irq-safe lock in the past:\n", irqclass); 1906 printk("but this lock was taken by another, %s-safe lock in the past:\n", irqclass);
1867 print_lock_name(other); 1907 print_lock_name(other);
1868 printk("\n\nand interrupts could create inverse lock ordering between them.\n\n"); 1908 printk("\n\nand interrupts could create inverse lock ordering between them.\n\n");
1869 1909
@@ -1933,7 +1973,7 @@ void print_irqtrace_events(struct task_struct *curr)
1933 print_ip_sym(curr->softirq_disable_ip); 1973 print_ip_sym(curr->softirq_disable_ip);
1934} 1974}
1935 1975
1936static int hardirq_verbose(struct lock_class *class) 1976static int HARDIRQ_verbose(struct lock_class *class)
1937{ 1977{
1938#if HARDIRQ_VERBOSE 1978#if HARDIRQ_VERBOSE
1939 return class_filter(class); 1979 return class_filter(class);
@@ -1941,7 +1981,7 @@ static int hardirq_verbose(struct lock_class *class)
1941 return 0; 1981 return 0;
1942} 1982}
1943 1983
1944static int softirq_verbose(struct lock_class *class) 1984static int SOFTIRQ_verbose(struct lock_class *class)
1945{ 1985{
1946#if SOFTIRQ_VERBOSE 1986#if SOFTIRQ_VERBOSE
1947 return class_filter(class); 1987 return class_filter(class);
@@ -1949,185 +1989,95 @@ static int softirq_verbose(struct lock_class *class)
1949 return 0; 1989 return 0;
1950} 1990}
1951 1991
1992static int RECLAIM_FS_verbose(struct lock_class *class)
1993{
1994#if RECLAIM_VERBOSE
1995 return class_filter(class);
1996#endif
1997 return 0;
1998}
1999
1952#define STRICT_READ_CHECKS 1 2000#define STRICT_READ_CHECKS 1
1953 2001
1954static int mark_lock_irq(struct task_struct *curr, struct held_lock *this, 2002static int (*state_verbose_f[])(struct lock_class *class) = {
2003#define LOCKDEP_STATE(__STATE) \
2004 __STATE##_verbose,
2005#include "lockdep_states.h"
2006#undef LOCKDEP_STATE
2007};
2008
2009static inline int state_verbose(enum lock_usage_bit bit,
2010 struct lock_class *class)
2011{
2012 return state_verbose_f[bit >> 2](class);
2013}
2014
2015typedef int (*check_usage_f)(struct task_struct *, struct held_lock *,
2016 enum lock_usage_bit bit, const char *name);
2017
2018static int
2019mark_lock_irq(struct task_struct *curr, struct held_lock *this,
1955 enum lock_usage_bit new_bit) 2020 enum lock_usage_bit new_bit)
1956{ 2021{
1957 int ret = 1; 2022 int excl_bit = exclusive_bit(new_bit);
2023 int read = new_bit & 1;
2024 int dir = new_bit & 2;
1958 2025
1959 switch(new_bit) { 2026 /*
1960 case LOCK_USED_IN_HARDIRQ: 2027 * mark USED_IN has to look forwards -- to ensure no dependency
1961 if (!valid_state(curr, this, new_bit, LOCK_ENABLED_HARDIRQS)) 2028 * has ENABLED state, which would allow recursion deadlocks.
1962 return 0; 2029 *
1963 if (!valid_state(curr, this, new_bit, 2030 * mark ENABLED has to look backwards -- to ensure no dependee
1964 LOCK_ENABLED_HARDIRQS_READ)) 2031 * has USED_IN state, which, again, would allow recursion deadlocks.
1965 return 0; 2032 */
1966 /* 2033 check_usage_f usage = dir ?
1967 * just marked it hardirq-safe, check that this lock 2034 check_usage_backwards : check_usage_forwards;
1968 * took no hardirq-unsafe lock in the past: 2035
1969 */ 2036 /*
1970 if (!check_usage_forwards(curr, this, 2037 * Validate that this particular lock does not have conflicting
1971 LOCK_ENABLED_HARDIRQS, "hard")) 2038 * usage states.
1972 return 0; 2039 */
1973#if STRICT_READ_CHECKS 2040 if (!valid_state(curr, this, new_bit, excl_bit))
1974 /* 2041 return 0;
1975 * just marked it hardirq-safe, check that this lock 2042
1976 * took no hardirq-unsafe-read lock in the past: 2043 /*
1977 */ 2044 * Validate that the lock dependencies don't have conflicting usage
1978 if (!check_usage_forwards(curr, this, 2045 * states.
1979 LOCK_ENABLED_HARDIRQS_READ, "hard-read")) 2046 */
1980 return 0; 2047 if ((!read || !dir || STRICT_READ_CHECKS) &&
1981#endif 2048 !usage(curr, this, excl_bit, state_name(new_bit & ~1)))
1982 if (hardirq_verbose(hlock_class(this))) 2049 return 0;
1983 ret = 2; 2050
1984 break; 2051 /*
1985 case LOCK_USED_IN_SOFTIRQ: 2052 * Check for read in write conflicts
1986 if (!valid_state(curr, this, new_bit, LOCK_ENABLED_SOFTIRQS)) 2053 */
1987 return 0; 2054 if (!read) {
1988 if (!valid_state(curr, this, new_bit, 2055 if (!valid_state(curr, this, new_bit, excl_bit + 1))
1989 LOCK_ENABLED_SOFTIRQS_READ))
1990 return 0;
1991 /*
1992 * just marked it softirq-safe, check that this lock
1993 * took no softirq-unsafe lock in the past:
1994 */
1995 if (!check_usage_forwards(curr, this,
1996 LOCK_ENABLED_SOFTIRQS, "soft"))
1997 return 0;
1998#if STRICT_READ_CHECKS
1999 /*
2000 * just marked it softirq-safe, check that this lock
2001 * took no softirq-unsafe-read lock in the past:
2002 */
2003 if (!check_usage_forwards(curr, this,
2004 LOCK_ENABLED_SOFTIRQS_READ, "soft-read"))
2005 return 0;
2006#endif
2007 if (softirq_verbose(hlock_class(this)))
2008 ret = 2;
2009 break;
2010 case LOCK_USED_IN_HARDIRQ_READ:
2011 if (!valid_state(curr, this, new_bit, LOCK_ENABLED_HARDIRQS))
2012 return 0;
2013 /*
2014 * just marked it hardirq-read-safe, check that this lock
2015 * took no hardirq-unsafe lock in the past:
2016 */
2017 if (!check_usage_forwards(curr, this,
2018 LOCK_ENABLED_HARDIRQS, "hard"))
2019 return 0;
2020 if (hardirq_verbose(hlock_class(this)))
2021 ret = 2;
2022 break;
2023 case LOCK_USED_IN_SOFTIRQ_READ:
2024 if (!valid_state(curr, this, new_bit, LOCK_ENABLED_SOFTIRQS))
2025 return 0;
2026 /*
2027 * just marked it softirq-read-safe, check that this lock
2028 * took no softirq-unsafe lock in the past:
2029 */
2030 if (!check_usage_forwards(curr, this,
2031 LOCK_ENABLED_SOFTIRQS, "soft"))
2032 return 0;
2033 if (softirq_verbose(hlock_class(this)))
2034 ret = 2;
2035 break;
2036 case LOCK_ENABLED_HARDIRQS:
2037 if (!valid_state(curr, this, new_bit, LOCK_USED_IN_HARDIRQ))
2038 return 0;
2039 if (!valid_state(curr, this, new_bit,
2040 LOCK_USED_IN_HARDIRQ_READ))
2041 return 0;
2042 /*
2043 * just marked it hardirq-unsafe, check that no hardirq-safe
2044 * lock in the system ever took it in the past:
2045 */
2046 if (!check_usage_backwards(curr, this,
2047 LOCK_USED_IN_HARDIRQ, "hard"))
2048 return 0;
2049#if STRICT_READ_CHECKS
2050 /*
2051 * just marked it hardirq-unsafe, check that no
2052 * hardirq-safe-read lock in the system ever took
2053 * it in the past:
2054 */
2055 if (!check_usage_backwards(curr, this,
2056 LOCK_USED_IN_HARDIRQ_READ, "hard-read"))
2057 return 0;
2058#endif
2059 if (hardirq_verbose(hlock_class(this)))
2060 ret = 2;
2061 break;
2062 case LOCK_ENABLED_SOFTIRQS:
2063 if (!valid_state(curr, this, new_bit, LOCK_USED_IN_SOFTIRQ))
2064 return 0;
2065 if (!valid_state(curr, this, new_bit,
2066 LOCK_USED_IN_SOFTIRQ_READ))
2067 return 0;
2068 /*
2069 * just marked it softirq-unsafe, check that no softirq-safe
2070 * lock in the system ever took it in the past:
2071 */
2072 if (!check_usage_backwards(curr, this,
2073 LOCK_USED_IN_SOFTIRQ, "soft"))
2074 return 0;
2075#if STRICT_READ_CHECKS
2076 /*
2077 * just marked it softirq-unsafe, check that no
2078 * softirq-safe-read lock in the system ever took
2079 * it in the past:
2080 */
2081 if (!check_usage_backwards(curr, this,
2082 LOCK_USED_IN_SOFTIRQ_READ, "soft-read"))
2083 return 0;
2084#endif
2085 if (softirq_verbose(hlock_class(this)))
2086 ret = 2;
2087 break;
2088 case LOCK_ENABLED_HARDIRQS_READ:
2089 if (!valid_state(curr, this, new_bit, LOCK_USED_IN_HARDIRQ))
2090 return 0;
2091#if STRICT_READ_CHECKS
2092 /*
2093 * just marked it hardirq-read-unsafe, check that no
2094 * hardirq-safe lock in the system ever took it in the past:
2095 */
2096 if (!check_usage_backwards(curr, this,
2097 LOCK_USED_IN_HARDIRQ, "hard"))
2098 return 0;
2099#endif
2100 if (hardirq_verbose(hlock_class(this)))
2101 ret = 2;
2102 break;
2103 case LOCK_ENABLED_SOFTIRQS_READ:
2104 if (!valid_state(curr, this, new_bit, LOCK_USED_IN_SOFTIRQ))
2105 return 0; 2056 return 0;
2106#if STRICT_READ_CHECKS 2057
2107 /* 2058 if (STRICT_READ_CHECKS &&
2108 * just marked it softirq-read-unsafe, check that no 2059 !usage(curr, this, excl_bit + 1,
2109 * softirq-safe lock in the system ever took it in the past: 2060 state_name(new_bit + 1)))
2110 */
2111 if (!check_usage_backwards(curr, this,
2112 LOCK_USED_IN_SOFTIRQ, "soft"))
2113 return 0; 2061 return 0;
2114#endif
2115 if (softirq_verbose(hlock_class(this)))
2116 ret = 2;
2117 break;
2118 default:
2119 WARN_ON(1);
2120 break;
2121 } 2062 }
2122 2063
2123 return ret; 2064 if (state_verbose(new_bit, hlock_class(this)))
2065 return 2;
2066
2067 return 1;
2124} 2068}
2125 2069
2070enum mark_type {
2071#define LOCKDEP_STATE(__STATE) __STATE,
2072#include "lockdep_states.h"
2073#undef LOCKDEP_STATE
2074};
2075
2126/* 2076/*
2127 * Mark all held locks with a usage bit: 2077 * Mark all held locks with a usage bit:
2128 */ 2078 */
2129static int 2079static int
2130mark_held_locks(struct task_struct *curr, int hardirq) 2080mark_held_locks(struct task_struct *curr, enum mark_type mark)
2131{ 2081{
2132 enum lock_usage_bit usage_bit; 2082 enum lock_usage_bit usage_bit;
2133 struct held_lock *hlock; 2083 struct held_lock *hlock;
@@ -2136,17 +2086,12 @@ mark_held_locks(struct task_struct *curr, int hardirq)
2136 for (i = 0; i < curr->lockdep_depth; i++) { 2086 for (i = 0; i < curr->lockdep_depth; i++) {
2137 hlock = curr->held_locks + i; 2087 hlock = curr->held_locks + i;
2138 2088
2139 if (hardirq) { 2089 usage_bit = 2 + (mark << 2); /* ENABLED */
2140 if (hlock->read) 2090 if (hlock->read)
2141 usage_bit = LOCK_ENABLED_HARDIRQS_READ; 2091 usage_bit += 1; /* READ */
2142 else 2092
2143 usage_bit = LOCK_ENABLED_HARDIRQS; 2093 BUG_ON(usage_bit >= LOCK_USAGE_STATES);
2144 } else { 2094
2145 if (hlock->read)
2146 usage_bit = LOCK_ENABLED_SOFTIRQS_READ;
2147 else
2148 usage_bit = LOCK_ENABLED_SOFTIRQS;
2149 }
2150 if (!mark_lock(curr, hlock, usage_bit)) 2095 if (!mark_lock(curr, hlock, usage_bit))
2151 return 0; 2096 return 0;
2152 } 2097 }
@@ -2200,7 +2145,7 @@ void trace_hardirqs_on_caller(unsigned long ip)
2200 * We are going to turn hardirqs on, so set the 2145 * We are going to turn hardirqs on, so set the
2201 * usage bit for all held locks: 2146 * usage bit for all held locks:
2202 */ 2147 */
2203 if (!mark_held_locks(curr, 1)) 2148 if (!mark_held_locks(curr, HARDIRQ))
2204 return; 2149 return;
2205 /* 2150 /*
2206 * If we have softirqs enabled, then set the usage 2151 * If we have softirqs enabled, then set the usage
@@ -2208,7 +2153,7 @@ void trace_hardirqs_on_caller(unsigned long ip)
2208 * this bit from being set before) 2153 * this bit from being set before)
2209 */ 2154 */
2210 if (curr->softirqs_enabled) 2155 if (curr->softirqs_enabled)
2211 if (!mark_held_locks(curr, 0)) 2156 if (!mark_held_locks(curr, SOFTIRQ))
2212 return; 2157 return;
2213 2158
2214 curr->hardirq_enable_ip = ip; 2159 curr->hardirq_enable_ip = ip;
@@ -2288,7 +2233,7 @@ void trace_softirqs_on(unsigned long ip)
2288 * enabled too: 2233 * enabled too:
2289 */ 2234 */
2290 if (curr->hardirqs_enabled) 2235 if (curr->hardirqs_enabled)
2291 mark_held_locks(curr, 0); 2236 mark_held_locks(curr, SOFTIRQ);
2292} 2237}
2293 2238
2294/* 2239/*
@@ -2317,6 +2262,31 @@ void trace_softirqs_off(unsigned long ip)
2317 debug_atomic_inc(&redundant_softirqs_off); 2262 debug_atomic_inc(&redundant_softirqs_off);
2318} 2263}
2319 2264
2265void lockdep_trace_alloc(gfp_t gfp_mask)
2266{
2267 struct task_struct *curr = current;
2268
2269 if (unlikely(!debug_locks))
2270 return;
2271
2272 /* no reclaim without waiting on it */
2273 if (!(gfp_mask & __GFP_WAIT))
2274 return;
2275
2276 /* this guy won't enter reclaim */
2277 if ((curr->flags & PF_MEMALLOC) && !(gfp_mask & __GFP_NOMEMALLOC))
2278 return;
2279
2280 /* We're only interested __GFP_FS allocations for now */
2281 if (!(gfp_mask & __GFP_FS))
2282 return;
2283
2284 if (DEBUG_LOCKS_WARN_ON(irqs_disabled()))
2285 return;
2286
2287 mark_held_locks(curr, RECLAIM_FS);
2288}
2289
2320static int mark_irqflags(struct task_struct *curr, struct held_lock *hlock) 2290static int mark_irqflags(struct task_struct *curr, struct held_lock *hlock)
2321{ 2291{
2322 /* 2292 /*
@@ -2345,19 +2315,35 @@ static int mark_irqflags(struct task_struct *curr, struct held_lock *hlock)
2345 if (!hlock->hardirqs_off) { 2315 if (!hlock->hardirqs_off) {
2346 if (hlock->read) { 2316 if (hlock->read) {
2347 if (!mark_lock(curr, hlock, 2317 if (!mark_lock(curr, hlock,
2348 LOCK_ENABLED_HARDIRQS_READ)) 2318 LOCK_ENABLED_HARDIRQ_READ))
2349 return 0; 2319 return 0;
2350 if (curr->softirqs_enabled) 2320 if (curr->softirqs_enabled)
2351 if (!mark_lock(curr, hlock, 2321 if (!mark_lock(curr, hlock,
2352 LOCK_ENABLED_SOFTIRQS_READ)) 2322 LOCK_ENABLED_SOFTIRQ_READ))
2353 return 0; 2323 return 0;
2354 } else { 2324 } else {
2355 if (!mark_lock(curr, hlock, 2325 if (!mark_lock(curr, hlock,
2356 LOCK_ENABLED_HARDIRQS)) 2326 LOCK_ENABLED_HARDIRQ))
2357 return 0; 2327 return 0;
2358 if (curr->softirqs_enabled) 2328 if (curr->softirqs_enabled)
2359 if (!mark_lock(curr, hlock, 2329 if (!mark_lock(curr, hlock,
2360 LOCK_ENABLED_SOFTIRQS)) 2330 LOCK_ENABLED_SOFTIRQ))
2331 return 0;
2332 }
2333 }
2334
2335 /*
2336 * We reuse the irq context infrastructure more broadly as a general
2337 * context checking code. This tests GFP_FS recursion (a lock taken
2338 * during reclaim for a GFP_FS allocation is held over a GFP_FS
2339 * allocation).
2340 */
2341 if (!hlock->trylock && (curr->lockdep_reclaim_gfp & __GFP_FS)) {
2342 if (hlock->read) {
2343 if (!mark_lock(curr, hlock, LOCK_USED_IN_RECLAIM_FS_READ))
2344 return 0;
2345 } else {
2346 if (!mark_lock(curr, hlock, LOCK_USED_IN_RECLAIM_FS))
2361 return 0; 2347 return 0;
2362 } 2348 }
2363 } 2349 }
@@ -2412,6 +2398,10 @@ static inline int separate_irq_context(struct task_struct *curr,
2412 return 0; 2398 return 0;
2413} 2399}
2414 2400
2401void lockdep_trace_alloc(gfp_t gfp_mask)
2402{
2403}
2404
2415#endif 2405#endif
2416 2406
2417/* 2407/*
@@ -2445,14 +2435,13 @@ static int mark_lock(struct task_struct *curr, struct held_lock *this,
2445 return 0; 2435 return 0;
2446 2436
2447 switch (new_bit) { 2437 switch (new_bit) {
2448 case LOCK_USED_IN_HARDIRQ: 2438#define LOCKDEP_STATE(__STATE) \
2449 case LOCK_USED_IN_SOFTIRQ: 2439 case LOCK_USED_IN_##__STATE: \
2450 case LOCK_USED_IN_HARDIRQ_READ: 2440 case LOCK_USED_IN_##__STATE##_READ: \
2451 case LOCK_USED_IN_SOFTIRQ_READ: 2441 case LOCK_ENABLED_##__STATE: \
2452 case LOCK_ENABLED_HARDIRQS: 2442 case LOCK_ENABLED_##__STATE##_READ:
2453 case LOCK_ENABLED_SOFTIRQS: 2443#include "lockdep_states.h"
2454 case LOCK_ENABLED_HARDIRQS_READ: 2444#undef LOCKDEP_STATE
2455 case LOCK_ENABLED_SOFTIRQS_READ:
2456 ret = mark_lock_irq(curr, this, new_bit); 2445 ret = mark_lock_irq(curr, this, new_bit);
2457 if (!ret) 2446 if (!ret)
2458 return 0; 2447 return 0;
@@ -2925,6 +2914,8 @@ void lock_set_class(struct lockdep_map *lock, const char *name,
2925} 2914}
2926EXPORT_SYMBOL_GPL(lock_set_class); 2915EXPORT_SYMBOL_GPL(lock_set_class);
2927 2916
2917DEFINE_TRACE(lock_acquire);
2918
2928/* 2919/*
2929 * We are not always called with irqs disabled - do that here, 2920 * We are not always called with irqs disabled - do that here,
2930 * and also avoid lockdep recursion: 2921 * and also avoid lockdep recursion:
@@ -2935,6 +2926,8 @@ void lock_acquire(struct lockdep_map *lock, unsigned int subclass,
2935{ 2926{
2936 unsigned long flags; 2927 unsigned long flags;
2937 2928
2929 trace_lock_acquire(lock, subclass, trylock, read, check, nest_lock, ip);
2930
2938 if (unlikely(current->lockdep_recursion)) 2931 if (unlikely(current->lockdep_recursion))
2939 return; 2932 return;
2940 2933
@@ -2949,11 +2942,15 @@ void lock_acquire(struct lockdep_map *lock, unsigned int subclass,
2949} 2942}
2950EXPORT_SYMBOL_GPL(lock_acquire); 2943EXPORT_SYMBOL_GPL(lock_acquire);
2951 2944
2945DEFINE_TRACE(lock_release);
2946
2952void lock_release(struct lockdep_map *lock, int nested, 2947void lock_release(struct lockdep_map *lock, int nested,
2953 unsigned long ip) 2948 unsigned long ip)
2954{ 2949{
2955 unsigned long flags; 2950 unsigned long flags;
2956 2951
2952 trace_lock_release(lock, nested, ip);
2953
2957 if (unlikely(current->lockdep_recursion)) 2954 if (unlikely(current->lockdep_recursion))
2958 return; 2955 return;
2959 2956
@@ -2966,6 +2963,16 @@ void lock_release(struct lockdep_map *lock, int nested,
2966} 2963}
2967EXPORT_SYMBOL_GPL(lock_release); 2964EXPORT_SYMBOL_GPL(lock_release);
2968 2965
2966void lockdep_set_current_reclaim_state(gfp_t gfp_mask)
2967{
2968 current->lockdep_reclaim_gfp = gfp_mask;
2969}
2970
2971void lockdep_clear_current_reclaim_state(void)
2972{
2973 current->lockdep_reclaim_gfp = 0;
2974}
2975
2969#ifdef CONFIG_LOCK_STAT 2976#ifdef CONFIG_LOCK_STAT
2970static int 2977static int
2971print_lock_contention_bug(struct task_struct *curr, struct lockdep_map *lock, 2978print_lock_contention_bug(struct task_struct *curr, struct lockdep_map *lock,
@@ -3092,10 +3099,14 @@ found_it:
3092 lock->ip = ip; 3099 lock->ip = ip;
3093} 3100}
3094 3101
3102DEFINE_TRACE(lock_contended);
3103
3095void lock_contended(struct lockdep_map *lock, unsigned long ip) 3104void lock_contended(struct lockdep_map *lock, unsigned long ip)
3096{ 3105{
3097 unsigned long flags; 3106 unsigned long flags;
3098 3107
3108 trace_lock_contended(lock, ip);
3109
3099 if (unlikely(!lock_stat)) 3110 if (unlikely(!lock_stat))
3100 return; 3111 return;
3101 3112
@@ -3111,10 +3122,14 @@ void lock_contended(struct lockdep_map *lock, unsigned long ip)
3111} 3122}
3112EXPORT_SYMBOL_GPL(lock_contended); 3123EXPORT_SYMBOL_GPL(lock_contended);
3113 3124
3125DEFINE_TRACE(lock_acquired);
3126
3114void lock_acquired(struct lockdep_map *lock, unsigned long ip) 3127void lock_acquired(struct lockdep_map *lock, unsigned long ip)
3115{ 3128{
3116 unsigned long flags; 3129 unsigned long flags;
3117 3130
3131 trace_lock_acquired(lock, ip);
3132
3118 if (unlikely(!lock_stat)) 3133 if (unlikely(!lock_stat))
3119 return; 3134 return;
3120 3135
diff --git a/kernel/lockdep_internals.h b/kernel/lockdep_internals.h
index 56b196932c08..a2cc7e9a6e84 100644
--- a/kernel/lockdep_internals.h
+++ b/kernel/lockdep_internals.h
@@ -7,6 +7,45 @@
7 */ 7 */
8 8
9/* 9/*
10 * Lock-class usage-state bits:
11 */
12enum lock_usage_bit {
13#define LOCKDEP_STATE(__STATE) \
14 LOCK_USED_IN_##__STATE, \
15 LOCK_USED_IN_##__STATE##_READ, \
16 LOCK_ENABLED_##__STATE, \
17 LOCK_ENABLED_##__STATE##_READ,
18#include "lockdep_states.h"
19#undef LOCKDEP_STATE
20 LOCK_USED,
21 LOCK_USAGE_STATES
22};
23
24/*
25 * Usage-state bitmasks:
26 */
27#define __LOCKF(__STATE) LOCKF_##__STATE = (1 << LOCK_##__STATE),
28
29enum {
30#define LOCKDEP_STATE(__STATE) \
31 __LOCKF(USED_IN_##__STATE) \
32 __LOCKF(USED_IN_##__STATE##_READ) \
33 __LOCKF(ENABLED_##__STATE) \
34 __LOCKF(ENABLED_##__STATE##_READ)
35#include "lockdep_states.h"
36#undef LOCKDEP_STATE
37 __LOCKF(USED)
38};
39
40#define LOCKF_ENABLED_IRQ (LOCKF_ENABLED_HARDIRQ | LOCKF_ENABLED_SOFTIRQ)
41#define LOCKF_USED_IN_IRQ (LOCKF_USED_IN_HARDIRQ | LOCKF_USED_IN_SOFTIRQ)
42
43#define LOCKF_ENABLED_IRQ_READ \
44 (LOCKF_ENABLED_HARDIRQ_READ | LOCKF_ENABLED_SOFTIRQ_READ)
45#define LOCKF_USED_IN_IRQ_READ \
46 (LOCKF_USED_IN_HARDIRQ_READ | LOCKF_USED_IN_SOFTIRQ_READ)
47
48/*
10 * MAX_LOCKDEP_ENTRIES is the maximum number of lock dependencies 49 * MAX_LOCKDEP_ENTRIES is the maximum number of lock dependencies
11 * we track. 50 * we track.
12 * 51 *
@@ -31,8 +70,10 @@
31extern struct list_head all_lock_classes; 70extern struct list_head all_lock_classes;
32extern struct lock_chain lock_chains[]; 71extern struct lock_chain lock_chains[];
33 72
34extern void 73#define LOCK_USAGE_CHARS (1+LOCK_USAGE_STATES/2)
35get_usage_chars(struct lock_class *class, char *c1, char *c2, char *c3, char *c4); 74
75extern void get_usage_chars(struct lock_class *class,
76 char usage[LOCK_USAGE_CHARS]);
36 77
37extern const char * __get_key_name(struct lockdep_subclass_key *key, char *str); 78extern const char * __get_key_name(struct lockdep_subclass_key *key, char *str);
38 79
diff --git a/kernel/lockdep_proc.c b/kernel/lockdep_proc.c
index 13716b813896..d7135aa2d2c4 100644
--- a/kernel/lockdep_proc.c
+++ b/kernel/lockdep_proc.c
@@ -84,7 +84,7 @@ static int l_show(struct seq_file *m, void *v)
84{ 84{
85 struct lock_class *class = v; 85 struct lock_class *class = v;
86 struct lock_list *entry; 86 struct lock_list *entry;
87 char c1, c2, c3, c4; 87 char usage[LOCK_USAGE_CHARS];
88 88
89 if (v == SEQ_START_TOKEN) { 89 if (v == SEQ_START_TOKEN) {
90 seq_printf(m, "all lock classes:\n"); 90 seq_printf(m, "all lock classes:\n");
@@ -100,8 +100,8 @@ static int l_show(struct seq_file *m, void *v)
100 seq_printf(m, " BD:%5ld", lockdep_count_backward_deps(class)); 100 seq_printf(m, " BD:%5ld", lockdep_count_backward_deps(class));
101#endif 101#endif
102 102
103 get_usage_chars(class, &c1, &c2, &c3, &c4); 103 get_usage_chars(class, usage);
104 seq_printf(m, " %c%c%c%c", c1, c2, c3, c4); 104 seq_printf(m, " %s", usage);
105 105
106 seq_printf(m, ": "); 106 seq_printf(m, ": ");
107 print_name(m, class); 107 print_name(m, class);
@@ -300,27 +300,27 @@ static int lockdep_stats_show(struct seq_file *m, void *v)
300 nr_uncategorized++; 300 nr_uncategorized++;
301 if (class->usage_mask & LOCKF_USED_IN_IRQ) 301 if (class->usage_mask & LOCKF_USED_IN_IRQ)
302 nr_irq_safe++; 302 nr_irq_safe++;
303 if (class->usage_mask & LOCKF_ENABLED_IRQS) 303 if (class->usage_mask & LOCKF_ENABLED_IRQ)
304 nr_irq_unsafe++; 304 nr_irq_unsafe++;
305 if (class->usage_mask & LOCKF_USED_IN_SOFTIRQ) 305 if (class->usage_mask & LOCKF_USED_IN_SOFTIRQ)
306 nr_softirq_safe++; 306 nr_softirq_safe++;
307 if (class->usage_mask & LOCKF_ENABLED_SOFTIRQS) 307 if (class->usage_mask & LOCKF_ENABLED_SOFTIRQ)
308 nr_softirq_unsafe++; 308 nr_softirq_unsafe++;
309 if (class->usage_mask & LOCKF_USED_IN_HARDIRQ) 309 if (class->usage_mask & LOCKF_USED_IN_HARDIRQ)
310 nr_hardirq_safe++; 310 nr_hardirq_safe++;
311 if (class->usage_mask & LOCKF_ENABLED_HARDIRQS) 311 if (class->usage_mask & LOCKF_ENABLED_HARDIRQ)
312 nr_hardirq_unsafe++; 312 nr_hardirq_unsafe++;
313 if (class->usage_mask & LOCKF_USED_IN_IRQ_READ) 313 if (class->usage_mask & LOCKF_USED_IN_IRQ_READ)
314 nr_irq_read_safe++; 314 nr_irq_read_safe++;
315 if (class->usage_mask & LOCKF_ENABLED_IRQS_READ) 315 if (class->usage_mask & LOCKF_ENABLED_IRQ_READ)
316 nr_irq_read_unsafe++; 316 nr_irq_read_unsafe++;
317 if (class->usage_mask & LOCKF_USED_IN_SOFTIRQ_READ) 317 if (class->usage_mask & LOCKF_USED_IN_SOFTIRQ_READ)
318 nr_softirq_read_safe++; 318 nr_softirq_read_safe++;
319 if (class->usage_mask & LOCKF_ENABLED_SOFTIRQS_READ) 319 if (class->usage_mask & LOCKF_ENABLED_SOFTIRQ_READ)
320 nr_softirq_read_unsafe++; 320 nr_softirq_read_unsafe++;
321 if (class->usage_mask & LOCKF_USED_IN_HARDIRQ_READ) 321 if (class->usage_mask & LOCKF_USED_IN_HARDIRQ_READ)
322 nr_hardirq_read_safe++; 322 nr_hardirq_read_safe++;
323 if (class->usage_mask & LOCKF_ENABLED_HARDIRQS_READ) 323 if (class->usage_mask & LOCKF_ENABLED_HARDIRQ_READ)
324 nr_hardirq_read_unsafe++; 324 nr_hardirq_read_unsafe++;
325 325
326#ifdef CONFIG_PROVE_LOCKING 326#ifdef CONFIG_PROVE_LOCKING
@@ -601,6 +601,10 @@ static void seq_stats(struct seq_file *m, struct lock_stat_data *data)
601static void seq_header(struct seq_file *m) 601static void seq_header(struct seq_file *m)
602{ 602{
603 seq_printf(m, "lock_stat version 0.3\n"); 603 seq_printf(m, "lock_stat version 0.3\n");
604
605 if (unlikely(!debug_locks))
606 seq_printf(m, "*WARNING* lock debugging disabled!! - possibly due to a lockdep warning\n");
607
604 seq_line(m, '-', 0, 40 + 1 + 10 * (14 + 1)); 608 seq_line(m, '-', 0, 40 + 1 + 10 * (14 + 1));
605 seq_printf(m, "%40s %14s %14s %14s %14s %14s %14s %14s %14s " 609 seq_printf(m, "%40s %14s %14s %14s %14s %14s %14s %14s %14s "
606 "%14s %14s\n", 610 "%14s %14s\n",
diff --git a/kernel/lockdep_states.h b/kernel/lockdep_states.h
new file mode 100644
index 000000000000..995b0cc2b84c
--- /dev/null
+++ b/kernel/lockdep_states.h
@@ -0,0 +1,9 @@
1/*
2 * Lockdep states,
3 *
4 * please update XXX_LOCK_USAGE_STATES in include/linux/lockdep.h whenever
5 * you add one, or come up with a nice dynamic solution.
6 */
7LOCKDEP_STATE(HARDIRQ)
8LOCKDEP_STATE(SOFTIRQ)
9LOCKDEP_STATE(RECLAIM_FS)
diff --git a/kernel/module.c b/kernel/module.c
index ba22484a987e..8b742f2b3845 100644
--- a/kernel/module.c
+++ b/kernel/module.c
@@ -51,6 +51,7 @@
51#include <linux/tracepoint.h> 51#include <linux/tracepoint.h>
52#include <linux/ftrace.h> 52#include <linux/ftrace.h>
53#include <linux/async.h> 53#include <linux/async.h>
54#include <linux/percpu.h>
54 55
55#if 0 56#if 0
56#define DEBUGP printk 57#define DEBUGP printk
@@ -366,6 +367,34 @@ static struct module *find_module(const char *name)
366} 367}
367 368
368#ifdef CONFIG_SMP 369#ifdef CONFIG_SMP
370
371#ifdef CONFIG_HAVE_DYNAMIC_PER_CPU_AREA
372
373static void *percpu_modalloc(unsigned long size, unsigned long align,
374 const char *name)
375{
376 void *ptr;
377
378 if (align > PAGE_SIZE) {
379 printk(KERN_WARNING "%s: per-cpu alignment %li > %li\n",
380 name, align, PAGE_SIZE);
381 align = PAGE_SIZE;
382 }
383
384 ptr = __alloc_reserved_percpu(size, align);
385 if (!ptr)
386 printk(KERN_WARNING
387 "Could not allocate %lu bytes percpu data\n", size);
388 return ptr;
389}
390
391static void percpu_modfree(void *freeme)
392{
393 free_percpu(freeme);
394}
395
396#else /* ... !CONFIG_HAVE_DYNAMIC_PER_CPU_AREA */
397
369/* Number of blocks used and allocated. */ 398/* Number of blocks used and allocated. */
370static unsigned int pcpu_num_used, pcpu_num_allocated; 399static unsigned int pcpu_num_used, pcpu_num_allocated;
371/* Size of each block. -ve means used. */ 400/* Size of each block. -ve means used. */
@@ -480,21 +509,6 @@ static void percpu_modfree(void *freeme)
480 } 509 }
481} 510}
482 511
483static unsigned int find_pcpusec(Elf_Ehdr *hdr,
484 Elf_Shdr *sechdrs,
485 const char *secstrings)
486{
487 return find_sec(hdr, sechdrs, secstrings, ".data.percpu");
488}
489
490static void percpu_modcopy(void *pcpudest, const void *from, unsigned long size)
491{
492 int cpu;
493
494 for_each_possible_cpu(cpu)
495 memcpy(pcpudest + per_cpu_offset(cpu), from, size);
496}
497
498static int percpu_modinit(void) 512static int percpu_modinit(void)
499{ 513{
500 pcpu_num_used = 2; 514 pcpu_num_used = 2;
@@ -513,7 +527,26 @@ static int percpu_modinit(void)
513 return 0; 527 return 0;
514} 528}
515__initcall(percpu_modinit); 529__initcall(percpu_modinit);
530
531#endif /* CONFIG_HAVE_DYNAMIC_PER_CPU_AREA */
532
533static unsigned int find_pcpusec(Elf_Ehdr *hdr,
534 Elf_Shdr *sechdrs,
535 const char *secstrings)
536{
537 return find_sec(hdr, sechdrs, secstrings, ".data.percpu");
538}
539
540static void percpu_modcopy(void *pcpudest, const void *from, unsigned long size)
541{
542 int cpu;
543
544 for_each_possible_cpu(cpu)
545 memcpy(pcpudest + per_cpu_offset(cpu), from, size);
546}
547
516#else /* ... !CONFIG_SMP */ 548#else /* ... !CONFIG_SMP */
549
517static inline void *percpu_modalloc(unsigned long size, unsigned long align, 550static inline void *percpu_modalloc(unsigned long size, unsigned long align,
518 const char *name) 551 const char *name)
519{ 552{
@@ -535,6 +568,7 @@ static inline void percpu_modcopy(void *pcpudst, const void *src,
535 /* pcpusec should be 0, and size of that section should be 0. */ 568 /* pcpusec should be 0, and size of that section should be 0. */
536 BUG_ON(size != 0); 569 BUG_ON(size != 0);
537} 570}
571
538#endif /* CONFIG_SMP */ 572#endif /* CONFIG_SMP */
539 573
540#define MODINFO_ATTR(field) \ 574#define MODINFO_ATTR(field) \
@@ -2735,7 +2769,7 @@ int is_module_address(unsigned long addr)
2735 2769
2736 2770
2737/* Is this a valid kernel address? */ 2771/* Is this a valid kernel address? */
2738__notrace_funcgraph struct module *__module_text_address(unsigned long addr) 2772struct module *__module_text_address(unsigned long addr)
2739{ 2773{
2740 struct module *mod; 2774 struct module *mod;
2741 2775
diff --git a/kernel/mutex-debug.c b/kernel/mutex-debug.c
index 1d94160eb532..50d022e5a560 100644
--- a/kernel/mutex-debug.c
+++ b/kernel/mutex-debug.c
@@ -26,11 +26,6 @@
26/* 26/*
27 * Must be called with lock->wait_lock held. 27 * Must be called with lock->wait_lock held.
28 */ 28 */
29void debug_mutex_set_owner(struct mutex *lock, struct thread_info *new_owner)
30{
31 lock->owner = new_owner;
32}
33
34void debug_mutex_lock_common(struct mutex *lock, struct mutex_waiter *waiter) 29void debug_mutex_lock_common(struct mutex *lock, struct mutex_waiter *waiter)
35{ 30{
36 memset(waiter, MUTEX_DEBUG_INIT, sizeof(*waiter)); 31 memset(waiter, MUTEX_DEBUG_INIT, sizeof(*waiter));
@@ -59,7 +54,6 @@ void debug_mutex_add_waiter(struct mutex *lock, struct mutex_waiter *waiter,
59 54
60 /* Mark the current thread as blocked on the lock: */ 55 /* Mark the current thread as blocked on the lock: */
61 ti->task->blocked_on = waiter; 56 ti->task->blocked_on = waiter;
62 waiter->lock = lock;
63} 57}
64 58
65void mutex_remove_waiter(struct mutex *lock, struct mutex_waiter *waiter, 59void mutex_remove_waiter(struct mutex *lock, struct mutex_waiter *waiter,
@@ -82,7 +76,7 @@ void debug_mutex_unlock(struct mutex *lock)
82 DEBUG_LOCKS_WARN_ON(lock->magic != lock); 76 DEBUG_LOCKS_WARN_ON(lock->magic != lock);
83 DEBUG_LOCKS_WARN_ON(lock->owner != current_thread_info()); 77 DEBUG_LOCKS_WARN_ON(lock->owner != current_thread_info());
84 DEBUG_LOCKS_WARN_ON(!lock->wait_list.prev && !lock->wait_list.next); 78 DEBUG_LOCKS_WARN_ON(!lock->wait_list.prev && !lock->wait_list.next);
85 DEBUG_LOCKS_WARN_ON(lock->owner != current_thread_info()); 79 mutex_clear_owner(lock);
86} 80}
87 81
88void debug_mutex_init(struct mutex *lock, const char *name, 82void debug_mutex_init(struct mutex *lock, const char *name,
@@ -95,7 +89,6 @@ void debug_mutex_init(struct mutex *lock, const char *name,
95 debug_check_no_locks_freed((void *)lock, sizeof(*lock)); 89 debug_check_no_locks_freed((void *)lock, sizeof(*lock));
96 lockdep_init_map(&lock->dep_map, name, key, 0); 90 lockdep_init_map(&lock->dep_map, name, key, 0);
97#endif 91#endif
98 lock->owner = NULL;
99 lock->magic = lock; 92 lock->magic = lock;
100} 93}
101 94
diff --git a/kernel/mutex-debug.h b/kernel/mutex-debug.h
index babfbdfc534b..6b2d735846a5 100644
--- a/kernel/mutex-debug.h
+++ b/kernel/mutex-debug.h
@@ -13,14 +13,6 @@
13/* 13/*
14 * This must be called with lock->wait_lock held. 14 * This must be called with lock->wait_lock held.
15 */ 15 */
16extern void
17debug_mutex_set_owner(struct mutex *lock, struct thread_info *new_owner);
18
19static inline void debug_mutex_clear_owner(struct mutex *lock)
20{
21 lock->owner = NULL;
22}
23
24extern void debug_mutex_lock_common(struct mutex *lock, 16extern void debug_mutex_lock_common(struct mutex *lock,
25 struct mutex_waiter *waiter); 17 struct mutex_waiter *waiter);
26extern void debug_mutex_wake_waiter(struct mutex *lock, 18extern void debug_mutex_wake_waiter(struct mutex *lock,
@@ -35,6 +27,16 @@ extern void debug_mutex_unlock(struct mutex *lock);
35extern void debug_mutex_init(struct mutex *lock, const char *name, 27extern void debug_mutex_init(struct mutex *lock, const char *name,
36 struct lock_class_key *key); 28 struct lock_class_key *key);
37 29
30static inline void mutex_set_owner(struct mutex *lock)
31{
32 lock->owner = current_thread_info();
33}
34
35static inline void mutex_clear_owner(struct mutex *lock)
36{
37 lock->owner = NULL;
38}
39
38#define spin_lock_mutex(lock, flags) \ 40#define spin_lock_mutex(lock, flags) \
39 do { \ 41 do { \
40 struct mutex *l = container_of(lock, struct mutex, wait_lock); \ 42 struct mutex *l = container_of(lock, struct mutex, wait_lock); \
diff --git a/kernel/mutex.c b/kernel/mutex.c
index 4f45d4b658ef..5d79781394a3 100644
--- a/kernel/mutex.c
+++ b/kernel/mutex.c
@@ -10,6 +10,11 @@
10 * Many thanks to Arjan van de Ven, Thomas Gleixner, Steven Rostedt and 10 * Many thanks to Arjan van de Ven, Thomas Gleixner, Steven Rostedt and
11 * David Howells for suggestions and improvements. 11 * David Howells for suggestions and improvements.
12 * 12 *
13 * - Adaptive spinning for mutexes by Peter Zijlstra. (Ported to mainline
14 * from the -rt tree, where it was originally implemented for rtmutexes
15 * by Steven Rostedt, based on work by Gregory Haskins, Peter Morreale
16 * and Sven Dietrich.
17 *
13 * Also see Documentation/mutex-design.txt. 18 * Also see Documentation/mutex-design.txt.
14 */ 19 */
15#include <linux/mutex.h> 20#include <linux/mutex.h>
@@ -46,6 +51,7 @@ __mutex_init(struct mutex *lock, const char *name, struct lock_class_key *key)
46 atomic_set(&lock->count, 1); 51 atomic_set(&lock->count, 1);
47 spin_lock_init(&lock->wait_lock); 52 spin_lock_init(&lock->wait_lock);
48 INIT_LIST_HEAD(&lock->wait_list); 53 INIT_LIST_HEAD(&lock->wait_list);
54 mutex_clear_owner(lock);
49 55
50 debug_mutex_init(lock, name, key); 56 debug_mutex_init(lock, name, key);
51} 57}
@@ -91,6 +97,7 @@ void inline __sched mutex_lock(struct mutex *lock)
91 * 'unlocked' into 'locked' state. 97 * 'unlocked' into 'locked' state.
92 */ 98 */
93 __mutex_fastpath_lock(&lock->count, __mutex_lock_slowpath); 99 __mutex_fastpath_lock(&lock->count, __mutex_lock_slowpath);
100 mutex_set_owner(lock);
94} 101}
95 102
96EXPORT_SYMBOL(mutex_lock); 103EXPORT_SYMBOL(mutex_lock);
@@ -115,6 +122,14 @@ void __sched mutex_unlock(struct mutex *lock)
115 * The unlocking fastpath is the 0->1 transition from 'locked' 122 * The unlocking fastpath is the 0->1 transition from 'locked'
116 * into 'unlocked' state: 123 * into 'unlocked' state:
117 */ 124 */
125#ifndef CONFIG_DEBUG_MUTEXES
126 /*
127 * When debugging is enabled we must not clear the owner before time,
128 * the slow path will always be taken, and that clears the owner field
129 * after verifying that it was indeed current.
130 */
131 mutex_clear_owner(lock);
132#endif
118 __mutex_fastpath_unlock(&lock->count, __mutex_unlock_slowpath); 133 __mutex_fastpath_unlock(&lock->count, __mutex_unlock_slowpath);
119} 134}
120 135
@@ -129,21 +144,75 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,
129{ 144{
130 struct task_struct *task = current; 145 struct task_struct *task = current;
131 struct mutex_waiter waiter; 146 struct mutex_waiter waiter;
132 unsigned int old_val;
133 unsigned long flags; 147 unsigned long flags;
134 148
149 preempt_disable();
150 mutex_acquire(&lock->dep_map, subclass, 0, ip);
151#if defined(CONFIG_SMP) && !defined(CONFIG_DEBUG_MUTEXES)
152 /*
153 * Optimistic spinning.
154 *
155 * We try to spin for acquisition when we find that there are no
156 * pending waiters and the lock owner is currently running on a
157 * (different) CPU.
158 *
159 * The rationale is that if the lock owner is running, it is likely to
160 * release the lock soon.
161 *
162 * Since this needs the lock owner, and this mutex implementation
163 * doesn't track the owner atomically in the lock field, we need to
164 * track it non-atomically.
165 *
166 * We can't do this for DEBUG_MUTEXES because that relies on wait_lock
167 * to serialize everything.
168 */
169
170 for (;;) {
171 struct thread_info *owner;
172
173 /*
174 * If there's an owner, wait for it to either
175 * release the lock or go to sleep.
176 */
177 owner = ACCESS_ONCE(lock->owner);
178 if (owner && !mutex_spin_on_owner(lock, owner))
179 break;
180
181 if (atomic_cmpxchg(&lock->count, 1, 0) == 1) {
182 lock_acquired(&lock->dep_map, ip);
183 mutex_set_owner(lock);
184 preempt_enable();
185 return 0;
186 }
187
188 /*
189 * When there's no owner, we might have preempted between the
190 * owner acquiring the lock and setting the owner field. If
191 * we're an RT task that will live-lock because we won't let
192 * the owner complete.
193 */
194 if (!owner && (need_resched() || rt_task(task)))
195 break;
196
197 /*
198 * The cpu_relax() call is a compiler barrier which forces
199 * everything in this loop to be re-loaded. We don't need
200 * memory barriers as we'll eventually observe the right
201 * values at the cost of a few extra spins.
202 */
203 cpu_relax();
204 }
205#endif
135 spin_lock_mutex(&lock->wait_lock, flags); 206 spin_lock_mutex(&lock->wait_lock, flags);
136 207
137 debug_mutex_lock_common(lock, &waiter); 208 debug_mutex_lock_common(lock, &waiter);
138 mutex_acquire(&lock->dep_map, subclass, 0, ip);
139 debug_mutex_add_waiter(lock, &waiter, task_thread_info(task)); 209 debug_mutex_add_waiter(lock, &waiter, task_thread_info(task));
140 210
141 /* add waiting tasks to the end of the waitqueue (FIFO): */ 211 /* add waiting tasks to the end of the waitqueue (FIFO): */
142 list_add_tail(&waiter.list, &lock->wait_list); 212 list_add_tail(&waiter.list, &lock->wait_list);
143 waiter.task = task; 213 waiter.task = task;
144 214
145 old_val = atomic_xchg(&lock->count, -1); 215 if (atomic_xchg(&lock->count, -1) == 1)
146 if (old_val == 1)
147 goto done; 216 goto done;
148 217
149 lock_contended(&lock->dep_map, ip); 218 lock_contended(&lock->dep_map, ip);
@@ -158,8 +227,7 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,
158 * that when we release the lock, we properly wake up the 227 * that when we release the lock, we properly wake up the
159 * other waiters: 228 * other waiters:
160 */ 229 */
161 old_val = atomic_xchg(&lock->count, -1); 230 if (atomic_xchg(&lock->count, -1) == 1)
162 if (old_val == 1)
163 break; 231 break;
164 232
165 /* 233 /*
@@ -173,21 +241,22 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,
173 spin_unlock_mutex(&lock->wait_lock, flags); 241 spin_unlock_mutex(&lock->wait_lock, flags);
174 242
175 debug_mutex_free_waiter(&waiter); 243 debug_mutex_free_waiter(&waiter);
244 preempt_enable();
176 return -EINTR; 245 return -EINTR;
177 } 246 }
178 __set_task_state(task, state); 247 __set_task_state(task, state);
179 248
180 /* didnt get the lock, go to sleep: */ 249 /* didnt get the lock, go to sleep: */
181 spin_unlock_mutex(&lock->wait_lock, flags); 250 spin_unlock_mutex(&lock->wait_lock, flags);
182 schedule(); 251 __schedule();
183 spin_lock_mutex(&lock->wait_lock, flags); 252 spin_lock_mutex(&lock->wait_lock, flags);
184 } 253 }
185 254
186done: 255done:
187 lock_acquired(&lock->dep_map, ip); 256 lock_acquired(&lock->dep_map, ip);
188 /* got the lock - rejoice! */ 257 /* got the lock - rejoice! */
189 mutex_remove_waiter(lock, &waiter, task_thread_info(task)); 258 mutex_remove_waiter(lock, &waiter, current_thread_info());
190 debug_mutex_set_owner(lock, task_thread_info(task)); 259 mutex_set_owner(lock);
191 260
192 /* set it to 0 if there are no waiters left: */ 261 /* set it to 0 if there are no waiters left: */
193 if (likely(list_empty(&lock->wait_list))) 262 if (likely(list_empty(&lock->wait_list)))
@@ -196,6 +265,7 @@ done:
196 spin_unlock_mutex(&lock->wait_lock, flags); 265 spin_unlock_mutex(&lock->wait_lock, flags);
197 266
198 debug_mutex_free_waiter(&waiter); 267 debug_mutex_free_waiter(&waiter);
268 preempt_enable();
199 269
200 return 0; 270 return 0;
201} 271}
@@ -222,7 +292,8 @@ int __sched
222mutex_lock_interruptible_nested(struct mutex *lock, unsigned int subclass) 292mutex_lock_interruptible_nested(struct mutex *lock, unsigned int subclass)
223{ 293{
224 might_sleep(); 294 might_sleep();
225 return __mutex_lock_common(lock, TASK_INTERRUPTIBLE, subclass, _RET_IP_); 295 return __mutex_lock_common(lock, TASK_INTERRUPTIBLE,
296 subclass, _RET_IP_);
226} 297}
227 298
228EXPORT_SYMBOL_GPL(mutex_lock_interruptible_nested); 299EXPORT_SYMBOL_GPL(mutex_lock_interruptible_nested);
@@ -260,8 +331,6 @@ __mutex_unlock_common_slowpath(atomic_t *lock_count, int nested)
260 wake_up_process(waiter->task); 331 wake_up_process(waiter->task);
261 } 332 }
262 333
263 debug_mutex_clear_owner(lock);
264
265 spin_unlock_mutex(&lock->wait_lock, flags); 334 spin_unlock_mutex(&lock->wait_lock, flags);
266} 335}
267 336
@@ -298,18 +367,30 @@ __mutex_lock_interruptible_slowpath(atomic_t *lock_count);
298 */ 367 */
299int __sched mutex_lock_interruptible(struct mutex *lock) 368int __sched mutex_lock_interruptible(struct mutex *lock)
300{ 369{
370 int ret;
371
301 might_sleep(); 372 might_sleep();
302 return __mutex_fastpath_lock_retval 373 ret = __mutex_fastpath_lock_retval
303 (&lock->count, __mutex_lock_interruptible_slowpath); 374 (&lock->count, __mutex_lock_interruptible_slowpath);
375 if (!ret)
376 mutex_set_owner(lock);
377
378 return ret;
304} 379}
305 380
306EXPORT_SYMBOL(mutex_lock_interruptible); 381EXPORT_SYMBOL(mutex_lock_interruptible);
307 382
308int __sched mutex_lock_killable(struct mutex *lock) 383int __sched mutex_lock_killable(struct mutex *lock)
309{ 384{
385 int ret;
386
310 might_sleep(); 387 might_sleep();
311 return __mutex_fastpath_lock_retval 388 ret = __mutex_fastpath_lock_retval
312 (&lock->count, __mutex_lock_killable_slowpath); 389 (&lock->count, __mutex_lock_killable_slowpath);
390 if (!ret)
391 mutex_set_owner(lock);
392
393 return ret;
313} 394}
314EXPORT_SYMBOL(mutex_lock_killable); 395EXPORT_SYMBOL(mutex_lock_killable);
315 396
@@ -352,9 +433,10 @@ static inline int __mutex_trylock_slowpath(atomic_t *lock_count)
352 433
353 prev = atomic_xchg(&lock->count, -1); 434 prev = atomic_xchg(&lock->count, -1);
354 if (likely(prev == 1)) { 435 if (likely(prev == 1)) {
355 debug_mutex_set_owner(lock, current_thread_info()); 436 mutex_set_owner(lock);
356 mutex_acquire(&lock->dep_map, 0, 1, _RET_IP_); 437 mutex_acquire(&lock->dep_map, 0, 1, _RET_IP_);
357 } 438 }
439
358 /* Set it back to 0 if there are no waiters: */ 440 /* Set it back to 0 if there are no waiters: */
359 if (likely(list_empty(&lock->wait_list))) 441 if (likely(list_empty(&lock->wait_list)))
360 atomic_set(&lock->count, 0); 442 atomic_set(&lock->count, 0);
@@ -380,8 +462,13 @@ static inline int __mutex_trylock_slowpath(atomic_t *lock_count)
380 */ 462 */
381int __sched mutex_trylock(struct mutex *lock) 463int __sched mutex_trylock(struct mutex *lock)
382{ 464{
383 return __mutex_fastpath_trylock(&lock->count, 465 int ret;
384 __mutex_trylock_slowpath); 466
467 ret = __mutex_fastpath_trylock(&lock->count, __mutex_trylock_slowpath);
468 if (ret)
469 mutex_set_owner(lock);
470
471 return ret;
385} 472}
386 473
387EXPORT_SYMBOL(mutex_trylock); 474EXPORT_SYMBOL(mutex_trylock);
diff --git a/kernel/mutex.h b/kernel/mutex.h
index a075dafbb290..67578ca48f94 100644
--- a/kernel/mutex.h
+++ b/kernel/mutex.h
@@ -16,8 +16,26 @@
16#define mutex_remove_waiter(lock, waiter, ti) \ 16#define mutex_remove_waiter(lock, waiter, ti) \
17 __list_del((waiter)->list.prev, (waiter)->list.next) 17 __list_del((waiter)->list.prev, (waiter)->list.next)
18 18
19#define debug_mutex_set_owner(lock, new_owner) do { } while (0) 19#ifdef CONFIG_SMP
20#define debug_mutex_clear_owner(lock) do { } while (0) 20static inline void mutex_set_owner(struct mutex *lock)
21{
22 lock->owner = current_thread_info();
23}
24
25static inline void mutex_clear_owner(struct mutex *lock)
26{
27 lock->owner = NULL;
28}
29#else
30static inline void mutex_set_owner(struct mutex *lock)
31{
32}
33
34static inline void mutex_clear_owner(struct mutex *lock)
35{
36}
37#endif
38
21#define debug_mutex_wake_waiter(lock, waiter) do { } while (0) 39#define debug_mutex_wake_waiter(lock, waiter) do { } while (0)
22#define debug_mutex_free_waiter(waiter) do { } while (0) 40#define debug_mutex_free_waiter(waiter) do { } while (0)
23#define debug_mutex_add_waiter(lock, waiter, ti) do { } while (0) 41#define debug_mutex_add_waiter(lock, waiter, ti) do { } while (0)
diff --git a/kernel/panic.c b/kernel/panic.c
index 2a2ff36ff44d..32fe4eff1b89 100644
--- a/kernel/panic.c
+++ b/kernel/panic.c
@@ -74,6 +74,9 @@ NORET_TYPE void panic(const char * fmt, ...)
74 vsnprintf(buf, sizeof(buf), fmt, args); 74 vsnprintf(buf, sizeof(buf), fmt, args);
75 va_end(args); 75 va_end(args);
76 printk(KERN_EMERG "Kernel panic - not syncing: %s\n",buf); 76 printk(KERN_EMERG "Kernel panic - not syncing: %s\n",buf);
77#ifdef CONFIG_DEBUG_BUGVERBOSE
78 dump_stack();
79#endif
77 bust_spinlocks(0); 80 bust_spinlocks(0);
78 81
79 /* 82 /*
@@ -355,15 +358,18 @@ EXPORT_SYMBOL(warn_slowpath);
355#endif 358#endif
356 359
357#ifdef CONFIG_CC_STACKPROTECTOR 360#ifdef CONFIG_CC_STACKPROTECTOR
361
358/* 362/*
359 * Called when gcc's -fstack-protector feature is used, and 363 * Called when gcc's -fstack-protector feature is used, and
360 * gcc detects corruption of the on-stack canary value 364 * gcc detects corruption of the on-stack canary value
361 */ 365 */
362void __stack_chk_fail(void) 366void __stack_chk_fail(void)
363{ 367{
364 panic("stack-protector: Kernel stack is corrupted"); 368 panic("stack-protector: Kernel stack is corrupted in: %p\n",
369 __builtin_return_address(0));
365} 370}
366EXPORT_SYMBOL(__stack_chk_fail); 371EXPORT_SYMBOL(__stack_chk_fail);
372
367#endif 373#endif
368 374
369core_param(panic, panic_timeout, int, 0644); 375core_param(panic, panic_timeout, int, 0644);
diff --git a/kernel/relay.c b/kernel/relay.c
index 9d79b7854fa6..edc0ba6d8160 100644
--- a/kernel/relay.c
+++ b/kernel/relay.c
@@ -677,9 +677,7 @@ int relay_late_setup_files(struct rchan *chan,
677 */ 677 */
678 for_each_online_cpu(i) { 678 for_each_online_cpu(i) {
679 if (unlikely(!chan->buf[i])) { 679 if (unlikely(!chan->buf[i])) {
680 printk(KERN_ERR "relay_late_setup_files: CPU %u " 680 WARN_ONCE(1, KERN_ERR "CPU has no buffer!\n");
681 "has no buffer, it must have!\n", i);
682 BUG();
683 err = -EINVAL; 681 err = -EINVAL;
684 break; 682 break;
685 } 683 }
diff --git a/kernel/sched.c b/kernel/sched.c
index 8e2558c2ba67..7299083e69e7 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -4404,10 +4404,7 @@ void scheduler_tick(void)
4404#endif 4404#endif
4405} 4405}
4406 4406
4407#if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \ 4407unsigned long get_parent_ip(unsigned long addr)
4408 defined(CONFIG_PREEMPT_TRACER))
4409
4410static inline unsigned long get_parent_ip(unsigned long addr)
4411{ 4408{
4412 if (in_lock_functions(addr)) { 4409 if (in_lock_functions(addr)) {
4413 addr = CALLER_ADDR2; 4410 addr = CALLER_ADDR2;
@@ -4417,6 +4414,9 @@ static inline unsigned long get_parent_ip(unsigned long addr)
4417 return addr; 4414 return addr;
4418} 4415}
4419 4416
4417#if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \
4418 defined(CONFIG_PREEMPT_TRACER))
4419
4420void __kprobes add_preempt_count(int val) 4420void __kprobes add_preempt_count(int val)
4421{ 4421{
4422#ifdef CONFIG_DEBUG_PREEMPT 4422#ifdef CONFIG_DEBUG_PREEMPT
@@ -4543,15 +4543,13 @@ pick_next_task(struct rq *rq, struct task_struct *prev)
4543/* 4543/*
4544 * schedule() is the main scheduler function. 4544 * schedule() is the main scheduler function.
4545 */ 4545 */
4546asmlinkage void __sched schedule(void) 4546asmlinkage void __sched __schedule(void)
4547{ 4547{
4548 struct task_struct *prev, *next; 4548 struct task_struct *prev, *next;
4549 unsigned long *switch_count; 4549 unsigned long *switch_count;
4550 struct rq *rq; 4550 struct rq *rq;
4551 int cpu; 4551 int cpu;
4552 4552
4553need_resched:
4554 preempt_disable();
4555 cpu = smp_processor_id(); 4553 cpu = smp_processor_id();
4556 rq = cpu_rq(cpu); 4554 rq = cpu_rq(cpu);
4557 rcu_qsctr_inc(cpu); 4555 rcu_qsctr_inc(cpu);
@@ -4608,13 +4606,80 @@ need_resched_nonpreemptible:
4608 4606
4609 if (unlikely(reacquire_kernel_lock(current) < 0)) 4607 if (unlikely(reacquire_kernel_lock(current) < 0))
4610 goto need_resched_nonpreemptible; 4608 goto need_resched_nonpreemptible;
4609}
4611 4610
4611asmlinkage void __sched schedule(void)
4612{
4613need_resched:
4614 preempt_disable();
4615 __schedule();
4612 preempt_enable_no_resched(); 4616 preempt_enable_no_resched();
4613 if (unlikely(test_thread_flag(TIF_NEED_RESCHED))) 4617 if (unlikely(test_thread_flag(TIF_NEED_RESCHED)))
4614 goto need_resched; 4618 goto need_resched;
4615} 4619}
4616EXPORT_SYMBOL(schedule); 4620EXPORT_SYMBOL(schedule);
4617 4621
4622#ifdef CONFIG_SMP
4623/*
4624 * Look out! "owner" is an entirely speculative pointer
4625 * access and not reliable.
4626 */
4627int mutex_spin_on_owner(struct mutex *lock, struct thread_info *owner)
4628{
4629 unsigned int cpu;
4630 struct rq *rq;
4631
4632 if (!sched_feat(OWNER_SPIN))
4633 return 0;
4634
4635#ifdef CONFIG_DEBUG_PAGEALLOC
4636 /*
4637 * Need to access the cpu field knowing that
4638 * DEBUG_PAGEALLOC could have unmapped it if
4639 * the mutex owner just released it and exited.
4640 */
4641 if (probe_kernel_address(&owner->cpu, cpu))
4642 goto out;
4643#else
4644 cpu = owner->cpu;
4645#endif
4646
4647 /*
4648 * Even if the access succeeded (likely case),
4649 * the cpu field may no longer be valid.
4650 */
4651 if (cpu >= nr_cpumask_bits)
4652 goto out;
4653
4654 /*
4655 * We need to validate that we can do a
4656 * get_cpu() and that we have the percpu area.
4657 */
4658 if (!cpu_online(cpu))
4659 goto out;
4660
4661 rq = cpu_rq(cpu);
4662
4663 for (;;) {
4664 /*
4665 * Owner changed, break to re-assess state.
4666 */
4667 if (lock->owner != owner)
4668 break;
4669
4670 /*
4671 * Is that owner really running on that cpu?
4672 */
4673 if (task_thread_info(rq->curr) != owner || need_resched())
4674 return 0;
4675
4676 cpu_relax();
4677 }
4678out:
4679 return 1;
4680}
4681#endif
4682
4618#ifdef CONFIG_PREEMPT 4683#ifdef CONFIG_PREEMPT
4619/* 4684/*
4620 * this is the entry point to schedule() from in-kernel preemption 4685 * this is the entry point to schedule() from in-kernel preemption
@@ -5944,12 +6009,7 @@ void sched_show_task(struct task_struct *p)
5944 printk(KERN_CONT " %016lx ", thread_saved_pc(p)); 6009 printk(KERN_CONT " %016lx ", thread_saved_pc(p));
5945#endif 6010#endif
5946#ifdef CONFIG_DEBUG_STACK_USAGE 6011#ifdef CONFIG_DEBUG_STACK_USAGE
5947 { 6012 free = stack_not_used(p);
5948 unsigned long *n = end_of_stack(p);
5949 while (!*n)
5950 n++;
5951 free = (unsigned long)n - (unsigned long)end_of_stack(p);
5952 }
5953#endif 6013#endif
5954 printk(KERN_CONT "%5lu %5d %6d\n", free, 6014 printk(KERN_CONT "%5lu %5d %6d\n", free,
5955 task_pid_nr(p), task_pid_nr(p->real_parent)); 6015 task_pid_nr(p), task_pid_nr(p->real_parent));
@@ -9490,7 +9550,7 @@ cpuacct_destroy(struct cgroup_subsys *ss, struct cgroup *cgrp)
9490 9550
9491static u64 cpuacct_cpuusage_read(struct cpuacct *ca, int cpu) 9551static u64 cpuacct_cpuusage_read(struct cpuacct *ca, int cpu)
9492{ 9552{
9493 u64 *cpuusage = percpu_ptr(ca->cpuusage, cpu); 9553 u64 *cpuusage = per_cpu_ptr(ca->cpuusage, cpu);
9494 u64 data; 9554 u64 data;
9495 9555
9496#ifndef CONFIG_64BIT 9556#ifndef CONFIG_64BIT
@@ -9509,7 +9569,7 @@ static u64 cpuacct_cpuusage_read(struct cpuacct *ca, int cpu)
9509 9569
9510static void cpuacct_cpuusage_write(struct cpuacct *ca, int cpu, u64 val) 9570static void cpuacct_cpuusage_write(struct cpuacct *ca, int cpu, u64 val)
9511{ 9571{
9512 u64 *cpuusage = percpu_ptr(ca->cpuusage, cpu); 9572 u64 *cpuusage = per_cpu_ptr(ca->cpuusage, cpu);
9513 9573
9514#ifndef CONFIG_64BIT 9574#ifndef CONFIG_64BIT
9515 /* 9575 /*
@@ -9605,7 +9665,7 @@ static void cpuacct_charge(struct task_struct *tsk, u64 cputime)
9605 ca = task_ca(tsk); 9665 ca = task_ca(tsk);
9606 9666
9607 for (; ca; ca = ca->parent) { 9667 for (; ca; ca = ca->parent) {
9608 u64 *cpuusage = percpu_ptr(ca->cpuusage, cpu); 9668 u64 *cpuusage = per_cpu_ptr(ca->cpuusage, cpu);
9609 *cpuusage += cputime; 9669 *cpuusage += cputime;
9610 } 9670 }
9611} 9671}
diff --git a/kernel/sched_clock.c b/kernel/sched_clock.c
index a0b0852414cc..7ec82c1c61c5 100644
--- a/kernel/sched_clock.c
+++ b/kernel/sched_clock.c
@@ -24,11 +24,12 @@
24 * The clock: sched_clock_cpu() is monotonic per cpu, and should be somewhat 24 * The clock: sched_clock_cpu() is monotonic per cpu, and should be somewhat
25 * consistent between cpus (never more than 2 jiffies difference). 25 * consistent between cpus (never more than 2 jiffies difference).
26 */ 26 */
27#include <linux/sched.h>
28#include <linux/percpu.h>
29#include <linux/spinlock.h> 27#include <linux/spinlock.h>
30#include <linux/ktime.h> 28#include <linux/hardirq.h>
31#include <linux/module.h> 29#include <linux/module.h>
30#include <linux/percpu.h>
31#include <linux/ktime.h>
32#include <linux/sched.h>
32 33
33/* 34/*
34 * Scheduler clock - returns current time in nanosec units. 35 * Scheduler clock - returns current time in nanosec units.
@@ -43,6 +44,10 @@ unsigned long long __attribute__((weak)) sched_clock(void)
43static __read_mostly int sched_clock_running; 44static __read_mostly int sched_clock_running;
44 45
45#ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK 46#ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
47__read_mostly int sched_clock_stable;
48#else
49static const int sched_clock_stable = 1;
50#endif
46 51
47struct sched_clock_data { 52struct sched_clock_data {
48 /* 53 /*
@@ -87,7 +92,7 @@ void sched_clock_init(void)
87} 92}
88 93
89/* 94/*
90 * min,max except they take wrapping into account 95 * min, max except they take wrapping into account
91 */ 96 */
92 97
93static inline u64 wrap_min(u64 x, u64 y) 98static inline u64 wrap_min(u64 x, u64 y)
@@ -116,10 +121,13 @@ static u64 __update_sched_clock(struct sched_clock_data *scd, u64 now)
116 if (unlikely(delta < 0)) 121 if (unlikely(delta < 0))
117 delta = 0; 122 delta = 0;
118 123
124 if (unlikely(!sched_clock_running))
125 return 0ull;
126
119 /* 127 /*
120 * scd->clock = clamp(scd->tick_gtod + delta, 128 * scd->clock = clamp(scd->tick_gtod + delta,
121 * max(scd->tick_gtod, scd->clock), 129 * max(scd->tick_gtod, scd->clock),
122 * scd->tick_gtod + TICK_NSEC); 130 * scd->tick_gtod + TICK_NSEC);
123 */ 131 */
124 132
125 clock = scd->tick_gtod + delta; 133 clock = scd->tick_gtod + delta;
@@ -148,8 +156,20 @@ static void lock_double_clock(struct sched_clock_data *data1,
148 156
149u64 sched_clock_cpu(int cpu) 157u64 sched_clock_cpu(int cpu)
150{ 158{
151 struct sched_clock_data *scd = cpu_sdc(cpu);
152 u64 now, clock, this_clock, remote_clock; 159 u64 now, clock, this_clock, remote_clock;
160 struct sched_clock_data *scd;
161
162 if (sched_clock_stable)
163 return sched_clock();
164
165 scd = cpu_sdc(cpu);
166
167 /*
168 * Normally this is not called in NMI context - but if it is,
169 * trying to do any locking here is totally lethal.
170 */
171 if (unlikely(in_nmi()))
172 return scd->clock;
153 173
154 if (unlikely(!sched_clock_running)) 174 if (unlikely(!sched_clock_running))
155 return 0ull; 175 return 0ull;
@@ -193,6 +213,8 @@ u64 sched_clock_cpu(int cpu)
193 return clock; 213 return clock;
194} 214}
195 215
216#ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
217
196void sched_clock_tick(void) 218void sched_clock_tick(void)
197{ 219{
198 struct sched_clock_data *scd = this_scd(); 220 struct sched_clock_data *scd = this_scd();
@@ -235,22 +257,7 @@ void sched_clock_idle_wakeup_event(u64 delta_ns)
235} 257}
236EXPORT_SYMBOL_GPL(sched_clock_idle_wakeup_event); 258EXPORT_SYMBOL_GPL(sched_clock_idle_wakeup_event);
237 259
238#else /* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */ 260#endif /* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */
239
240void sched_clock_init(void)
241{
242 sched_clock_running = 1;
243}
244
245u64 sched_clock_cpu(int cpu)
246{
247 if (unlikely(!sched_clock_running))
248 return 0;
249
250 return sched_clock();
251}
252
253#endif
254 261
255unsigned long long cpu_clock(int cpu) 262unsigned long long cpu_clock(int cpu)
256{ 263{
diff --git a/kernel/sched_features.h b/kernel/sched_features.h
index da5d93b5d2c6..07bc02e99ab1 100644
--- a/kernel/sched_features.h
+++ b/kernel/sched_features.h
@@ -13,3 +13,4 @@ SCHED_FEAT(LB_WAKEUP_UPDATE, 1)
13SCHED_FEAT(ASYM_EFF_LOAD, 1) 13SCHED_FEAT(ASYM_EFF_LOAD, 1)
14SCHED_FEAT(WAKEUP_OVERLAP, 0) 14SCHED_FEAT(WAKEUP_OVERLAP, 0)
15SCHED_FEAT(LAST_BUDDY, 1) 15SCHED_FEAT(LAST_BUDDY, 1)
16SCHED_FEAT(OWNER_SPIN, 1)
diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c
index bac1061cea2f..da932f4c8524 100644
--- a/kernel/sched_rt.c
+++ b/kernel/sched_rt.c
@@ -960,12 +960,13 @@ static struct task_struct *pick_next_highest_task_rt(struct rq *rq, int cpu)
960 960
961static DEFINE_PER_CPU(cpumask_var_t, local_cpu_mask); 961static DEFINE_PER_CPU(cpumask_var_t, local_cpu_mask);
962 962
963static inline int pick_optimal_cpu(int this_cpu, cpumask_t *mask) 963static inline int pick_optimal_cpu(int this_cpu,
964 const struct cpumask *mask)
964{ 965{
965 int first; 966 int first;
966 967
967 /* "this_cpu" is cheaper to preempt than a remote processor */ 968 /* "this_cpu" is cheaper to preempt than a remote processor */
968 if ((this_cpu != -1) && cpu_isset(this_cpu, *mask)) 969 if ((this_cpu != -1) && cpumask_test_cpu(this_cpu, mask))
969 return this_cpu; 970 return this_cpu;
970 971
971 first = cpumask_first(mask); 972 first = cpumask_first(mask);
@@ -981,6 +982,7 @@ static int find_lowest_rq(struct task_struct *task)
981 struct cpumask *lowest_mask = __get_cpu_var(local_cpu_mask); 982 struct cpumask *lowest_mask = __get_cpu_var(local_cpu_mask);
982 int this_cpu = smp_processor_id(); 983 int this_cpu = smp_processor_id();
983 int cpu = task_cpu(task); 984 int cpu = task_cpu(task);
985 cpumask_var_t domain_mask;
984 986
985 if (task->rt.nr_cpus_allowed == 1) 987 if (task->rt.nr_cpus_allowed == 1)
986 return -1; /* No other targets possible */ 988 return -1; /* No other targets possible */
@@ -1013,19 +1015,25 @@ static int find_lowest_rq(struct task_struct *task)
1013 if (this_cpu == cpu) 1015 if (this_cpu == cpu)
1014 this_cpu = -1; /* Skip this_cpu opt if the same */ 1016 this_cpu = -1; /* Skip this_cpu opt if the same */
1015 1017
1016 for_each_domain(cpu, sd) { 1018 if (alloc_cpumask_var(&domain_mask, GFP_ATOMIC)) {
1017 if (sd->flags & SD_WAKE_AFFINE) { 1019 for_each_domain(cpu, sd) {
1018 cpumask_t domain_mask; 1020 if (sd->flags & SD_WAKE_AFFINE) {
1019 int best_cpu; 1021 int best_cpu;
1020 1022
1021 cpumask_and(&domain_mask, sched_domain_span(sd), 1023 cpumask_and(domain_mask,
1022 lowest_mask); 1024 sched_domain_span(sd),
1025 lowest_mask);
1023 1026
1024 best_cpu = pick_optimal_cpu(this_cpu, 1027 best_cpu = pick_optimal_cpu(this_cpu,
1025 &domain_mask); 1028 domain_mask);
1026 if (best_cpu != -1) 1029
1027 return best_cpu; 1030 if (best_cpu != -1) {
1031 free_cpumask_var(domain_mask);
1032 return best_cpu;
1033 }
1034 }
1028 } 1035 }
1036 free_cpumask_var(domain_mask);
1029 } 1037 }
1030 1038
1031 /* 1039 /*
diff --git a/kernel/softirq.c b/kernel/softirq.c
index 9041ea7948fe..bbf6d6496f05 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -21,6 +21,7 @@
21#include <linux/freezer.h> 21#include <linux/freezer.h>
22#include <linux/kthread.h> 22#include <linux/kthread.h>
23#include <linux/rcupdate.h> 23#include <linux/rcupdate.h>
24#include <linux/ftrace.h>
24#include <linux/smp.h> 25#include <linux/smp.h>
25#include <linux/tick.h> 26#include <linux/tick.h>
26 27
@@ -79,13 +80,23 @@ static void __local_bh_disable(unsigned long ip)
79 WARN_ON_ONCE(in_irq()); 80 WARN_ON_ONCE(in_irq());
80 81
81 raw_local_irq_save(flags); 82 raw_local_irq_save(flags);
82 add_preempt_count(SOFTIRQ_OFFSET); 83 /*
84 * The preempt tracer hooks into add_preempt_count and will break
85 * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
86 * is set and before current->softirq_enabled is cleared.
87 * We must manually increment preempt_count here and manually
88 * call the trace_preempt_off later.
89 */
90 preempt_count() += SOFTIRQ_OFFSET;
83 /* 91 /*
84 * Were softirqs turned off above: 92 * Were softirqs turned off above:
85 */ 93 */
86 if (softirq_count() == SOFTIRQ_OFFSET) 94 if (softirq_count() == SOFTIRQ_OFFSET)
87 trace_softirqs_off(ip); 95 trace_softirqs_off(ip);
88 raw_local_irq_restore(flags); 96 raw_local_irq_restore(flags);
97
98 if (preempt_count() == SOFTIRQ_OFFSET)
99 trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
89} 100}
90#else /* !CONFIG_TRACE_IRQFLAGS */ 101#else /* !CONFIG_TRACE_IRQFLAGS */
91static inline void __local_bh_disable(unsigned long ip) 102static inline void __local_bh_disable(unsigned long ip)
@@ -796,6 +807,11 @@ int __init __weak early_irq_init(void)
796 return 0; 807 return 0;
797} 808}
798 809
810int __init __weak arch_probe_nr_irqs(void)
811{
812 return 0;
813}
814
799int __init __weak arch_early_irq_init(void) 815int __init __weak arch_early_irq_init(void)
800{ 816{
801 return 0; 817 return 0;
diff --git a/kernel/stop_machine.c b/kernel/stop_machine.c
index 0cd415ee62a2..74541ca49536 100644
--- a/kernel/stop_machine.c
+++ b/kernel/stop_machine.c
@@ -170,7 +170,7 @@ int __stop_machine(int (*fn)(void *), void *data, const struct cpumask *cpus)
170 * doesn't hit this CPU until we're ready. */ 170 * doesn't hit this CPU until we're ready. */
171 get_cpu(); 171 get_cpu();
172 for_each_online_cpu(i) { 172 for_each_online_cpu(i) {
173 sm_work = percpu_ptr(stop_machine_work, i); 173 sm_work = per_cpu_ptr(stop_machine_work, i);
174 INIT_WORK(sm_work, stop_cpu); 174 INIT_WORK(sm_work, stop_cpu);
175 queue_work_on(i, stop_machine_wq, sm_work); 175 queue_work_on(i, stop_machine_wq, sm_work);
176 } 176 }
diff --git a/kernel/timer.c b/kernel/timer.c
index 13dd64fe143d..ef1c385bc572 100644
--- a/kernel/timer.c
+++ b/kernel/timer.c
@@ -491,14 +491,18 @@ static inline void debug_timer_free(struct timer_list *timer)
491 debug_object_free(timer, &timer_debug_descr); 491 debug_object_free(timer, &timer_debug_descr);
492} 492}
493 493
494static void __init_timer(struct timer_list *timer); 494static void __init_timer(struct timer_list *timer,
495 const char *name,
496 struct lock_class_key *key);
495 497
496void init_timer_on_stack(struct timer_list *timer) 498void init_timer_on_stack_key(struct timer_list *timer,
499 const char *name,
500 struct lock_class_key *key)
497{ 501{
498 debug_object_init_on_stack(timer, &timer_debug_descr); 502 debug_object_init_on_stack(timer, &timer_debug_descr);
499 __init_timer(timer); 503 __init_timer(timer, name, key);
500} 504}
501EXPORT_SYMBOL_GPL(init_timer_on_stack); 505EXPORT_SYMBOL_GPL(init_timer_on_stack_key);
502 506
503void destroy_timer_on_stack(struct timer_list *timer) 507void destroy_timer_on_stack(struct timer_list *timer)
504{ 508{
@@ -512,7 +516,9 @@ static inline void debug_timer_activate(struct timer_list *timer) { }
512static inline void debug_timer_deactivate(struct timer_list *timer) { } 516static inline void debug_timer_deactivate(struct timer_list *timer) { }
513#endif 517#endif
514 518
515static void __init_timer(struct timer_list *timer) 519static void __init_timer(struct timer_list *timer,
520 const char *name,
521 struct lock_class_key *key)
516{ 522{
517 timer->entry.next = NULL; 523 timer->entry.next = NULL;
518 timer->base = __raw_get_cpu_var(tvec_bases); 524 timer->base = __raw_get_cpu_var(tvec_bases);
@@ -521,6 +527,7 @@ static void __init_timer(struct timer_list *timer)
521 timer->start_pid = -1; 527 timer->start_pid = -1;
522 memset(timer->start_comm, 0, TASK_COMM_LEN); 528 memset(timer->start_comm, 0, TASK_COMM_LEN);
523#endif 529#endif
530 lockdep_init_map(&timer->lockdep_map, name, key, 0);
524} 531}
525 532
526/** 533/**
@@ -530,19 +537,23 @@ static void __init_timer(struct timer_list *timer)
530 * init_timer() must be done to a timer prior calling *any* of the 537 * init_timer() must be done to a timer prior calling *any* of the
531 * other timer functions. 538 * other timer functions.
532 */ 539 */
533void init_timer(struct timer_list *timer) 540void init_timer_key(struct timer_list *timer,
541 const char *name,
542 struct lock_class_key *key)
534{ 543{
535 debug_timer_init(timer); 544 debug_timer_init(timer);
536 __init_timer(timer); 545 __init_timer(timer, name, key);
537} 546}
538EXPORT_SYMBOL(init_timer); 547EXPORT_SYMBOL(init_timer_key);
539 548
540void init_timer_deferrable(struct timer_list *timer) 549void init_timer_deferrable_key(struct timer_list *timer,
550 const char *name,
551 struct lock_class_key *key)
541{ 552{
542 init_timer(timer); 553 init_timer_key(timer, name, key);
543 timer_set_deferrable(timer); 554 timer_set_deferrable(timer);
544} 555}
545EXPORT_SYMBOL(init_timer_deferrable); 556EXPORT_SYMBOL(init_timer_deferrable_key);
546 557
547static inline void detach_timer(struct timer_list *timer, 558static inline void detach_timer(struct timer_list *timer,
548 int clear_pending) 559 int clear_pending)
@@ -789,6 +800,15 @@ EXPORT_SYMBOL(try_to_del_timer_sync);
789 */ 800 */
790int del_timer_sync(struct timer_list *timer) 801int del_timer_sync(struct timer_list *timer)
791{ 802{
803#ifdef CONFIG_LOCKDEP
804 unsigned long flags;
805
806 local_irq_save(flags);
807 lock_map_acquire(&timer->lockdep_map);
808 lock_map_release(&timer->lockdep_map);
809 local_irq_restore(flags);
810#endif
811
792 for (;;) { 812 for (;;) {
793 int ret = try_to_del_timer_sync(timer); 813 int ret = try_to_del_timer_sync(timer);
794 if (ret >= 0) 814 if (ret >= 0)
@@ -861,10 +881,36 @@ static inline void __run_timers(struct tvec_base *base)
861 881
862 set_running_timer(base, timer); 882 set_running_timer(base, timer);
863 detach_timer(timer, 1); 883 detach_timer(timer, 1);
884
864 spin_unlock_irq(&base->lock); 885 spin_unlock_irq(&base->lock);
865 { 886 {
866 int preempt_count = preempt_count(); 887 int preempt_count = preempt_count();
888
889#ifdef CONFIG_LOCKDEP
890 /*
891 * It is permissible to free the timer from
892 * inside the function that is called from
893 * it, this we need to take into account for
894 * lockdep too. To avoid bogus "held lock
895 * freed" warnings as well as problems when
896 * looking into timer->lockdep_map, make a
897 * copy and use that here.
898 */
899 struct lockdep_map lockdep_map =
900 timer->lockdep_map;
901#endif
902 /*
903 * Couple the lock chain with the lock chain at
904 * del_timer_sync() by acquiring the lock_map
905 * around the fn() call here and in
906 * del_timer_sync().
907 */
908 lock_map_acquire(&lockdep_map);
909
867 fn(data); 910 fn(data);
911
912 lock_map_release(&lockdep_map);
913
868 if (preempt_count != preempt_count()) { 914 if (preempt_count != preempt_count()) {
869 printk(KERN_ERR "huh, entered %p " 915 printk(KERN_ERR "huh, entered %p "
870 "with preempt_count %08x, exited" 916 "with preempt_count %08x, exited"
diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig
index 34e707e5ab87..8e4a2a61cd75 100644
--- a/kernel/trace/Kconfig
+++ b/kernel/trace/Kconfig
@@ -9,6 +9,9 @@ config USER_STACKTRACE_SUPPORT
9config NOP_TRACER 9config NOP_TRACER
10 bool 10 bool
11 11
12config HAVE_FTRACE_NMI_ENTER
13 bool
14
12config HAVE_FUNCTION_TRACER 15config HAVE_FUNCTION_TRACER
13 bool 16 bool
14 17
@@ -37,6 +40,11 @@ config TRACER_MAX_TRACE
37config RING_BUFFER 40config RING_BUFFER
38 bool 41 bool
39 42
43config FTRACE_NMI_ENTER
44 bool
45 depends on HAVE_FTRACE_NMI_ENTER
46 default y
47
40config TRACING 48config TRACING
41 bool 49 bool
42 select DEBUG_FS 50 select DEBUG_FS
@@ -44,13 +52,25 @@ config TRACING
44 select STACKTRACE if STACKTRACE_SUPPORT 52 select STACKTRACE if STACKTRACE_SUPPORT
45 select TRACEPOINTS 53 select TRACEPOINTS
46 select NOP_TRACER 54 select NOP_TRACER
55 select BINARY_PRINTF
56
57#
58# Minimum requirements an architecture has to meet for us to
59# be able to offer generic tracing facilities:
60#
61config TRACING_SUPPORT
62 bool
63 depends on TRACE_IRQFLAGS_SUPPORT
64 depends on STACKTRACE_SUPPORT
65 default y
66
67if TRACING_SUPPORT
47 68
48menu "Tracers" 69menu "Tracers"
49 70
50config FUNCTION_TRACER 71config FUNCTION_TRACER
51 bool "Kernel Function Tracer" 72 bool "Kernel Function Tracer"
52 depends on HAVE_FUNCTION_TRACER 73 depends on HAVE_FUNCTION_TRACER
53 depends on DEBUG_KERNEL
54 select FRAME_POINTER 74 select FRAME_POINTER
55 select KALLSYMS 75 select KALLSYMS
56 select TRACING 76 select TRACING
@@ -83,7 +103,6 @@ config IRQSOFF_TRACER
83 default n 103 default n
84 depends on TRACE_IRQFLAGS_SUPPORT 104 depends on TRACE_IRQFLAGS_SUPPORT
85 depends on GENERIC_TIME 105 depends on GENERIC_TIME
86 depends on DEBUG_KERNEL
87 select TRACE_IRQFLAGS 106 select TRACE_IRQFLAGS
88 select TRACING 107 select TRACING
89 select TRACER_MAX_TRACE 108 select TRACER_MAX_TRACE
@@ -106,7 +125,6 @@ config PREEMPT_TRACER
106 default n 125 default n
107 depends on GENERIC_TIME 126 depends on GENERIC_TIME
108 depends on PREEMPT 127 depends on PREEMPT
109 depends on DEBUG_KERNEL
110 select TRACING 128 select TRACING
111 select TRACER_MAX_TRACE 129 select TRACER_MAX_TRACE
112 help 130 help
@@ -127,13 +145,13 @@ config SYSPROF_TRACER
127 bool "Sysprof Tracer" 145 bool "Sysprof Tracer"
128 depends on X86 146 depends on X86
129 select TRACING 147 select TRACING
148 select CONTEXT_SWITCH_TRACER
130 help 149 help
131 This tracer provides the trace needed by the 'Sysprof' userspace 150 This tracer provides the trace needed by the 'Sysprof' userspace
132 tool. 151 tool.
133 152
134config SCHED_TRACER 153config SCHED_TRACER
135 bool "Scheduling Latency Tracer" 154 bool "Scheduling Latency Tracer"
136 depends on DEBUG_KERNEL
137 select TRACING 155 select TRACING
138 select CONTEXT_SWITCH_TRACER 156 select CONTEXT_SWITCH_TRACER
139 select TRACER_MAX_TRACE 157 select TRACER_MAX_TRACE
@@ -143,16 +161,22 @@ config SCHED_TRACER
143 161
144config CONTEXT_SWITCH_TRACER 162config CONTEXT_SWITCH_TRACER
145 bool "Trace process context switches" 163 bool "Trace process context switches"
146 depends on DEBUG_KERNEL
147 select TRACING 164 select TRACING
148 select MARKERS 165 select MARKERS
149 help 166 help
150 This tracer gets called from the context switch and records 167 This tracer gets called from the context switch and records
151 all switching of tasks. 168 all switching of tasks.
152 169
170config EVENT_TRACER
171 bool "Trace various events in the kernel"
172 select TRACING
173 help
174 This tracer hooks to various trace points in the kernel
175 allowing the user to pick and choose which trace point they
176 want to trace.
177
153config BOOT_TRACER 178config BOOT_TRACER
154 bool "Trace boot initcalls" 179 bool "Trace boot initcalls"
155 depends on DEBUG_KERNEL
156 select TRACING 180 select TRACING
157 select CONTEXT_SWITCH_TRACER 181 select CONTEXT_SWITCH_TRACER
158 help 182 help
@@ -165,13 +189,11 @@ config BOOT_TRACER
165 representation of the delays during initcalls - but the raw 189 representation of the delays during initcalls - but the raw
166 /debug/tracing/trace text output is readable too. 190 /debug/tracing/trace text output is readable too.
167 191
168 ( Note that tracing self tests can't be enabled if this tracer is 192 You must pass in ftrace=initcall to the kernel command line
169 selected, because the self-tests are an initcall as well and that 193 to enable this on bootup.
170 would invalidate the boot trace. )
171 194
172config TRACE_BRANCH_PROFILING 195config TRACE_BRANCH_PROFILING
173 bool "Trace likely/unlikely profiler" 196 bool "Trace likely/unlikely profiler"
174 depends on DEBUG_KERNEL
175 select TRACING 197 select TRACING
176 help 198 help
177 This tracer profiles all the the likely and unlikely macros 199 This tracer profiles all the the likely and unlikely macros
@@ -224,7 +246,6 @@ config BRANCH_TRACER
224 246
225config POWER_TRACER 247config POWER_TRACER
226 bool "Trace power consumption behavior" 248 bool "Trace power consumption behavior"
227 depends on DEBUG_KERNEL
228 depends on X86 249 depends on X86
229 select TRACING 250 select TRACING
230 help 251 help
@@ -236,7 +257,6 @@ config POWER_TRACER
236config STACK_TRACER 257config STACK_TRACER
237 bool "Trace max stack" 258 bool "Trace max stack"
238 depends on HAVE_FUNCTION_TRACER 259 depends on HAVE_FUNCTION_TRACER
239 depends on DEBUG_KERNEL
240 select FUNCTION_TRACER 260 select FUNCTION_TRACER
241 select STACKTRACE 261 select STACKTRACE
242 select KALLSYMS 262 select KALLSYMS
@@ -266,11 +286,66 @@ config HW_BRANCH_TRACER
266 This tracer records all branches on the system in a circular 286 This tracer records all branches on the system in a circular
267 buffer giving access to the last N branches for each cpu. 287 buffer giving access to the last N branches for each cpu.
268 288
289config KMEMTRACE
290 bool "Trace SLAB allocations"
291 select TRACING
292 help
293 kmemtrace provides tracing for slab allocator functions, such as
294 kmalloc, kfree, kmem_cache_alloc, kmem_cache_free etc.. Collected
295 data is then fed to the userspace application in order to analyse
296 allocation hotspots, internal fragmentation and so on, making it
297 possible to see how well an allocator performs, as well as debug
298 and profile kernel code.
299
300 This requires an userspace application to use. See
301 Documentation/vm/kmemtrace.txt for more information.
302
303 Saying Y will make the kernel somewhat larger and slower. However,
304 if you disable kmemtrace at run-time or boot-time, the performance
305 impact is minimal (depending on the arch the kernel is built for).
306
307 If unsure, say N.
308
309config WORKQUEUE_TRACER
310 bool "Trace workqueues"
311 select TRACING
312 help
313 The workqueue tracer provides some statistical informations
314 about each cpu workqueue thread such as the number of the
315 works inserted and executed since their creation. It can help
316 to evaluate the amount of work each of them have to perform.
317 For example it can help a developer to decide whether he should
318 choose a per cpu workqueue instead of a singlethreaded one.
319
320config BLK_DEV_IO_TRACE
321 bool "Support for tracing block io actions"
322 depends on SYSFS
323 depends on BLOCK
324 select RELAY
325 select DEBUG_FS
326 select TRACEPOINTS
327 select TRACING
328 select STACKTRACE
329 help
330 Say Y here if you want to be able to trace the block layer actions
331 on a given queue. Tracing allows you to see any traffic happening
332 on a block device queue. For more information (and the userspace
333 support tools needed), fetch the blktrace tools from:
334
335 git://git.kernel.dk/blktrace.git
336
337 Tracing also is possible using the ftrace interface, e.g.:
338
339 echo 1 > /sys/block/sda/sda1/trace/enable
340 echo blk > /sys/kernel/debug/tracing/current_tracer
341 cat /sys/kernel/debug/tracing/trace_pipe
342
343 If unsure, say N.
344
269config DYNAMIC_FTRACE 345config DYNAMIC_FTRACE
270 bool "enable/disable ftrace tracepoints dynamically" 346 bool "enable/disable ftrace tracepoints dynamically"
271 depends on FUNCTION_TRACER 347 depends on FUNCTION_TRACER
272 depends on HAVE_DYNAMIC_FTRACE 348 depends on HAVE_DYNAMIC_FTRACE
273 depends on DEBUG_KERNEL
274 default y 349 default y
275 help 350 help
276 This option will modify all the calls to ftrace dynamically 351 This option will modify all the calls to ftrace dynamically
@@ -296,7 +371,7 @@ config FTRACE_SELFTEST
296 371
297config FTRACE_STARTUP_TEST 372config FTRACE_STARTUP_TEST
298 bool "Perform a startup test on ftrace" 373 bool "Perform a startup test on ftrace"
299 depends on TRACING && DEBUG_KERNEL && !BOOT_TRACER 374 depends on TRACING
300 select FTRACE_SELFTEST 375 select FTRACE_SELFTEST
301 help 376 help
302 This option performs a series of startup tests on ftrace. On bootup 377 This option performs a series of startup tests on ftrace. On bootup
@@ -306,7 +381,7 @@ config FTRACE_STARTUP_TEST
306 381
307config MMIOTRACE 382config MMIOTRACE
308 bool "Memory mapped IO tracing" 383 bool "Memory mapped IO tracing"
309 depends on HAVE_MMIOTRACE_SUPPORT && DEBUG_KERNEL && PCI 384 depends on HAVE_MMIOTRACE_SUPPORT && PCI
310 select TRACING 385 select TRACING
311 help 386 help
312 Mmiotrace traces Memory Mapped I/O access and is meant for 387 Mmiotrace traces Memory Mapped I/O access and is meant for
@@ -328,3 +403,6 @@ config MMIOTRACE_TEST
328 Say N, unless you absolutely know what you are doing. 403 Say N, unless you absolutely know what you are doing.
329 404
330endmenu 405endmenu
406
407endif # TRACING_SUPPORT
408
diff --git a/kernel/trace/Makefile b/kernel/trace/Makefile
index 349d5a93653f..c7a2943796eb 100644
--- a/kernel/trace/Makefile
+++ b/kernel/trace/Makefile
@@ -19,6 +19,10 @@ obj-$(CONFIG_FUNCTION_TRACER) += libftrace.o
19obj-$(CONFIG_RING_BUFFER) += ring_buffer.o 19obj-$(CONFIG_RING_BUFFER) += ring_buffer.o
20 20
21obj-$(CONFIG_TRACING) += trace.o 21obj-$(CONFIG_TRACING) += trace.o
22obj-$(CONFIG_TRACING) += trace_clock.o
23obj-$(CONFIG_TRACING) += trace_output.o
24obj-$(CONFIG_TRACING) += trace_stat.o
25obj-$(CONFIG_TRACING) += trace_printk.o
22obj-$(CONFIG_CONTEXT_SWITCH_TRACER) += trace_sched_switch.o 26obj-$(CONFIG_CONTEXT_SWITCH_TRACER) += trace_sched_switch.o
23obj-$(CONFIG_SYSPROF_TRACER) += trace_sysprof.o 27obj-$(CONFIG_SYSPROF_TRACER) += trace_sysprof.o
24obj-$(CONFIG_FUNCTION_TRACER) += trace_functions.o 28obj-$(CONFIG_FUNCTION_TRACER) += trace_functions.o
@@ -33,5 +37,11 @@ obj-$(CONFIG_FUNCTION_GRAPH_TRACER) += trace_functions_graph.o
33obj-$(CONFIG_TRACE_BRANCH_PROFILING) += trace_branch.o 37obj-$(CONFIG_TRACE_BRANCH_PROFILING) += trace_branch.o
34obj-$(CONFIG_HW_BRANCH_TRACER) += trace_hw_branches.o 38obj-$(CONFIG_HW_BRANCH_TRACER) += trace_hw_branches.o
35obj-$(CONFIG_POWER_TRACER) += trace_power.o 39obj-$(CONFIG_POWER_TRACER) += trace_power.o
40obj-$(CONFIG_KMEMTRACE) += kmemtrace.o
41obj-$(CONFIG_WORKQUEUE_TRACER) += trace_workqueue.o
42obj-$(CONFIG_BLK_DEV_IO_TRACE) += blktrace.o
43obj-$(CONFIG_EVENT_TRACER) += trace_events.o
44obj-$(CONFIG_EVENT_TRACER) += events.o
45obj-$(CONFIG_EVENT_TRACER) += trace_export.o
36 46
37libftrace-y := ftrace.o 47libftrace-y := ftrace.o
diff --git a/kernel/trace/blktrace.c b/kernel/trace/blktrace.c
new file mode 100644
index 000000000000..d24a10b8411a
--- /dev/null
+++ b/kernel/trace/blktrace.c
@@ -0,0 +1,1537 @@
1/*
2 * Copyright (C) 2006 Jens Axboe <axboe@kernel.dk>
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * along with this program; if not, write to the Free Software
15 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
16 *
17 */
18#include <linux/kernel.h>
19#include <linux/blkdev.h>
20#include <linux/blktrace_api.h>
21#include <linux/percpu.h>
22#include <linux/init.h>
23#include <linux/mutex.h>
24#include <linux/debugfs.h>
25#include <linux/time.h>
26#include <trace/block.h>
27#include <linux/uaccess.h>
28#include "trace_output.h"
29
30static unsigned int blktrace_seq __read_mostly = 1;
31
32static struct trace_array *blk_tr;
33static int __read_mostly blk_tracer_enabled;
34
35/* Select an alternative, minimalistic output than the original one */
36#define TRACE_BLK_OPT_CLASSIC 0x1
37
38static struct tracer_opt blk_tracer_opts[] = {
39 /* Default disable the minimalistic output */
40 { TRACER_OPT(blk_classic, TRACE_BLK_OPT_CLASSIC) },
41 { }
42};
43
44static struct tracer_flags blk_tracer_flags = {
45 .val = 0,
46 .opts = blk_tracer_opts,
47};
48
49/* Global reference count of probes */
50static DEFINE_MUTEX(blk_probe_mutex);
51static atomic_t blk_probes_ref = ATOMIC_INIT(0);
52
53static int blk_register_tracepoints(void);
54static void blk_unregister_tracepoints(void);
55
56/*
57 * Send out a notify message.
58 */
59static void trace_note(struct blk_trace *bt, pid_t pid, int action,
60 const void *data, size_t len)
61{
62 struct blk_io_trace *t;
63
64 if (!bt->rchan)
65 return;
66
67 t = relay_reserve(bt->rchan, sizeof(*t) + len);
68 if (t) {
69 const int cpu = smp_processor_id();
70
71 t->magic = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE_VERSION;
72 t->time = ktime_to_ns(ktime_get());
73 t->device = bt->dev;
74 t->action = action;
75 t->pid = pid;
76 t->cpu = cpu;
77 t->pdu_len = len;
78 memcpy((void *) t + sizeof(*t), data, len);
79 }
80}
81
82/*
83 * Send out a notify for this process, if we haven't done so since a trace
84 * started
85 */
86static void trace_note_tsk(struct blk_trace *bt, struct task_struct *tsk)
87{
88 tsk->btrace_seq = blktrace_seq;
89 trace_note(bt, tsk->pid, BLK_TN_PROCESS, tsk->comm, sizeof(tsk->comm));
90}
91
92static void trace_note_time(struct blk_trace *bt)
93{
94 struct timespec now;
95 unsigned long flags;
96 u32 words[2];
97
98 getnstimeofday(&now);
99 words[0] = now.tv_sec;
100 words[1] = now.tv_nsec;
101
102 local_irq_save(flags);
103 trace_note(bt, 0, BLK_TN_TIMESTAMP, words, sizeof(words));
104 local_irq_restore(flags);
105}
106
107void __trace_note_message(struct blk_trace *bt, const char *fmt, ...)
108{
109 int n;
110 va_list args;
111 unsigned long flags;
112 char *buf;
113
114 if (blk_tr) {
115 va_start(args, fmt);
116 ftrace_vprintk(fmt, args);
117 va_end(args);
118 return;
119 }
120
121 if (!bt->msg_data)
122 return;
123
124 local_irq_save(flags);
125 buf = per_cpu_ptr(bt->msg_data, smp_processor_id());
126 va_start(args, fmt);
127 n = vscnprintf(buf, BLK_TN_MAX_MSG, fmt, args);
128 va_end(args);
129
130 trace_note(bt, 0, BLK_TN_MESSAGE, buf, n);
131 local_irq_restore(flags);
132}
133EXPORT_SYMBOL_GPL(__trace_note_message);
134
135static int act_log_check(struct blk_trace *bt, u32 what, sector_t sector,
136 pid_t pid)
137{
138 if (((bt->act_mask << BLK_TC_SHIFT) & what) == 0)
139 return 1;
140 if (sector < bt->start_lba || sector > bt->end_lba)
141 return 1;
142 if (bt->pid && pid != bt->pid)
143 return 1;
144
145 return 0;
146}
147
148/*
149 * Data direction bit lookup
150 */
151static u32 ddir_act[2] __read_mostly = { BLK_TC_ACT(BLK_TC_READ),
152 BLK_TC_ACT(BLK_TC_WRITE) };
153
154/* The ilog2() calls fall out because they're constant */
155#define MASK_TC_BIT(rw, __name) ((rw & (1 << BIO_RW_ ## __name)) << \
156 (ilog2(BLK_TC_ ## __name) + BLK_TC_SHIFT - BIO_RW_ ## __name))
157
158/*
159 * The worker for the various blk_add_trace*() types. Fills out a
160 * blk_io_trace structure and places it in a per-cpu subbuffer.
161 */
162static void __blk_add_trace(struct blk_trace *bt, sector_t sector, int bytes,
163 int rw, u32 what, int error, int pdu_len, void *pdu_data)
164{
165 struct task_struct *tsk = current;
166 struct ring_buffer_event *event = NULL;
167 struct blk_io_trace *t;
168 unsigned long flags = 0;
169 unsigned long *sequence;
170 pid_t pid;
171 int cpu, pc = 0;
172
173 if (unlikely(bt->trace_state != Blktrace_running ||
174 !blk_tracer_enabled))
175 return;
176
177 what |= ddir_act[rw & WRITE];
178 what |= MASK_TC_BIT(rw, BARRIER);
179 what |= MASK_TC_BIT(rw, SYNCIO);
180 what |= MASK_TC_BIT(rw, AHEAD);
181 what |= MASK_TC_BIT(rw, META);
182 what |= MASK_TC_BIT(rw, DISCARD);
183
184 pid = tsk->pid;
185 if (unlikely(act_log_check(bt, what, sector, pid)))
186 return;
187 cpu = raw_smp_processor_id();
188
189 if (blk_tr) {
190 tracing_record_cmdline(current);
191
192 pc = preempt_count();
193 event = trace_buffer_lock_reserve(blk_tr, TRACE_BLK,
194 sizeof(*t) + pdu_len,
195 0, pc);
196 if (!event)
197 return;
198 t = ring_buffer_event_data(event);
199 goto record_it;
200 }
201
202 /*
203 * A word about the locking here - we disable interrupts to reserve
204 * some space in the relay per-cpu buffer, to prevent an irq
205 * from coming in and stepping on our toes.
206 */
207 local_irq_save(flags);
208
209 if (unlikely(tsk->btrace_seq != blktrace_seq))
210 trace_note_tsk(bt, tsk);
211
212 t = relay_reserve(bt->rchan, sizeof(*t) + pdu_len);
213 if (t) {
214 sequence = per_cpu_ptr(bt->sequence, cpu);
215
216 t->magic = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE_VERSION;
217 t->sequence = ++(*sequence);
218 t->time = ktime_to_ns(ktime_get());
219record_it:
220 /*
221 * These two are not needed in ftrace as they are in the
222 * generic trace_entry, filled by tracing_generic_entry_update,
223 * but for the trace_event->bin() synthesizer benefit we do it
224 * here too.
225 */
226 t->cpu = cpu;
227 t->pid = pid;
228
229 t->sector = sector;
230 t->bytes = bytes;
231 t->action = what;
232 t->device = bt->dev;
233 t->error = error;
234 t->pdu_len = pdu_len;
235
236 if (pdu_len)
237 memcpy((void *) t + sizeof(*t), pdu_data, pdu_len);
238
239 if (blk_tr) {
240 trace_buffer_unlock_commit(blk_tr, event, 0, pc);
241 return;
242 }
243 }
244
245 local_irq_restore(flags);
246}
247
248static struct dentry *blk_tree_root;
249static DEFINE_MUTEX(blk_tree_mutex);
250
251static void blk_trace_cleanup(struct blk_trace *bt)
252{
253 debugfs_remove(bt->msg_file);
254 debugfs_remove(bt->dropped_file);
255 relay_close(bt->rchan);
256 free_percpu(bt->sequence);
257 free_percpu(bt->msg_data);
258 kfree(bt);
259 mutex_lock(&blk_probe_mutex);
260 if (atomic_dec_and_test(&blk_probes_ref))
261 blk_unregister_tracepoints();
262 mutex_unlock(&blk_probe_mutex);
263}
264
265int blk_trace_remove(struct request_queue *q)
266{
267 struct blk_trace *bt;
268
269 bt = xchg(&q->blk_trace, NULL);
270 if (!bt)
271 return -EINVAL;
272
273 if (bt->trace_state == Blktrace_setup ||
274 bt->trace_state == Blktrace_stopped)
275 blk_trace_cleanup(bt);
276
277 return 0;
278}
279EXPORT_SYMBOL_GPL(blk_trace_remove);
280
281static int blk_dropped_open(struct inode *inode, struct file *filp)
282{
283 filp->private_data = inode->i_private;
284
285 return 0;
286}
287
288static ssize_t blk_dropped_read(struct file *filp, char __user *buffer,
289 size_t count, loff_t *ppos)
290{
291 struct blk_trace *bt = filp->private_data;
292 char buf[16];
293
294 snprintf(buf, sizeof(buf), "%u\n", atomic_read(&bt->dropped));
295
296 return simple_read_from_buffer(buffer, count, ppos, buf, strlen(buf));
297}
298
299static const struct file_operations blk_dropped_fops = {
300 .owner = THIS_MODULE,
301 .open = blk_dropped_open,
302 .read = blk_dropped_read,
303};
304
305static int blk_msg_open(struct inode *inode, struct file *filp)
306{
307 filp->private_data = inode->i_private;
308
309 return 0;
310}
311
312static ssize_t blk_msg_write(struct file *filp, const char __user *buffer,
313 size_t count, loff_t *ppos)
314{
315 char *msg;
316 struct blk_trace *bt;
317
318 if (count > BLK_TN_MAX_MSG)
319 return -EINVAL;
320
321 msg = kmalloc(count, GFP_KERNEL);
322 if (msg == NULL)
323 return -ENOMEM;
324
325 if (copy_from_user(msg, buffer, count)) {
326 kfree(msg);
327 return -EFAULT;
328 }
329
330 bt = filp->private_data;
331 __trace_note_message(bt, "%s", msg);
332 kfree(msg);
333
334 return count;
335}
336
337static const struct file_operations blk_msg_fops = {
338 .owner = THIS_MODULE,
339 .open = blk_msg_open,
340 .write = blk_msg_write,
341};
342
343/*
344 * Keep track of how many times we encountered a full subbuffer, to aid
345 * the user space app in telling how many lost events there were.
346 */
347static int blk_subbuf_start_callback(struct rchan_buf *buf, void *subbuf,
348 void *prev_subbuf, size_t prev_padding)
349{
350 struct blk_trace *bt;
351
352 if (!relay_buf_full(buf))
353 return 1;
354
355 bt = buf->chan->private_data;
356 atomic_inc(&bt->dropped);
357 return 0;
358}
359
360static int blk_remove_buf_file_callback(struct dentry *dentry)
361{
362 struct dentry *parent = dentry->d_parent;
363 debugfs_remove(dentry);
364
365 /*
366 * this will fail for all but the last file, but that is ok. what we
367 * care about is the top level buts->name directory going away, when
368 * the last trace file is gone. Then we don't have to rmdir() that
369 * manually on trace stop, so it nicely solves the issue with
370 * force killing of running traces.
371 */
372
373 debugfs_remove(parent);
374 return 0;
375}
376
377static struct dentry *blk_create_buf_file_callback(const char *filename,
378 struct dentry *parent,
379 int mode,
380 struct rchan_buf *buf,
381 int *is_global)
382{
383 return debugfs_create_file(filename, mode, parent, buf,
384 &relay_file_operations);
385}
386
387static struct rchan_callbacks blk_relay_callbacks = {
388 .subbuf_start = blk_subbuf_start_callback,
389 .create_buf_file = blk_create_buf_file_callback,
390 .remove_buf_file = blk_remove_buf_file_callback,
391};
392
393/*
394 * Setup everything required to start tracing
395 */
396int do_blk_trace_setup(struct request_queue *q, char *name, dev_t dev,
397 struct blk_user_trace_setup *buts)
398{
399 struct blk_trace *old_bt, *bt = NULL;
400 struct dentry *dir = NULL;
401 int ret, i;
402
403 if (!buts->buf_size || !buts->buf_nr)
404 return -EINVAL;
405
406 strncpy(buts->name, name, BLKTRACE_BDEV_SIZE);
407 buts->name[BLKTRACE_BDEV_SIZE - 1] = '\0';
408
409 /*
410 * some device names have larger paths - convert the slashes
411 * to underscores for this to work as expected
412 */
413 for (i = 0; i < strlen(buts->name); i++)
414 if (buts->name[i] == '/')
415 buts->name[i] = '_';
416
417 ret = -ENOMEM;
418 bt = kzalloc(sizeof(*bt), GFP_KERNEL);
419 if (!bt)
420 goto err;
421
422 bt->sequence = alloc_percpu(unsigned long);
423 if (!bt->sequence)
424 goto err;
425
426 bt->msg_data = __alloc_percpu(BLK_TN_MAX_MSG, __alignof__(char));
427 if (!bt->msg_data)
428 goto err;
429
430 ret = -ENOENT;
431
432 if (!blk_tree_root) {
433 blk_tree_root = debugfs_create_dir("block", NULL);
434 if (!blk_tree_root)
435 return -ENOMEM;
436 }
437
438 dir = debugfs_create_dir(buts->name, blk_tree_root);
439
440 if (!dir)
441 goto err;
442
443 bt->dir = dir;
444 bt->dev = dev;
445 atomic_set(&bt->dropped, 0);
446
447 ret = -EIO;
448 bt->dropped_file = debugfs_create_file("dropped", 0444, dir, bt,
449 &blk_dropped_fops);
450 if (!bt->dropped_file)
451 goto err;
452
453 bt->msg_file = debugfs_create_file("msg", 0222, dir, bt, &blk_msg_fops);
454 if (!bt->msg_file)
455 goto err;
456
457 bt->rchan = relay_open("trace", dir, buts->buf_size,
458 buts->buf_nr, &blk_relay_callbacks, bt);
459 if (!bt->rchan)
460 goto err;
461
462 bt->act_mask = buts->act_mask;
463 if (!bt->act_mask)
464 bt->act_mask = (u16) -1;
465
466 bt->start_lba = buts->start_lba;
467 bt->end_lba = buts->end_lba;
468 if (!bt->end_lba)
469 bt->end_lba = -1ULL;
470
471 bt->pid = buts->pid;
472 bt->trace_state = Blktrace_setup;
473
474 mutex_lock(&blk_probe_mutex);
475 if (atomic_add_return(1, &blk_probes_ref) == 1) {
476 ret = blk_register_tracepoints();
477 if (ret)
478 goto probe_err;
479 }
480 mutex_unlock(&blk_probe_mutex);
481
482 ret = -EBUSY;
483 old_bt = xchg(&q->blk_trace, bt);
484 if (old_bt) {
485 (void) xchg(&q->blk_trace, old_bt);
486 goto err;
487 }
488
489 return 0;
490probe_err:
491 atomic_dec(&blk_probes_ref);
492 mutex_unlock(&blk_probe_mutex);
493err:
494 if (bt) {
495 if (bt->msg_file)
496 debugfs_remove(bt->msg_file);
497 if (bt->dropped_file)
498 debugfs_remove(bt->dropped_file);
499 free_percpu(bt->sequence);
500 free_percpu(bt->msg_data);
501 if (bt->rchan)
502 relay_close(bt->rchan);
503 kfree(bt);
504 }
505 return ret;
506}
507
508int blk_trace_setup(struct request_queue *q, char *name, dev_t dev,
509 char __user *arg)
510{
511 struct blk_user_trace_setup buts;
512 int ret;
513
514 ret = copy_from_user(&buts, arg, sizeof(buts));
515 if (ret)
516 return -EFAULT;
517
518 ret = do_blk_trace_setup(q, name, dev, &buts);
519 if (ret)
520 return ret;
521
522 if (copy_to_user(arg, &buts, sizeof(buts)))
523 return -EFAULT;
524
525 return 0;
526}
527EXPORT_SYMBOL_GPL(blk_trace_setup);
528
529int blk_trace_startstop(struct request_queue *q, int start)
530{
531 int ret;
532 struct blk_trace *bt = q->blk_trace;
533
534 if (bt == NULL)
535 return -EINVAL;
536
537 /*
538 * For starting a trace, we can transition from a setup or stopped
539 * trace. For stopping a trace, the state must be running
540 */
541 ret = -EINVAL;
542 if (start) {
543 if (bt->trace_state == Blktrace_setup ||
544 bt->trace_state == Blktrace_stopped) {
545 blktrace_seq++;
546 smp_mb();
547 bt->trace_state = Blktrace_running;
548
549 trace_note_time(bt);
550 ret = 0;
551 }
552 } else {
553 if (bt->trace_state == Blktrace_running) {
554 bt->trace_state = Blktrace_stopped;
555 relay_flush(bt->rchan);
556 ret = 0;
557 }
558 }
559
560 return ret;
561}
562EXPORT_SYMBOL_GPL(blk_trace_startstop);
563
564/**
565 * blk_trace_ioctl: - handle the ioctls associated with tracing
566 * @bdev: the block device
567 * @cmd: the ioctl cmd
568 * @arg: the argument data, if any
569 *
570 **/
571int blk_trace_ioctl(struct block_device *bdev, unsigned cmd, char __user *arg)
572{
573 struct request_queue *q;
574 int ret, start = 0;
575 char b[BDEVNAME_SIZE];
576
577 q = bdev_get_queue(bdev);
578 if (!q)
579 return -ENXIO;
580
581 mutex_lock(&bdev->bd_mutex);
582
583 switch (cmd) {
584 case BLKTRACESETUP:
585 bdevname(bdev, b);
586 ret = blk_trace_setup(q, b, bdev->bd_dev, arg);
587 break;
588 case BLKTRACESTART:
589 start = 1;
590 case BLKTRACESTOP:
591 ret = blk_trace_startstop(q, start);
592 break;
593 case BLKTRACETEARDOWN:
594 ret = blk_trace_remove(q);
595 break;
596 default:
597 ret = -ENOTTY;
598 break;
599 }
600
601 mutex_unlock(&bdev->bd_mutex);
602 return ret;
603}
604
605/**
606 * blk_trace_shutdown: - stop and cleanup trace structures
607 * @q: the request queue associated with the device
608 *
609 **/
610void blk_trace_shutdown(struct request_queue *q)
611{
612 if (q->blk_trace) {
613 blk_trace_startstop(q, 0);
614 blk_trace_remove(q);
615 }
616}
617
618/*
619 * blktrace probes
620 */
621
622/**
623 * blk_add_trace_rq - Add a trace for a request oriented action
624 * @q: queue the io is for
625 * @rq: the source request
626 * @what: the action
627 *
628 * Description:
629 * Records an action against a request. Will log the bio offset + size.
630 *
631 **/
632static void blk_add_trace_rq(struct request_queue *q, struct request *rq,
633 u32 what)
634{
635 struct blk_trace *bt = q->blk_trace;
636 int rw = rq->cmd_flags & 0x03;
637
638 if (likely(!bt))
639 return;
640
641 if (blk_discard_rq(rq))
642 rw |= (1 << BIO_RW_DISCARD);
643
644 if (blk_pc_request(rq)) {
645 what |= BLK_TC_ACT(BLK_TC_PC);
646 __blk_add_trace(bt, 0, rq->data_len, rw, what, rq->errors,
647 sizeof(rq->cmd), rq->cmd);
648 } else {
649 what |= BLK_TC_ACT(BLK_TC_FS);
650 __blk_add_trace(bt, rq->hard_sector, rq->hard_nr_sectors << 9,
651 rw, what, rq->errors, 0, NULL);
652 }
653}
654
655static void blk_add_trace_rq_abort(struct request_queue *q, struct request *rq)
656{
657 blk_add_trace_rq(q, rq, BLK_TA_ABORT);
658}
659
660static void blk_add_trace_rq_insert(struct request_queue *q, struct request *rq)
661{
662 blk_add_trace_rq(q, rq, BLK_TA_INSERT);
663}
664
665static void blk_add_trace_rq_issue(struct request_queue *q, struct request *rq)
666{
667 blk_add_trace_rq(q, rq, BLK_TA_ISSUE);
668}
669
670static void blk_add_trace_rq_requeue(struct request_queue *q,
671 struct request *rq)
672{
673 blk_add_trace_rq(q, rq, BLK_TA_REQUEUE);
674}
675
676static void blk_add_trace_rq_complete(struct request_queue *q,
677 struct request *rq)
678{
679 blk_add_trace_rq(q, rq, BLK_TA_COMPLETE);
680}
681
682/**
683 * blk_add_trace_bio - Add a trace for a bio oriented action
684 * @q: queue the io is for
685 * @bio: the source bio
686 * @what: the action
687 *
688 * Description:
689 * Records an action against a bio. Will log the bio offset + size.
690 *
691 **/
692static void blk_add_trace_bio(struct request_queue *q, struct bio *bio,
693 u32 what)
694{
695 struct blk_trace *bt = q->blk_trace;
696
697 if (likely(!bt))
698 return;
699
700 __blk_add_trace(bt, bio->bi_sector, bio->bi_size, bio->bi_rw, what,
701 !bio_flagged(bio, BIO_UPTODATE), 0, NULL);
702}
703
704static void blk_add_trace_bio_bounce(struct request_queue *q, struct bio *bio)
705{
706 blk_add_trace_bio(q, bio, BLK_TA_BOUNCE);
707}
708
709static void blk_add_trace_bio_complete(struct request_queue *q, struct bio *bio)
710{
711 blk_add_trace_bio(q, bio, BLK_TA_COMPLETE);
712}
713
714static void blk_add_trace_bio_backmerge(struct request_queue *q,
715 struct bio *bio)
716{
717 blk_add_trace_bio(q, bio, BLK_TA_BACKMERGE);
718}
719
720static void blk_add_trace_bio_frontmerge(struct request_queue *q,
721 struct bio *bio)
722{
723 blk_add_trace_bio(q, bio, BLK_TA_FRONTMERGE);
724}
725
726static void blk_add_trace_bio_queue(struct request_queue *q, struct bio *bio)
727{
728 blk_add_trace_bio(q, bio, BLK_TA_QUEUE);
729}
730
731static void blk_add_trace_getrq(struct request_queue *q,
732 struct bio *bio, int rw)
733{
734 if (bio)
735 blk_add_trace_bio(q, bio, BLK_TA_GETRQ);
736 else {
737 struct blk_trace *bt = q->blk_trace;
738
739 if (bt)
740 __blk_add_trace(bt, 0, 0, rw, BLK_TA_GETRQ, 0, 0, NULL);
741 }
742}
743
744
745static void blk_add_trace_sleeprq(struct request_queue *q,
746 struct bio *bio, int rw)
747{
748 if (bio)
749 blk_add_trace_bio(q, bio, BLK_TA_SLEEPRQ);
750 else {
751 struct blk_trace *bt = q->blk_trace;
752
753 if (bt)
754 __blk_add_trace(bt, 0, 0, rw, BLK_TA_SLEEPRQ,
755 0, 0, NULL);
756 }
757}
758
759static void blk_add_trace_plug(struct request_queue *q)
760{
761 struct blk_trace *bt = q->blk_trace;
762
763 if (bt)
764 __blk_add_trace(bt, 0, 0, 0, BLK_TA_PLUG, 0, 0, NULL);
765}
766
767static void blk_add_trace_unplug_io(struct request_queue *q)
768{
769 struct blk_trace *bt = q->blk_trace;
770
771 if (bt) {
772 unsigned int pdu = q->rq.count[READ] + q->rq.count[WRITE];
773 __be64 rpdu = cpu_to_be64(pdu);
774
775 __blk_add_trace(bt, 0, 0, 0, BLK_TA_UNPLUG_IO, 0,
776 sizeof(rpdu), &rpdu);
777 }
778}
779
780static void blk_add_trace_unplug_timer(struct request_queue *q)
781{
782 struct blk_trace *bt = q->blk_trace;
783
784 if (bt) {
785 unsigned int pdu = q->rq.count[READ] + q->rq.count[WRITE];
786 __be64 rpdu = cpu_to_be64(pdu);
787
788 __blk_add_trace(bt, 0, 0, 0, BLK_TA_UNPLUG_TIMER, 0,
789 sizeof(rpdu), &rpdu);
790 }
791}
792
793static void blk_add_trace_split(struct request_queue *q, struct bio *bio,
794 unsigned int pdu)
795{
796 struct blk_trace *bt = q->blk_trace;
797
798 if (bt) {
799 __be64 rpdu = cpu_to_be64(pdu);
800
801 __blk_add_trace(bt, bio->bi_sector, bio->bi_size, bio->bi_rw,
802 BLK_TA_SPLIT, !bio_flagged(bio, BIO_UPTODATE),
803 sizeof(rpdu), &rpdu);
804 }
805}
806
807/**
808 * blk_add_trace_remap - Add a trace for a remap operation
809 * @q: queue the io is for
810 * @bio: the source bio
811 * @dev: target device
812 * @from: source sector
813 * @to: target sector
814 *
815 * Description:
816 * Device mapper or raid target sometimes need to split a bio because
817 * it spans a stripe (or similar). Add a trace for that action.
818 *
819 **/
820static void blk_add_trace_remap(struct request_queue *q, struct bio *bio,
821 dev_t dev, sector_t from, sector_t to)
822{
823 struct blk_trace *bt = q->blk_trace;
824 struct blk_io_trace_remap r;
825
826 if (likely(!bt))
827 return;
828
829 r.device = cpu_to_be32(dev);
830 r.device_from = cpu_to_be32(bio->bi_bdev->bd_dev);
831 r.sector = cpu_to_be64(to);
832
833 __blk_add_trace(bt, from, bio->bi_size, bio->bi_rw, BLK_TA_REMAP,
834 !bio_flagged(bio, BIO_UPTODATE), sizeof(r), &r);
835}
836
837/**
838 * blk_add_driver_data - Add binary message with driver-specific data
839 * @q: queue the io is for
840 * @rq: io request
841 * @data: driver-specific data
842 * @len: length of driver-specific data
843 *
844 * Description:
845 * Some drivers might want to write driver-specific data per request.
846 *
847 **/
848void blk_add_driver_data(struct request_queue *q,
849 struct request *rq,
850 void *data, size_t len)
851{
852 struct blk_trace *bt = q->blk_trace;
853
854 if (likely(!bt))
855 return;
856
857 if (blk_pc_request(rq))
858 __blk_add_trace(bt, 0, rq->data_len, 0, BLK_TA_DRV_DATA,
859 rq->errors, len, data);
860 else
861 __blk_add_trace(bt, rq->hard_sector, rq->hard_nr_sectors << 9,
862 0, BLK_TA_DRV_DATA, rq->errors, len, data);
863}
864EXPORT_SYMBOL_GPL(blk_add_driver_data);
865
866static int blk_register_tracepoints(void)
867{
868 int ret;
869
870 ret = register_trace_block_rq_abort(blk_add_trace_rq_abort);
871 WARN_ON(ret);
872 ret = register_trace_block_rq_insert(blk_add_trace_rq_insert);
873 WARN_ON(ret);
874 ret = register_trace_block_rq_issue(blk_add_trace_rq_issue);
875 WARN_ON(ret);
876 ret = register_trace_block_rq_requeue(blk_add_trace_rq_requeue);
877 WARN_ON(ret);
878 ret = register_trace_block_rq_complete(blk_add_trace_rq_complete);
879 WARN_ON(ret);
880 ret = register_trace_block_bio_bounce(blk_add_trace_bio_bounce);
881 WARN_ON(ret);
882 ret = register_trace_block_bio_complete(blk_add_trace_bio_complete);
883 WARN_ON(ret);
884 ret = register_trace_block_bio_backmerge(blk_add_trace_bio_backmerge);
885 WARN_ON(ret);
886 ret = register_trace_block_bio_frontmerge(blk_add_trace_bio_frontmerge);
887 WARN_ON(ret);
888 ret = register_trace_block_bio_queue(blk_add_trace_bio_queue);
889 WARN_ON(ret);
890 ret = register_trace_block_getrq(blk_add_trace_getrq);
891 WARN_ON(ret);
892 ret = register_trace_block_sleeprq(blk_add_trace_sleeprq);
893 WARN_ON(ret);
894 ret = register_trace_block_plug(blk_add_trace_plug);
895 WARN_ON(ret);
896 ret = register_trace_block_unplug_timer(blk_add_trace_unplug_timer);
897 WARN_ON(ret);
898 ret = register_trace_block_unplug_io(blk_add_trace_unplug_io);
899 WARN_ON(ret);
900 ret = register_trace_block_split(blk_add_trace_split);
901 WARN_ON(ret);
902 ret = register_trace_block_remap(blk_add_trace_remap);
903 WARN_ON(ret);
904 return 0;
905}
906
907static void blk_unregister_tracepoints(void)
908{
909 unregister_trace_block_remap(blk_add_trace_remap);
910 unregister_trace_block_split(blk_add_trace_split);
911 unregister_trace_block_unplug_io(blk_add_trace_unplug_io);
912 unregister_trace_block_unplug_timer(blk_add_trace_unplug_timer);
913 unregister_trace_block_plug(blk_add_trace_plug);
914 unregister_trace_block_sleeprq(blk_add_trace_sleeprq);
915 unregister_trace_block_getrq(blk_add_trace_getrq);
916 unregister_trace_block_bio_queue(blk_add_trace_bio_queue);
917 unregister_trace_block_bio_frontmerge(blk_add_trace_bio_frontmerge);
918 unregister_trace_block_bio_backmerge(blk_add_trace_bio_backmerge);
919 unregister_trace_block_bio_complete(blk_add_trace_bio_complete);
920 unregister_trace_block_bio_bounce(blk_add_trace_bio_bounce);
921 unregister_trace_block_rq_complete(blk_add_trace_rq_complete);
922 unregister_trace_block_rq_requeue(blk_add_trace_rq_requeue);
923 unregister_trace_block_rq_issue(blk_add_trace_rq_issue);
924 unregister_trace_block_rq_insert(blk_add_trace_rq_insert);
925 unregister_trace_block_rq_abort(blk_add_trace_rq_abort);
926
927 tracepoint_synchronize_unregister();
928}
929
930/*
931 * struct blk_io_tracer formatting routines
932 */
933
934static void fill_rwbs(char *rwbs, const struct blk_io_trace *t)
935{
936 int i = 0;
937
938 if (t->action & BLK_TC_DISCARD)
939 rwbs[i++] = 'D';
940 else if (t->action & BLK_TC_WRITE)
941 rwbs[i++] = 'W';
942 else if (t->bytes)
943 rwbs[i++] = 'R';
944 else
945 rwbs[i++] = 'N';
946
947 if (t->action & BLK_TC_AHEAD)
948 rwbs[i++] = 'A';
949 if (t->action & BLK_TC_BARRIER)
950 rwbs[i++] = 'B';
951 if (t->action & BLK_TC_SYNC)
952 rwbs[i++] = 'S';
953 if (t->action & BLK_TC_META)
954 rwbs[i++] = 'M';
955
956 rwbs[i] = '\0';
957}
958
959static inline
960const struct blk_io_trace *te_blk_io_trace(const struct trace_entry *ent)
961{
962 return (const struct blk_io_trace *)ent;
963}
964
965static inline const void *pdu_start(const struct trace_entry *ent)
966{
967 return te_blk_io_trace(ent) + 1;
968}
969
970static inline u32 t_sec(const struct trace_entry *ent)
971{
972 return te_blk_io_trace(ent)->bytes >> 9;
973}
974
975static inline unsigned long long t_sector(const struct trace_entry *ent)
976{
977 return te_blk_io_trace(ent)->sector;
978}
979
980static inline __u16 t_error(const struct trace_entry *ent)
981{
982 return te_blk_io_trace(ent)->sector;
983}
984
985static __u64 get_pdu_int(const struct trace_entry *ent)
986{
987 const __u64 *val = pdu_start(ent);
988 return be64_to_cpu(*val);
989}
990
991static void get_pdu_remap(const struct trace_entry *ent,
992 struct blk_io_trace_remap *r)
993{
994 const struct blk_io_trace_remap *__r = pdu_start(ent);
995 __u64 sector = __r->sector;
996
997 r->device = be32_to_cpu(__r->device);
998 r->device_from = be32_to_cpu(__r->device_from);
999 r->sector = be64_to_cpu(sector);
1000}
1001
1002static int blk_log_action_iter(struct trace_iterator *iter, const char *act)
1003{
1004 char rwbs[6];
1005 unsigned long long ts = ns2usecs(iter->ts);
1006 unsigned long usec_rem = do_div(ts, USEC_PER_SEC);
1007 unsigned secs = (unsigned long)ts;
1008 const struct trace_entry *ent = iter->ent;
1009 const struct blk_io_trace *t = (const struct blk_io_trace *)ent;
1010
1011 fill_rwbs(rwbs, t);
1012
1013 return trace_seq_printf(&iter->seq,
1014 "%3d,%-3d %2d %5d.%06lu %5u %2s %3s ",
1015 MAJOR(t->device), MINOR(t->device), iter->cpu,
1016 secs, usec_rem, ent->pid, act, rwbs);
1017}
1018
1019static int blk_log_action_seq(struct trace_seq *s, const struct blk_io_trace *t,
1020 const char *act)
1021{
1022 char rwbs[6];
1023 fill_rwbs(rwbs, t);
1024 return trace_seq_printf(s, "%3d,%-3d %2s %3s ",
1025 MAJOR(t->device), MINOR(t->device), act, rwbs);
1026}
1027
1028static int blk_log_generic(struct trace_seq *s, const struct trace_entry *ent)
1029{
1030 const char *cmd = trace_find_cmdline(ent->pid);
1031
1032 if (t_sec(ent))
1033 return trace_seq_printf(s, "%llu + %u [%s]\n",
1034 t_sector(ent), t_sec(ent), cmd);
1035 return trace_seq_printf(s, "[%s]\n", cmd);
1036}
1037
1038static int blk_log_with_error(struct trace_seq *s,
1039 const struct trace_entry *ent)
1040{
1041 if (t_sec(ent))
1042 return trace_seq_printf(s, "%llu + %u [%d]\n", t_sector(ent),
1043 t_sec(ent), t_error(ent));
1044 return trace_seq_printf(s, "%llu [%d]\n", t_sector(ent), t_error(ent));
1045}
1046
1047static int blk_log_remap(struct trace_seq *s, const struct trace_entry *ent)
1048{
1049 struct blk_io_trace_remap r = { .device = 0, };
1050
1051 get_pdu_remap(ent, &r);
1052 return trace_seq_printf(s, "%llu + %u <- (%d,%d) %llu\n",
1053 t_sector(ent),
1054 t_sec(ent), MAJOR(r.device), MINOR(r.device),
1055 (unsigned long long)r.sector);
1056}
1057
1058static int blk_log_plug(struct trace_seq *s, const struct trace_entry *ent)
1059{
1060 return trace_seq_printf(s, "[%s]\n", trace_find_cmdline(ent->pid));
1061}
1062
1063static int blk_log_unplug(struct trace_seq *s, const struct trace_entry *ent)
1064{
1065 return trace_seq_printf(s, "[%s] %llu\n", trace_find_cmdline(ent->pid),
1066 get_pdu_int(ent));
1067}
1068
1069static int blk_log_split(struct trace_seq *s, const struct trace_entry *ent)
1070{
1071 return trace_seq_printf(s, "%llu / %llu [%s]\n", t_sector(ent),
1072 get_pdu_int(ent), trace_find_cmdline(ent->pid));
1073}
1074
1075/*
1076 * struct tracer operations
1077 */
1078
1079static void blk_tracer_print_header(struct seq_file *m)
1080{
1081 if (!(blk_tracer_flags.val & TRACE_BLK_OPT_CLASSIC))
1082 return;
1083 seq_puts(m, "# DEV CPU TIMESTAMP PID ACT FLG\n"
1084 "# | | | | | |\n");
1085}
1086
1087static void blk_tracer_start(struct trace_array *tr)
1088{
1089 mutex_lock(&blk_probe_mutex);
1090 if (atomic_add_return(1, &blk_probes_ref) == 1)
1091 if (blk_register_tracepoints())
1092 atomic_dec(&blk_probes_ref);
1093 mutex_unlock(&blk_probe_mutex);
1094 trace_flags &= ~TRACE_ITER_CONTEXT_INFO;
1095}
1096
1097static int blk_tracer_init(struct trace_array *tr)
1098{
1099 blk_tr = tr;
1100 blk_tracer_start(tr);
1101 mutex_lock(&blk_probe_mutex);
1102 blk_tracer_enabled++;
1103 mutex_unlock(&blk_probe_mutex);
1104 return 0;
1105}
1106
1107static void blk_tracer_stop(struct trace_array *tr)
1108{
1109 trace_flags |= TRACE_ITER_CONTEXT_INFO;
1110 mutex_lock(&blk_probe_mutex);
1111 if (atomic_dec_and_test(&blk_probes_ref))
1112 blk_unregister_tracepoints();
1113 mutex_unlock(&blk_probe_mutex);
1114}
1115
1116static void blk_tracer_reset(struct trace_array *tr)
1117{
1118 if (!atomic_read(&blk_probes_ref))
1119 return;
1120
1121 mutex_lock(&blk_probe_mutex);
1122 blk_tracer_enabled--;
1123 WARN_ON(blk_tracer_enabled < 0);
1124 mutex_unlock(&blk_probe_mutex);
1125
1126 blk_tracer_stop(tr);
1127}
1128
1129static struct {
1130 const char *act[2];
1131 int (*print)(struct trace_seq *s, const struct trace_entry *ent);
1132} what2act[] __read_mostly = {
1133 [__BLK_TA_QUEUE] = {{ "Q", "queue" }, blk_log_generic },
1134 [__BLK_TA_BACKMERGE] = {{ "M", "backmerge" }, blk_log_generic },
1135 [__BLK_TA_FRONTMERGE] = {{ "F", "frontmerge" }, blk_log_generic },
1136 [__BLK_TA_GETRQ] = {{ "G", "getrq" }, blk_log_generic },
1137 [__BLK_TA_SLEEPRQ] = {{ "S", "sleeprq" }, blk_log_generic },
1138 [__BLK_TA_REQUEUE] = {{ "R", "requeue" }, blk_log_with_error },
1139 [__BLK_TA_ISSUE] = {{ "D", "issue" }, blk_log_generic },
1140 [__BLK_TA_COMPLETE] = {{ "C", "complete" }, blk_log_with_error },
1141 [__BLK_TA_PLUG] = {{ "P", "plug" }, blk_log_plug },
1142 [__BLK_TA_UNPLUG_IO] = {{ "U", "unplug_io" }, blk_log_unplug },
1143 [__BLK_TA_UNPLUG_TIMER] = {{ "UT", "unplug_timer" }, blk_log_unplug },
1144 [__BLK_TA_INSERT] = {{ "I", "insert" }, blk_log_generic },
1145 [__BLK_TA_SPLIT] = {{ "X", "split" }, blk_log_split },
1146 [__BLK_TA_BOUNCE] = {{ "B", "bounce" }, blk_log_generic },
1147 [__BLK_TA_REMAP] = {{ "A", "remap" }, blk_log_remap },
1148};
1149
1150static enum print_line_t blk_trace_event_print(struct trace_iterator *iter,
1151 int flags)
1152{
1153 struct trace_seq *s = &iter->seq;
1154 const struct blk_io_trace *t = (struct blk_io_trace *)iter->ent;
1155 const u16 what = t->action & ((1 << BLK_TC_SHIFT) - 1);
1156 int ret;
1157
1158 if (!trace_print_context(iter))
1159 return TRACE_TYPE_PARTIAL_LINE;
1160
1161 if (unlikely(what == 0 || what > ARRAY_SIZE(what2act)))
1162 ret = trace_seq_printf(s, "Bad pc action %x\n", what);
1163 else {
1164 const bool long_act = !!(trace_flags & TRACE_ITER_VERBOSE);
1165 ret = blk_log_action_seq(s, t, what2act[what].act[long_act]);
1166 if (ret)
1167 ret = what2act[what].print(s, iter->ent);
1168 }
1169
1170 return ret ? TRACE_TYPE_HANDLED : TRACE_TYPE_PARTIAL_LINE;
1171}
1172
1173static int blk_trace_synthesize_old_trace(struct trace_iterator *iter)
1174{
1175 struct trace_seq *s = &iter->seq;
1176 struct blk_io_trace *t = (struct blk_io_trace *)iter->ent;
1177 const int offset = offsetof(struct blk_io_trace, sector);
1178 struct blk_io_trace old = {
1179 .magic = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE_VERSION,
1180 .time = ns2usecs(iter->ts),
1181 };
1182
1183 if (!trace_seq_putmem(s, &old, offset))
1184 return 0;
1185 return trace_seq_putmem(s, &t->sector,
1186 sizeof(old) - offset + t->pdu_len);
1187}
1188
1189static enum print_line_t
1190blk_trace_event_print_binary(struct trace_iterator *iter, int flags)
1191{
1192 return blk_trace_synthesize_old_trace(iter) ?
1193 TRACE_TYPE_HANDLED : TRACE_TYPE_PARTIAL_LINE;
1194}
1195
1196static enum print_line_t blk_tracer_print_line(struct trace_iterator *iter)
1197{
1198 const struct blk_io_trace *t;
1199 u16 what;
1200 int ret;
1201
1202 if (!(blk_tracer_flags.val & TRACE_BLK_OPT_CLASSIC))
1203 return TRACE_TYPE_UNHANDLED;
1204
1205 t = (const struct blk_io_trace *)iter->ent;
1206 what = t->action & ((1 << BLK_TC_SHIFT) - 1);
1207
1208 if (unlikely(what == 0 || what > ARRAY_SIZE(what2act)))
1209 ret = trace_seq_printf(&iter->seq, "Bad pc action %x\n", what);
1210 else {
1211 const bool long_act = !!(trace_flags & TRACE_ITER_VERBOSE);
1212 ret = blk_log_action_iter(iter, what2act[what].act[long_act]);
1213 if (ret)
1214 ret = what2act[what].print(&iter->seq, iter->ent);
1215 }
1216
1217 return ret ? TRACE_TYPE_HANDLED : TRACE_TYPE_PARTIAL_LINE;
1218}
1219
1220static struct tracer blk_tracer __read_mostly = {
1221 .name = "blk",
1222 .init = blk_tracer_init,
1223 .reset = blk_tracer_reset,
1224 .start = blk_tracer_start,
1225 .stop = blk_tracer_stop,
1226 .print_header = blk_tracer_print_header,
1227 .print_line = blk_tracer_print_line,
1228 .flags = &blk_tracer_flags,
1229};
1230
1231static struct trace_event trace_blk_event = {
1232 .type = TRACE_BLK,
1233 .trace = blk_trace_event_print,
1234 .binary = blk_trace_event_print_binary,
1235};
1236
1237static int __init init_blk_tracer(void)
1238{
1239 if (!register_ftrace_event(&trace_blk_event)) {
1240 pr_warning("Warning: could not register block events\n");
1241 return 1;
1242 }
1243
1244 if (register_tracer(&blk_tracer) != 0) {
1245 pr_warning("Warning: could not register the block tracer\n");
1246 unregister_ftrace_event(&trace_blk_event);
1247 return 1;
1248 }
1249
1250 return 0;
1251}
1252
1253device_initcall(init_blk_tracer);
1254
1255static int blk_trace_remove_queue(struct request_queue *q)
1256{
1257 struct blk_trace *bt;
1258
1259 bt = xchg(&q->blk_trace, NULL);
1260 if (bt == NULL)
1261 return -EINVAL;
1262
1263 kfree(bt);
1264 return 0;
1265}
1266
1267/*
1268 * Setup everything required to start tracing
1269 */
1270static int blk_trace_setup_queue(struct request_queue *q, dev_t dev)
1271{
1272 struct blk_trace *old_bt, *bt = NULL;
1273 int ret;
1274
1275 ret = -ENOMEM;
1276 bt = kzalloc(sizeof(*bt), GFP_KERNEL);
1277 if (!bt)
1278 goto err;
1279
1280 bt->dev = dev;
1281 bt->act_mask = (u16)-1;
1282 bt->end_lba = -1ULL;
1283 bt->trace_state = Blktrace_running;
1284
1285 old_bt = xchg(&q->blk_trace, bt);
1286 if (old_bt != NULL) {
1287 (void)xchg(&q->blk_trace, old_bt);
1288 kfree(bt);
1289 ret = -EBUSY;
1290 }
1291 return 0;
1292err:
1293 return ret;
1294}
1295
1296/*
1297 * sysfs interface to enable and configure tracing
1298 */
1299
1300static ssize_t sysfs_blk_trace_enable_show(struct device *dev,
1301 struct device_attribute *attr,
1302 char *buf)
1303{
1304 struct hd_struct *p = dev_to_part(dev);
1305 struct block_device *bdev;
1306 ssize_t ret = -ENXIO;
1307
1308 lock_kernel();
1309 bdev = bdget(part_devt(p));
1310 if (bdev != NULL) {
1311 struct request_queue *q = bdev_get_queue(bdev);
1312
1313 if (q != NULL) {
1314 mutex_lock(&bdev->bd_mutex);
1315 ret = sprintf(buf, "%u\n", !!q->blk_trace);
1316 mutex_unlock(&bdev->bd_mutex);
1317 }
1318
1319 bdput(bdev);
1320 }
1321
1322 unlock_kernel();
1323 return ret;
1324}
1325
1326static ssize_t sysfs_blk_trace_enable_store(struct device *dev,
1327 struct device_attribute *attr,
1328 const char *buf, size_t count)
1329{
1330 struct block_device *bdev;
1331 struct request_queue *q;
1332 struct hd_struct *p;
1333 int value;
1334 ssize_t ret = -ENXIO;
1335
1336 if (count == 0 || sscanf(buf, "%d", &value) != 1)
1337 goto out;
1338
1339 lock_kernel();
1340 p = dev_to_part(dev);
1341 bdev = bdget(part_devt(p));
1342 if (bdev == NULL)
1343 goto out_unlock_kernel;
1344
1345 q = bdev_get_queue(bdev);
1346 if (q == NULL)
1347 goto out_bdput;
1348
1349 mutex_lock(&bdev->bd_mutex);
1350 if (value)
1351 ret = blk_trace_setup_queue(q, bdev->bd_dev);
1352 else
1353 ret = blk_trace_remove_queue(q);
1354 mutex_unlock(&bdev->bd_mutex);
1355
1356 if (ret == 0)
1357 ret = count;
1358out_bdput:
1359 bdput(bdev);
1360out_unlock_kernel:
1361 unlock_kernel();
1362out:
1363 return ret;
1364}
1365
1366static ssize_t sysfs_blk_trace_attr_show(struct device *dev,
1367 struct device_attribute *attr,
1368 char *buf);
1369static ssize_t sysfs_blk_trace_attr_store(struct device *dev,
1370 struct device_attribute *attr,
1371 const char *buf, size_t count);
1372#define BLK_TRACE_DEVICE_ATTR(_name) \
1373 DEVICE_ATTR(_name, S_IRUGO | S_IWUSR, \
1374 sysfs_blk_trace_attr_show, \
1375 sysfs_blk_trace_attr_store)
1376
1377static DEVICE_ATTR(enable, S_IRUGO | S_IWUSR,
1378 sysfs_blk_trace_enable_show, sysfs_blk_trace_enable_store);
1379static BLK_TRACE_DEVICE_ATTR(act_mask);
1380static BLK_TRACE_DEVICE_ATTR(pid);
1381static BLK_TRACE_DEVICE_ATTR(start_lba);
1382static BLK_TRACE_DEVICE_ATTR(end_lba);
1383
1384static struct attribute *blk_trace_attrs[] = {
1385 &dev_attr_enable.attr,
1386 &dev_attr_act_mask.attr,
1387 &dev_attr_pid.attr,
1388 &dev_attr_start_lba.attr,
1389 &dev_attr_end_lba.attr,
1390 NULL
1391};
1392
1393struct attribute_group blk_trace_attr_group = {
1394 .name = "trace",
1395 .attrs = blk_trace_attrs,
1396};
1397
1398static int blk_str2act_mask(const char *str)
1399{
1400 int mask = 0;
1401 char *copy = kstrdup(str, GFP_KERNEL), *s;
1402
1403 if (copy == NULL)
1404 return -ENOMEM;
1405
1406 s = strstrip(copy);
1407
1408 while (1) {
1409 char *sep = strchr(s, ',');
1410
1411 if (sep != NULL)
1412 *sep = '\0';
1413
1414 if (strcasecmp(s, "barrier") == 0)
1415 mask |= BLK_TC_BARRIER;
1416 else if (strcasecmp(s, "complete") == 0)
1417 mask |= BLK_TC_COMPLETE;
1418 else if (strcasecmp(s, "fs") == 0)
1419 mask |= BLK_TC_FS;
1420 else if (strcasecmp(s, "issue") == 0)
1421 mask |= BLK_TC_ISSUE;
1422 else if (strcasecmp(s, "pc") == 0)
1423 mask |= BLK_TC_PC;
1424 else if (strcasecmp(s, "queue") == 0)
1425 mask |= BLK_TC_QUEUE;
1426 else if (strcasecmp(s, "read") == 0)
1427 mask |= BLK_TC_READ;
1428 else if (strcasecmp(s, "requeue") == 0)
1429 mask |= BLK_TC_REQUEUE;
1430 else if (strcasecmp(s, "sync") == 0)
1431 mask |= BLK_TC_SYNC;
1432 else if (strcasecmp(s, "write") == 0)
1433 mask |= BLK_TC_WRITE;
1434
1435 if (sep == NULL)
1436 break;
1437
1438 s = sep + 1;
1439 }
1440 kfree(copy);
1441
1442 return mask;
1443}
1444
1445static ssize_t sysfs_blk_trace_attr_show(struct device *dev,
1446 struct device_attribute *attr,
1447 char *buf)
1448{
1449 struct hd_struct *p = dev_to_part(dev);
1450 struct request_queue *q;
1451 struct block_device *bdev;
1452 ssize_t ret = -ENXIO;
1453
1454 lock_kernel();
1455 bdev = bdget(part_devt(p));
1456 if (bdev == NULL)
1457 goto out_unlock_kernel;
1458
1459 q = bdev_get_queue(bdev);
1460 if (q == NULL)
1461 goto out_bdput;
1462 mutex_lock(&bdev->bd_mutex);
1463 if (q->blk_trace == NULL)
1464 ret = sprintf(buf, "disabled\n");
1465 else if (attr == &dev_attr_act_mask)
1466 ret = sprintf(buf, "%#x\n", q->blk_trace->act_mask);
1467 else if (attr == &dev_attr_pid)
1468 ret = sprintf(buf, "%u\n", q->blk_trace->pid);
1469 else if (attr == &dev_attr_start_lba)
1470 ret = sprintf(buf, "%llu\n", q->blk_trace->start_lba);
1471 else if (attr == &dev_attr_end_lba)
1472 ret = sprintf(buf, "%llu\n", q->blk_trace->end_lba);
1473 mutex_unlock(&bdev->bd_mutex);
1474out_bdput:
1475 bdput(bdev);
1476out_unlock_kernel:
1477 unlock_kernel();
1478 return ret;
1479}
1480
1481static ssize_t sysfs_blk_trace_attr_store(struct device *dev,
1482 struct device_attribute *attr,
1483 const char *buf, size_t count)
1484{
1485 struct block_device *bdev;
1486 struct request_queue *q;
1487 struct hd_struct *p;
1488 u64 value;
1489 ssize_t ret = -ENXIO;
1490
1491 if (count == 0)
1492 goto out;
1493
1494 if (attr == &dev_attr_act_mask) {
1495 if (sscanf(buf, "%llx", &value) != 1) {
1496 /* Assume it is a list of trace category names */
1497 value = blk_str2act_mask(buf);
1498 if (value < 0)
1499 goto out;
1500 }
1501 } else if (sscanf(buf, "%llu", &value) != 1)
1502 goto out;
1503
1504 lock_kernel();
1505 p = dev_to_part(dev);
1506 bdev = bdget(part_devt(p));
1507 if (bdev == NULL)
1508 goto out_unlock_kernel;
1509
1510 q = bdev_get_queue(bdev);
1511 if (q == NULL)
1512 goto out_bdput;
1513
1514 mutex_lock(&bdev->bd_mutex);
1515 ret = 0;
1516 if (q->blk_trace == NULL)
1517 ret = blk_trace_setup_queue(q, bdev->bd_dev);
1518
1519 if (ret == 0) {
1520 if (attr == &dev_attr_act_mask)
1521 q->blk_trace->act_mask = value;
1522 else if (attr == &dev_attr_pid)
1523 q->blk_trace->pid = value;
1524 else if (attr == &dev_attr_start_lba)
1525 q->blk_trace->start_lba = value;
1526 else if (attr == &dev_attr_end_lba)
1527 q->blk_trace->end_lba = value;
1528 ret = count;
1529 }
1530 mutex_unlock(&bdev->bd_mutex);
1531out_bdput:
1532 bdput(bdev);
1533out_unlock_kernel:
1534 unlock_kernel();
1535out:
1536 return ret;
1537}
diff --git a/kernel/trace/events.c b/kernel/trace/events.c
new file mode 100644
index 000000000000..9fc918da404f
--- /dev/null
+++ b/kernel/trace/events.c
@@ -0,0 +1,15 @@
1/*
2 * This is the place to register all trace points as events.
3 */
4
5#include <linux/stringify.h>
6
7#include <trace/trace_events.h>
8
9#include "trace_output.h"
10
11#include "trace_events_stage_1.h"
12#include "trace_events_stage_2.h"
13#include "trace_events_stage_3.h"
14
15#include <trace/trace_event_types.h>
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
index fdf913dfc7e8..d33d306bdcf4 100644
--- a/kernel/trace/ftrace.c
+++ b/kernel/trace/ftrace.c
@@ -27,6 +27,7 @@
27#include <linux/sysctl.h> 27#include <linux/sysctl.h>
28#include <linux/ctype.h> 28#include <linux/ctype.h>
29#include <linux/list.h> 29#include <linux/list.h>
30#include <linux/hash.h>
30 31
31#include <asm/ftrace.h> 32#include <asm/ftrace.h>
32 33
@@ -44,14 +45,14 @@
44 ftrace_kill(); \ 45 ftrace_kill(); \
45 } while (0) 46 } while (0)
46 47
48/* hash bits for specific function selection */
49#define FTRACE_HASH_BITS 7
50#define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)
51
47/* ftrace_enabled is a method to turn ftrace on or off */ 52/* ftrace_enabled is a method to turn ftrace on or off */
48int ftrace_enabled __read_mostly; 53int ftrace_enabled __read_mostly;
49static int last_ftrace_enabled; 54static int last_ftrace_enabled;
50 55
51/* set when tracing only a pid */
52struct pid *ftrace_pid_trace;
53static struct pid * const ftrace_swapper_pid = &init_struct_pid;
54
55/* Quick disabling of function tracer. */ 56/* Quick disabling of function tracer. */
56int function_trace_stop; 57int function_trace_stop;
57 58
@@ -61,9 +62,7 @@ int function_trace_stop;
61 */ 62 */
62static int ftrace_disabled __read_mostly; 63static int ftrace_disabled __read_mostly;
63 64
64static DEFINE_SPINLOCK(ftrace_lock); 65static DEFINE_MUTEX(ftrace_lock);
65static DEFINE_MUTEX(ftrace_sysctl_lock);
66static DEFINE_MUTEX(ftrace_start_lock);
67 66
68static struct ftrace_ops ftrace_list_end __read_mostly = 67static struct ftrace_ops ftrace_list_end __read_mostly =
69{ 68{
@@ -134,9 +133,6 @@ static void ftrace_test_stop_func(unsigned long ip, unsigned long parent_ip)
134 133
135static int __register_ftrace_function(struct ftrace_ops *ops) 134static int __register_ftrace_function(struct ftrace_ops *ops)
136{ 135{
137 /* should not be called from interrupt context */
138 spin_lock(&ftrace_lock);
139
140 ops->next = ftrace_list; 136 ops->next = ftrace_list;
141 /* 137 /*
142 * We are entering ops into the ftrace_list but another 138 * We are entering ops into the ftrace_list but another
@@ -172,18 +168,12 @@ static int __register_ftrace_function(struct ftrace_ops *ops)
172#endif 168#endif
173 } 169 }
174 170
175 spin_unlock(&ftrace_lock);
176
177 return 0; 171 return 0;
178} 172}
179 173
180static int __unregister_ftrace_function(struct ftrace_ops *ops) 174static int __unregister_ftrace_function(struct ftrace_ops *ops)
181{ 175{
182 struct ftrace_ops **p; 176 struct ftrace_ops **p;
183 int ret = 0;
184
185 /* should not be called from interrupt context */
186 spin_lock(&ftrace_lock);
187 177
188 /* 178 /*
189 * If we are removing the last function, then simply point 179 * If we are removing the last function, then simply point
@@ -192,17 +182,15 @@ static int __unregister_ftrace_function(struct ftrace_ops *ops)
192 if (ftrace_list == ops && ops->next == &ftrace_list_end) { 182 if (ftrace_list == ops && ops->next == &ftrace_list_end) {
193 ftrace_trace_function = ftrace_stub; 183 ftrace_trace_function = ftrace_stub;
194 ftrace_list = &ftrace_list_end; 184 ftrace_list = &ftrace_list_end;
195 goto out; 185 return 0;
196 } 186 }
197 187
198 for (p = &ftrace_list; *p != &ftrace_list_end; p = &(*p)->next) 188 for (p = &ftrace_list; *p != &ftrace_list_end; p = &(*p)->next)
199 if (*p == ops) 189 if (*p == ops)
200 break; 190 break;
201 191
202 if (*p != ops) { 192 if (*p != ops)
203 ret = -1; 193 return -1;
204 goto out;
205 }
206 194
207 *p = (*p)->next; 195 *p = (*p)->next;
208 196
@@ -223,21 +211,15 @@ static int __unregister_ftrace_function(struct ftrace_ops *ops)
223 } 211 }
224 } 212 }
225 213
226 out: 214 return 0;
227 spin_unlock(&ftrace_lock);
228
229 return ret;
230} 215}
231 216
232static void ftrace_update_pid_func(void) 217static void ftrace_update_pid_func(void)
233{ 218{
234 ftrace_func_t func; 219 ftrace_func_t func;
235 220
236 /* should not be called from interrupt context */
237 spin_lock(&ftrace_lock);
238
239 if (ftrace_trace_function == ftrace_stub) 221 if (ftrace_trace_function == ftrace_stub)
240 goto out; 222 return;
241 223
242 func = ftrace_trace_function; 224 func = ftrace_trace_function;
243 225
@@ -254,23 +236,29 @@ static void ftrace_update_pid_func(void)
254#else 236#else
255 __ftrace_trace_function = func; 237 __ftrace_trace_function = func;
256#endif 238#endif
257
258 out:
259 spin_unlock(&ftrace_lock);
260} 239}
261 240
241/* set when tracing only a pid */
242struct pid *ftrace_pid_trace;
243static struct pid * const ftrace_swapper_pid = &init_struct_pid;
244
262#ifdef CONFIG_DYNAMIC_FTRACE 245#ifdef CONFIG_DYNAMIC_FTRACE
246
263#ifndef CONFIG_FTRACE_MCOUNT_RECORD 247#ifndef CONFIG_FTRACE_MCOUNT_RECORD
264# error Dynamic ftrace depends on MCOUNT_RECORD 248# error Dynamic ftrace depends on MCOUNT_RECORD
265#endif 249#endif
266 250
267/* 251static struct hlist_head ftrace_func_hash[FTRACE_FUNC_HASHSIZE] __read_mostly;
268 * Since MCOUNT_ADDR may point to mcount itself, we do not want 252
269 * to get it confused by reading a reference in the code as we 253struct ftrace_func_probe {
270 * are parsing on objcopy output of text. Use a variable for 254 struct hlist_node node;
271 * it instead. 255 struct ftrace_probe_ops *ops;
272 */ 256 unsigned long flags;
273static unsigned long mcount_addr = MCOUNT_ADDR; 257 unsigned long ip;
258 void *data;
259 struct rcu_head rcu;
260};
261
274 262
275enum { 263enum {
276 FTRACE_ENABLE_CALLS = (1 << 0), 264 FTRACE_ENABLE_CALLS = (1 << 0),
@@ -290,7 +278,7 @@ static DEFINE_MUTEX(ftrace_regex_lock);
290 278
291struct ftrace_page { 279struct ftrace_page {
292 struct ftrace_page *next; 280 struct ftrace_page *next;
293 unsigned long index; 281 int index;
294 struct dyn_ftrace records[]; 282 struct dyn_ftrace records[];
295}; 283};
296 284
@@ -305,6 +293,19 @@ static struct ftrace_page *ftrace_pages;
305 293
306static struct dyn_ftrace *ftrace_free_records; 294static struct dyn_ftrace *ftrace_free_records;
307 295
296/*
297 * This is a double for. Do not use 'break' to break out of the loop,
298 * you must use a goto.
299 */
300#define do_for_each_ftrace_rec(pg, rec) \
301 for (pg = ftrace_pages_start; pg; pg = pg->next) { \
302 int _____i; \
303 for (_____i = 0; _____i < pg->index; _____i++) { \
304 rec = &pg->records[_____i];
305
306#define while_for_each_ftrace_rec() \
307 } \
308 }
308 309
309#ifdef CONFIG_KPROBES 310#ifdef CONFIG_KPROBES
310 311
@@ -349,23 +350,16 @@ void ftrace_release(void *start, unsigned long size)
349 struct ftrace_page *pg; 350 struct ftrace_page *pg;
350 unsigned long s = (unsigned long)start; 351 unsigned long s = (unsigned long)start;
351 unsigned long e = s + size; 352 unsigned long e = s + size;
352 int i;
353 353
354 if (ftrace_disabled || !start) 354 if (ftrace_disabled || !start)
355 return; 355 return;
356 356
357 /* should not be called from interrupt context */ 357 mutex_lock(&ftrace_lock);
358 spin_lock(&ftrace_lock); 358 do_for_each_ftrace_rec(pg, rec) {
359 359 if ((rec->ip >= s) && (rec->ip < e))
360 for (pg = ftrace_pages_start; pg; pg = pg->next) { 360 ftrace_free_rec(rec);
361 for (i = 0; i < pg->index; i++) { 361 } while_for_each_ftrace_rec();
362 rec = &pg->records[i]; 362 mutex_unlock(&ftrace_lock);
363
364 if ((rec->ip >= s) && (rec->ip < e))
365 ftrace_free_rec(rec);
366 }
367 }
368 spin_unlock(&ftrace_lock);
369} 363}
370 364
371static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip) 365static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip)
@@ -461,10 +455,10 @@ static void ftrace_bug(int failed, unsigned long ip)
461static int 455static int
462__ftrace_replace_code(struct dyn_ftrace *rec, int enable) 456__ftrace_replace_code(struct dyn_ftrace *rec, int enable)
463{ 457{
464 unsigned long ip, fl;
465 unsigned long ftrace_addr; 458 unsigned long ftrace_addr;
459 unsigned long ip, fl;
466 460
467 ftrace_addr = (unsigned long)ftrace_caller; 461 ftrace_addr = (unsigned long)FTRACE_ADDR;
468 462
469 ip = rec->ip; 463 ip = rec->ip;
470 464
@@ -473,7 +467,7 @@ __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
473 * it is not enabled then do nothing. 467 * it is not enabled then do nothing.
474 * 468 *
475 * If this record is not to be traced and 469 * If this record is not to be traced and
476 * it is enabled then disabled it. 470 * it is enabled then disable it.
477 * 471 *
478 */ 472 */
479 if (rec->flags & FTRACE_FL_NOTRACE) { 473 if (rec->flags & FTRACE_FL_NOTRACE) {
@@ -493,7 +487,7 @@ __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
493 if (fl == (FTRACE_FL_FILTER | FTRACE_FL_ENABLED)) 487 if (fl == (FTRACE_FL_FILTER | FTRACE_FL_ENABLED))
494 return 0; 488 return 0;
495 489
496 /* Record is not filtered and is not enabled do nothing */ 490 /* Record is not filtered or enabled, do nothing */
497 if (!fl) 491 if (!fl)
498 return 0; 492 return 0;
499 493
@@ -515,7 +509,7 @@ __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
515 509
516 } else { 510 } else {
517 511
518 /* if record is not enabled do nothing */ 512 /* if record is not enabled, do nothing */
519 if (!(rec->flags & FTRACE_FL_ENABLED)) 513 if (!(rec->flags & FTRACE_FL_ENABLED))
520 return 0; 514 return 0;
521 515
@@ -531,41 +525,40 @@ __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
531 525
532static void ftrace_replace_code(int enable) 526static void ftrace_replace_code(int enable)
533{ 527{
534 int i, failed;
535 struct dyn_ftrace *rec; 528 struct dyn_ftrace *rec;
536 struct ftrace_page *pg; 529 struct ftrace_page *pg;
530 int failed;
537 531
538 for (pg = ftrace_pages_start; pg; pg = pg->next) { 532 do_for_each_ftrace_rec(pg, rec) {
539 for (i = 0; i < pg->index; i++) { 533 /*
540 rec = &pg->records[i]; 534 * Skip over free records and records that have
541 535 * failed.
542 /* 536 */
543 * Skip over free records and records that have 537 if (rec->flags & FTRACE_FL_FREE ||
544 * failed. 538 rec->flags & FTRACE_FL_FAILED)
545 */ 539 continue;
546 if (rec->flags & FTRACE_FL_FREE ||
547 rec->flags & FTRACE_FL_FAILED)
548 continue;
549 540
550 /* ignore updates to this record's mcount site */ 541 /* ignore updates to this record's mcount site */
551 if (get_kprobe((void *)rec->ip)) { 542 if (get_kprobe((void *)rec->ip)) {
552 freeze_record(rec); 543 freeze_record(rec);
553 continue; 544 continue;
554 } else { 545 } else {
555 unfreeze_record(rec); 546 unfreeze_record(rec);
556 } 547 }
557 548
558 failed = __ftrace_replace_code(rec, enable); 549 failed = __ftrace_replace_code(rec, enable);
559 if (failed && (rec->flags & FTRACE_FL_CONVERTED)) { 550 if (failed && (rec->flags & FTRACE_FL_CONVERTED)) {
560 rec->flags |= FTRACE_FL_FAILED; 551 rec->flags |= FTRACE_FL_FAILED;
561 if ((system_state == SYSTEM_BOOTING) || 552 if ((system_state == SYSTEM_BOOTING) ||
562 !core_kernel_text(rec->ip)) { 553 !core_kernel_text(rec->ip)) {
563 ftrace_free_rec(rec); 554 ftrace_free_rec(rec);
564 } else 555 } else {
565 ftrace_bug(failed, rec->ip); 556 ftrace_bug(failed, rec->ip);
566 } 557 /* Stop processing */
558 return;
559 }
567 } 560 }
568 } 561 } while_for_each_ftrace_rec();
569} 562}
570 563
571static int 564static int
@@ -576,7 +569,7 @@ ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
576 569
577 ip = rec->ip; 570 ip = rec->ip;
578 571
579 ret = ftrace_make_nop(mod, rec, mcount_addr); 572 ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
580 if (ret) { 573 if (ret) {
581 ftrace_bug(ret, ip); 574 ftrace_bug(ret, ip);
582 rec->flags |= FTRACE_FL_FAILED; 575 rec->flags |= FTRACE_FL_FAILED;
@@ -585,6 +578,24 @@ ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
585 return 1; 578 return 1;
586} 579}
587 580
581/*
582 * archs can override this function if they must do something
583 * before the modifying code is performed.
584 */
585int __weak ftrace_arch_code_modify_prepare(void)
586{
587 return 0;
588}
589
590/*
591 * archs can override this function if they must do something
592 * after the modifying code is performed.
593 */
594int __weak ftrace_arch_code_modify_post_process(void)
595{
596 return 0;
597}
598
588static int __ftrace_modify_code(void *data) 599static int __ftrace_modify_code(void *data)
589{ 600{
590 int *command = data; 601 int *command = data;
@@ -607,7 +618,17 @@ static int __ftrace_modify_code(void *data)
607 618
608static void ftrace_run_update_code(int command) 619static void ftrace_run_update_code(int command)
609{ 620{
621 int ret;
622
623 ret = ftrace_arch_code_modify_prepare();
624 FTRACE_WARN_ON(ret);
625 if (ret)
626 return;
627
610 stop_machine(__ftrace_modify_code, &command, NULL); 628 stop_machine(__ftrace_modify_code, &command, NULL);
629
630 ret = ftrace_arch_code_modify_post_process();
631 FTRACE_WARN_ON(ret);
611} 632}
612 633
613static ftrace_func_t saved_ftrace_func; 634static ftrace_func_t saved_ftrace_func;
@@ -631,13 +652,10 @@ static void ftrace_startup(int command)
631 if (unlikely(ftrace_disabled)) 652 if (unlikely(ftrace_disabled))
632 return; 653 return;
633 654
634 mutex_lock(&ftrace_start_lock);
635 ftrace_start_up++; 655 ftrace_start_up++;
636 command |= FTRACE_ENABLE_CALLS; 656 command |= FTRACE_ENABLE_CALLS;
637 657
638 ftrace_startup_enable(command); 658 ftrace_startup_enable(command);
639
640 mutex_unlock(&ftrace_start_lock);
641} 659}
642 660
643static void ftrace_shutdown(int command) 661static void ftrace_shutdown(int command)
@@ -645,7 +663,6 @@ static void ftrace_shutdown(int command)
645 if (unlikely(ftrace_disabled)) 663 if (unlikely(ftrace_disabled))
646 return; 664 return;
647 665
648 mutex_lock(&ftrace_start_lock);
649 ftrace_start_up--; 666 ftrace_start_up--;
650 if (!ftrace_start_up) 667 if (!ftrace_start_up)
651 command |= FTRACE_DISABLE_CALLS; 668 command |= FTRACE_DISABLE_CALLS;
@@ -656,11 +673,9 @@ static void ftrace_shutdown(int command)
656 } 673 }
657 674
658 if (!command || !ftrace_enabled) 675 if (!command || !ftrace_enabled)
659 goto out; 676 return;
660 677
661 ftrace_run_update_code(command); 678 ftrace_run_update_code(command);
662 out:
663 mutex_unlock(&ftrace_start_lock);
664} 679}
665 680
666static void ftrace_startup_sysctl(void) 681static void ftrace_startup_sysctl(void)
@@ -670,7 +685,6 @@ static void ftrace_startup_sysctl(void)
670 if (unlikely(ftrace_disabled)) 685 if (unlikely(ftrace_disabled))
671 return; 686 return;
672 687
673 mutex_lock(&ftrace_start_lock);
674 /* Force update next time */ 688 /* Force update next time */
675 saved_ftrace_func = NULL; 689 saved_ftrace_func = NULL;
676 /* ftrace_start_up is true if we want ftrace running */ 690 /* ftrace_start_up is true if we want ftrace running */
@@ -678,7 +692,6 @@ static void ftrace_startup_sysctl(void)
678 command |= FTRACE_ENABLE_CALLS; 692 command |= FTRACE_ENABLE_CALLS;
679 693
680 ftrace_run_update_code(command); 694 ftrace_run_update_code(command);
681 mutex_unlock(&ftrace_start_lock);
682} 695}
683 696
684static void ftrace_shutdown_sysctl(void) 697static void ftrace_shutdown_sysctl(void)
@@ -688,13 +701,11 @@ static void ftrace_shutdown_sysctl(void)
688 if (unlikely(ftrace_disabled)) 701 if (unlikely(ftrace_disabled))
689 return; 702 return;
690 703
691 mutex_lock(&ftrace_start_lock);
692 /* ftrace_start_up is true if ftrace is running */ 704 /* ftrace_start_up is true if ftrace is running */
693 if (ftrace_start_up) 705 if (ftrace_start_up)
694 command |= FTRACE_DISABLE_CALLS; 706 command |= FTRACE_DISABLE_CALLS;
695 707
696 ftrace_run_update_code(command); 708 ftrace_run_update_code(command);
697 mutex_unlock(&ftrace_start_lock);
698} 709}
699 710
700static cycle_t ftrace_update_time; 711static cycle_t ftrace_update_time;
@@ -781,13 +792,16 @@ enum {
781 FTRACE_ITER_CONT = (1 << 1), 792 FTRACE_ITER_CONT = (1 << 1),
782 FTRACE_ITER_NOTRACE = (1 << 2), 793 FTRACE_ITER_NOTRACE = (1 << 2),
783 FTRACE_ITER_FAILURES = (1 << 3), 794 FTRACE_ITER_FAILURES = (1 << 3),
795 FTRACE_ITER_PRINTALL = (1 << 4),
796 FTRACE_ITER_HASH = (1 << 5),
784}; 797};
785 798
786#define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */ 799#define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
787 800
788struct ftrace_iterator { 801struct ftrace_iterator {
789 struct ftrace_page *pg; 802 struct ftrace_page *pg;
790 unsigned idx; 803 int hidx;
804 int idx;
791 unsigned flags; 805 unsigned flags;
792 unsigned char buffer[FTRACE_BUFF_MAX+1]; 806 unsigned char buffer[FTRACE_BUFF_MAX+1];
793 unsigned buffer_idx; 807 unsigned buffer_idx;
@@ -795,15 +809,89 @@ struct ftrace_iterator {
795}; 809};
796 810
797static void * 811static void *
812t_hash_next(struct seq_file *m, void *v, loff_t *pos)
813{
814 struct ftrace_iterator *iter = m->private;
815 struct hlist_node *hnd = v;
816 struct hlist_head *hhd;
817
818 WARN_ON(!(iter->flags & FTRACE_ITER_HASH));
819
820 (*pos)++;
821
822 retry:
823 if (iter->hidx >= FTRACE_FUNC_HASHSIZE)
824 return NULL;
825
826 hhd = &ftrace_func_hash[iter->hidx];
827
828 if (hlist_empty(hhd)) {
829 iter->hidx++;
830 hnd = NULL;
831 goto retry;
832 }
833
834 if (!hnd)
835 hnd = hhd->first;
836 else {
837 hnd = hnd->next;
838 if (!hnd) {
839 iter->hidx++;
840 goto retry;
841 }
842 }
843
844 return hnd;
845}
846
847static void *t_hash_start(struct seq_file *m, loff_t *pos)
848{
849 struct ftrace_iterator *iter = m->private;
850 void *p = NULL;
851
852 iter->flags |= FTRACE_ITER_HASH;
853
854 return t_hash_next(m, p, pos);
855}
856
857static int t_hash_show(struct seq_file *m, void *v)
858{
859 struct ftrace_func_probe *rec;
860 struct hlist_node *hnd = v;
861 char str[KSYM_SYMBOL_LEN];
862
863 rec = hlist_entry(hnd, struct ftrace_func_probe, node);
864
865 if (rec->ops->print)
866 return rec->ops->print(m, rec->ip, rec->ops, rec->data);
867
868 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
869 seq_printf(m, "%s:", str);
870
871 kallsyms_lookup((unsigned long)rec->ops->func, NULL, NULL, NULL, str);
872 seq_printf(m, "%s", str);
873
874 if (rec->data)
875 seq_printf(m, ":%p", rec->data);
876 seq_putc(m, '\n');
877
878 return 0;
879}
880
881static void *
798t_next(struct seq_file *m, void *v, loff_t *pos) 882t_next(struct seq_file *m, void *v, loff_t *pos)
799{ 883{
800 struct ftrace_iterator *iter = m->private; 884 struct ftrace_iterator *iter = m->private;
801 struct dyn_ftrace *rec = NULL; 885 struct dyn_ftrace *rec = NULL;
802 886
887 if (iter->flags & FTRACE_ITER_HASH)
888 return t_hash_next(m, v, pos);
889
803 (*pos)++; 890 (*pos)++;
804 891
805 /* should not be called from interrupt context */ 892 if (iter->flags & FTRACE_ITER_PRINTALL)
806 spin_lock(&ftrace_lock); 893 return NULL;
894
807 retry: 895 retry:
808 if (iter->idx >= iter->pg->index) { 896 if (iter->idx >= iter->pg->index) {
809 if (iter->pg->next) { 897 if (iter->pg->next) {
@@ -832,7 +920,6 @@ t_next(struct seq_file *m, void *v, loff_t *pos)
832 goto retry; 920 goto retry;
833 } 921 }
834 } 922 }
835 spin_unlock(&ftrace_lock);
836 923
837 return rec; 924 return rec;
838} 925}
@@ -842,6 +929,23 @@ static void *t_start(struct seq_file *m, loff_t *pos)
842 struct ftrace_iterator *iter = m->private; 929 struct ftrace_iterator *iter = m->private;
843 void *p = NULL; 930 void *p = NULL;
844 931
932 mutex_lock(&ftrace_lock);
933 /*
934 * For set_ftrace_filter reading, if we have the filter
935 * off, we can short cut and just print out that all
936 * functions are enabled.
937 */
938 if (iter->flags & FTRACE_ITER_FILTER && !ftrace_filtered) {
939 if (*pos > 0)
940 return t_hash_start(m, pos);
941 iter->flags |= FTRACE_ITER_PRINTALL;
942 (*pos)++;
943 return iter;
944 }
945
946 if (iter->flags & FTRACE_ITER_HASH)
947 return t_hash_start(m, pos);
948
845 if (*pos > 0) { 949 if (*pos > 0) {
846 if (iter->idx < 0) 950 if (iter->idx < 0)
847 return p; 951 return p;
@@ -851,18 +955,31 @@ static void *t_start(struct seq_file *m, loff_t *pos)
851 955
852 p = t_next(m, p, pos); 956 p = t_next(m, p, pos);
853 957
958 if (!p)
959 return t_hash_start(m, pos);
960
854 return p; 961 return p;
855} 962}
856 963
857static void t_stop(struct seq_file *m, void *p) 964static void t_stop(struct seq_file *m, void *p)
858{ 965{
966 mutex_unlock(&ftrace_lock);
859} 967}
860 968
861static int t_show(struct seq_file *m, void *v) 969static int t_show(struct seq_file *m, void *v)
862{ 970{
971 struct ftrace_iterator *iter = m->private;
863 struct dyn_ftrace *rec = v; 972 struct dyn_ftrace *rec = v;
864 char str[KSYM_SYMBOL_LEN]; 973 char str[KSYM_SYMBOL_LEN];
865 974
975 if (iter->flags & FTRACE_ITER_HASH)
976 return t_hash_show(m, v);
977
978 if (iter->flags & FTRACE_ITER_PRINTALL) {
979 seq_printf(m, "#### all functions enabled ####\n");
980 return 0;
981 }
982
866 if (!rec) 983 if (!rec)
867 return 0; 984 return 0;
868 985
@@ -941,23 +1058,16 @@ static void ftrace_filter_reset(int enable)
941 struct ftrace_page *pg; 1058 struct ftrace_page *pg;
942 struct dyn_ftrace *rec; 1059 struct dyn_ftrace *rec;
943 unsigned long type = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE; 1060 unsigned long type = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
944 unsigned i;
945 1061
946 /* should not be called from interrupt context */ 1062 mutex_lock(&ftrace_lock);
947 spin_lock(&ftrace_lock);
948 if (enable) 1063 if (enable)
949 ftrace_filtered = 0; 1064 ftrace_filtered = 0;
950 pg = ftrace_pages_start; 1065 do_for_each_ftrace_rec(pg, rec) {
951 while (pg) { 1066 if (rec->flags & FTRACE_FL_FAILED)
952 for (i = 0; i < pg->index; i++) { 1067 continue;
953 rec = &pg->records[i]; 1068 rec->flags &= ~type;
954 if (rec->flags & FTRACE_FL_FAILED) 1069 } while_for_each_ftrace_rec();
955 continue; 1070 mutex_unlock(&ftrace_lock);
956 rec->flags &= ~type;
957 }
958 pg = pg->next;
959 }
960 spin_unlock(&ftrace_lock);
961} 1071}
962 1072
963static int 1073static int
@@ -1038,86 +1148,536 @@ enum {
1038 MATCH_END_ONLY, 1148 MATCH_END_ONLY,
1039}; 1149};
1040 1150
1041static void 1151/*
1042ftrace_match(unsigned char *buff, int len, int enable) 1152 * (static function - no need for kernel doc)
1153 *
1154 * Pass in a buffer containing a glob and this function will
1155 * set search to point to the search part of the buffer and
1156 * return the type of search it is (see enum above).
1157 * This does modify buff.
1158 *
1159 * Returns enum type.
1160 * search returns the pointer to use for comparison.
1161 * not returns 1 if buff started with a '!'
1162 * 0 otherwise.
1163 */
1164static int
1165ftrace_setup_glob(char *buff, int len, char **search, int *not)
1043{ 1166{
1044 char str[KSYM_SYMBOL_LEN];
1045 char *search = NULL;
1046 struct ftrace_page *pg;
1047 struct dyn_ftrace *rec;
1048 int type = MATCH_FULL; 1167 int type = MATCH_FULL;
1049 unsigned long flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE; 1168 int i;
1050 unsigned i, match = 0, search_len = 0;
1051 int not = 0;
1052 1169
1053 if (buff[0] == '!') { 1170 if (buff[0] == '!') {
1054 not = 1; 1171 *not = 1;
1055 buff++; 1172 buff++;
1056 len--; 1173 len--;
1057 } 1174 } else
1175 *not = 0;
1176
1177 *search = buff;
1058 1178
1059 for (i = 0; i < len; i++) { 1179 for (i = 0; i < len; i++) {
1060 if (buff[i] == '*') { 1180 if (buff[i] == '*') {
1061 if (!i) { 1181 if (!i) {
1062 search = buff + i + 1; 1182 *search = buff + 1;
1063 type = MATCH_END_ONLY; 1183 type = MATCH_END_ONLY;
1064 search_len = len - (i + 1);
1065 } else { 1184 } else {
1066 if (type == MATCH_END_ONLY) { 1185 if (type == MATCH_END_ONLY)
1067 type = MATCH_MIDDLE_ONLY; 1186 type = MATCH_MIDDLE_ONLY;
1068 } else { 1187 else
1069 match = i;
1070 type = MATCH_FRONT_ONLY; 1188 type = MATCH_FRONT_ONLY;
1071 }
1072 buff[i] = 0; 1189 buff[i] = 0;
1073 break; 1190 break;
1074 } 1191 }
1075 } 1192 }
1076 } 1193 }
1077 1194
1078 /* should not be called from interrupt context */ 1195 return type;
1079 spin_lock(&ftrace_lock); 1196}
1080 if (enable) 1197
1081 ftrace_filtered = 1; 1198static int ftrace_match(char *str, char *regex, int len, int type)
1082 pg = ftrace_pages_start; 1199{
1083 while (pg) { 1200 int matched = 0;
1084 for (i = 0; i < pg->index; i++) { 1201 char *ptr;
1085 int matched = 0; 1202
1086 char *ptr; 1203 switch (type) {
1087 1204 case MATCH_FULL:
1088 rec = &pg->records[i]; 1205 if (strcmp(str, regex) == 0)
1089 if (rec->flags & FTRACE_FL_FAILED) 1206 matched = 1;
1207 break;
1208 case MATCH_FRONT_ONLY:
1209 if (strncmp(str, regex, len) == 0)
1210 matched = 1;
1211 break;
1212 case MATCH_MIDDLE_ONLY:
1213 if (strstr(str, regex))
1214 matched = 1;
1215 break;
1216 case MATCH_END_ONLY:
1217 ptr = strstr(str, regex);
1218 if (ptr && (ptr[len] == 0))
1219 matched = 1;
1220 break;
1221 }
1222
1223 return matched;
1224}
1225
1226static int
1227ftrace_match_record(struct dyn_ftrace *rec, char *regex, int len, int type)
1228{
1229 char str[KSYM_SYMBOL_LEN];
1230
1231 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
1232 return ftrace_match(str, regex, len, type);
1233}
1234
1235static void ftrace_match_records(char *buff, int len, int enable)
1236{
1237 unsigned int search_len;
1238 struct ftrace_page *pg;
1239 struct dyn_ftrace *rec;
1240 unsigned long flag;
1241 char *search;
1242 int type;
1243 int not;
1244
1245 flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1246 type = ftrace_setup_glob(buff, len, &search, &not);
1247
1248 search_len = strlen(search);
1249
1250 mutex_lock(&ftrace_lock);
1251 do_for_each_ftrace_rec(pg, rec) {
1252
1253 if (rec->flags & FTRACE_FL_FAILED)
1254 continue;
1255
1256 if (ftrace_match_record(rec, search, search_len, type)) {
1257 if (not)
1258 rec->flags &= ~flag;
1259 else
1260 rec->flags |= flag;
1261 }
1262 /*
1263 * Only enable filtering if we have a function that
1264 * is filtered on.
1265 */
1266 if (enable && (rec->flags & FTRACE_FL_FILTER))
1267 ftrace_filtered = 1;
1268 } while_for_each_ftrace_rec();
1269 mutex_unlock(&ftrace_lock);
1270}
1271
1272static int
1273ftrace_match_module_record(struct dyn_ftrace *rec, char *mod,
1274 char *regex, int len, int type)
1275{
1276 char str[KSYM_SYMBOL_LEN];
1277 char *modname;
1278
1279 kallsyms_lookup(rec->ip, NULL, NULL, &modname, str);
1280
1281 if (!modname || strcmp(modname, mod))
1282 return 0;
1283
1284 /* blank search means to match all funcs in the mod */
1285 if (len)
1286 return ftrace_match(str, regex, len, type);
1287 else
1288 return 1;
1289}
1290
1291static void ftrace_match_module_records(char *buff, char *mod, int enable)
1292{
1293 unsigned search_len = 0;
1294 struct ftrace_page *pg;
1295 struct dyn_ftrace *rec;
1296 int type = MATCH_FULL;
1297 char *search = buff;
1298 unsigned long flag;
1299 int not = 0;
1300
1301 flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1302
1303 /* blank or '*' mean the same */
1304 if (strcmp(buff, "*") == 0)
1305 buff[0] = 0;
1306
1307 /* handle the case of 'dont filter this module' */
1308 if (strcmp(buff, "!") == 0 || strcmp(buff, "!*") == 0) {
1309 buff[0] = 0;
1310 not = 1;
1311 }
1312
1313 if (strlen(buff)) {
1314 type = ftrace_setup_glob(buff, strlen(buff), &search, &not);
1315 search_len = strlen(search);
1316 }
1317
1318 mutex_lock(&ftrace_lock);
1319 do_for_each_ftrace_rec(pg, rec) {
1320
1321 if (rec->flags & FTRACE_FL_FAILED)
1322 continue;
1323
1324 if (ftrace_match_module_record(rec, mod,
1325 search, search_len, type)) {
1326 if (not)
1327 rec->flags &= ~flag;
1328 else
1329 rec->flags |= flag;
1330 }
1331 if (enable && (rec->flags & FTRACE_FL_FILTER))
1332 ftrace_filtered = 1;
1333
1334 } while_for_each_ftrace_rec();
1335 mutex_unlock(&ftrace_lock);
1336}
1337
1338/*
1339 * We register the module command as a template to show others how
1340 * to register the a command as well.
1341 */
1342
1343static int
1344ftrace_mod_callback(char *func, char *cmd, char *param, int enable)
1345{
1346 char *mod;
1347
1348 /*
1349 * cmd == 'mod' because we only registered this func
1350 * for the 'mod' ftrace_func_command.
1351 * But if you register one func with multiple commands,
1352 * you can tell which command was used by the cmd
1353 * parameter.
1354 */
1355
1356 /* we must have a module name */
1357 if (!param)
1358 return -EINVAL;
1359
1360 mod = strsep(&param, ":");
1361 if (!strlen(mod))
1362 return -EINVAL;
1363
1364 ftrace_match_module_records(func, mod, enable);
1365 return 0;
1366}
1367
1368static struct ftrace_func_command ftrace_mod_cmd = {
1369 .name = "mod",
1370 .func = ftrace_mod_callback,
1371};
1372
1373static int __init ftrace_mod_cmd_init(void)
1374{
1375 return register_ftrace_command(&ftrace_mod_cmd);
1376}
1377device_initcall(ftrace_mod_cmd_init);
1378
1379static void
1380function_trace_probe_call(unsigned long ip, unsigned long parent_ip)
1381{
1382 struct ftrace_func_probe *entry;
1383 struct hlist_head *hhd;
1384 struct hlist_node *n;
1385 unsigned long key;
1386 int resched;
1387
1388 key = hash_long(ip, FTRACE_HASH_BITS);
1389
1390 hhd = &ftrace_func_hash[key];
1391
1392 if (hlist_empty(hhd))
1393 return;
1394
1395 /*
1396 * Disable preemption for these calls to prevent a RCU grace
1397 * period. This syncs the hash iteration and freeing of items
1398 * on the hash. rcu_read_lock is too dangerous here.
1399 */
1400 resched = ftrace_preempt_disable();
1401 hlist_for_each_entry_rcu(entry, n, hhd, node) {
1402 if (entry->ip == ip)
1403 entry->ops->func(ip, parent_ip, &entry->data);
1404 }
1405 ftrace_preempt_enable(resched);
1406}
1407
1408static struct ftrace_ops trace_probe_ops __read_mostly =
1409{
1410 .func = function_trace_probe_call,
1411};
1412
1413static int ftrace_probe_registered;
1414
1415static void __enable_ftrace_function_probe(void)
1416{
1417 int i;
1418
1419 if (ftrace_probe_registered)
1420 return;
1421
1422 for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
1423 struct hlist_head *hhd = &ftrace_func_hash[i];
1424 if (hhd->first)
1425 break;
1426 }
1427 /* Nothing registered? */
1428 if (i == FTRACE_FUNC_HASHSIZE)
1429 return;
1430
1431 __register_ftrace_function(&trace_probe_ops);
1432 ftrace_startup(0);
1433 ftrace_probe_registered = 1;
1434}
1435
1436static void __disable_ftrace_function_probe(void)
1437{
1438 int i;
1439
1440 if (!ftrace_probe_registered)
1441 return;
1442
1443 for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
1444 struct hlist_head *hhd = &ftrace_func_hash[i];
1445 if (hhd->first)
1446 return;
1447 }
1448
1449 /* no more funcs left */
1450 __unregister_ftrace_function(&trace_probe_ops);
1451 ftrace_shutdown(0);
1452 ftrace_probe_registered = 0;
1453}
1454
1455
1456static void ftrace_free_entry_rcu(struct rcu_head *rhp)
1457{
1458 struct ftrace_func_probe *entry =
1459 container_of(rhp, struct ftrace_func_probe, rcu);
1460
1461 if (entry->ops->free)
1462 entry->ops->free(&entry->data);
1463 kfree(entry);
1464}
1465
1466
1467int
1468register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
1469 void *data)
1470{
1471 struct ftrace_func_probe *entry;
1472 struct ftrace_page *pg;
1473 struct dyn_ftrace *rec;
1474 int type, len, not;
1475 unsigned long key;
1476 int count = 0;
1477 char *search;
1478
1479 type = ftrace_setup_glob(glob, strlen(glob), &search, &not);
1480 len = strlen(search);
1481
1482 /* we do not support '!' for function probes */
1483 if (WARN_ON(not))
1484 return -EINVAL;
1485
1486 mutex_lock(&ftrace_lock);
1487 do_for_each_ftrace_rec(pg, rec) {
1488
1489 if (rec->flags & FTRACE_FL_FAILED)
1490 continue;
1491
1492 if (!ftrace_match_record(rec, search, len, type))
1493 continue;
1494
1495 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
1496 if (!entry) {
1497 /* If we did not process any, then return error */
1498 if (!count)
1499 count = -ENOMEM;
1500 goto out_unlock;
1501 }
1502
1503 count++;
1504
1505 entry->data = data;
1506
1507 /*
1508 * The caller might want to do something special
1509 * for each function we find. We call the callback
1510 * to give the caller an opportunity to do so.
1511 */
1512 if (ops->callback) {
1513 if (ops->callback(rec->ip, &entry->data) < 0) {
1514 /* caller does not like this func */
1515 kfree(entry);
1090 continue; 1516 continue;
1091 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
1092 switch (type) {
1093 case MATCH_FULL:
1094 if (strcmp(str, buff) == 0)
1095 matched = 1;
1096 break;
1097 case MATCH_FRONT_ONLY:
1098 if (memcmp(str, buff, match) == 0)
1099 matched = 1;
1100 break;
1101 case MATCH_MIDDLE_ONLY:
1102 if (strstr(str, search))
1103 matched = 1;
1104 break;
1105 case MATCH_END_ONLY:
1106 ptr = strstr(str, search);
1107 if (ptr && (ptr[search_len] == 0))
1108 matched = 1;
1109 break;
1110 } 1517 }
1111 if (matched) { 1518 }
1112 if (not) 1519
1113 rec->flags &= ~flag; 1520 entry->ops = ops;
1114 else 1521 entry->ip = rec->ip;
1115 rec->flags |= flag; 1522
1523 key = hash_long(entry->ip, FTRACE_HASH_BITS);
1524 hlist_add_head_rcu(&entry->node, &ftrace_func_hash[key]);
1525
1526 } while_for_each_ftrace_rec();
1527 __enable_ftrace_function_probe();
1528
1529 out_unlock:
1530 mutex_unlock(&ftrace_lock);
1531
1532 return count;
1533}
1534
1535enum {
1536 PROBE_TEST_FUNC = 1,
1537 PROBE_TEST_DATA = 2
1538};
1539
1540static void
1541__unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
1542 void *data, int flags)
1543{
1544 struct ftrace_func_probe *entry;
1545 struct hlist_node *n, *tmp;
1546 char str[KSYM_SYMBOL_LEN];
1547 int type = MATCH_FULL;
1548 int i, len = 0;
1549 char *search;
1550
1551 if (glob && (strcmp(glob, "*") || !strlen(glob)))
1552 glob = NULL;
1553 else {
1554 int not;
1555
1556 type = ftrace_setup_glob(glob, strlen(glob), &search, &not);
1557 len = strlen(search);
1558
1559 /* we do not support '!' for function probes */
1560 if (WARN_ON(not))
1561 return;
1562 }
1563
1564 mutex_lock(&ftrace_lock);
1565 for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
1566 struct hlist_head *hhd = &ftrace_func_hash[i];
1567
1568 hlist_for_each_entry_safe(entry, n, tmp, hhd, node) {
1569
1570 /* break up if statements for readability */
1571 if ((flags & PROBE_TEST_FUNC) && entry->ops != ops)
1572 continue;
1573
1574 if ((flags & PROBE_TEST_DATA) && entry->data != data)
1575 continue;
1576
1577 /* do this last, since it is the most expensive */
1578 if (glob) {
1579 kallsyms_lookup(entry->ip, NULL, NULL,
1580 NULL, str);
1581 if (!ftrace_match(str, glob, len, type))
1582 continue;
1116 } 1583 }
1584
1585 hlist_del(&entry->node);
1586 call_rcu(&entry->rcu, ftrace_free_entry_rcu);
1587 }
1588 }
1589 __disable_ftrace_function_probe();
1590 mutex_unlock(&ftrace_lock);
1591}
1592
1593void
1594unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
1595 void *data)
1596{
1597 __unregister_ftrace_function_probe(glob, ops, data,
1598 PROBE_TEST_FUNC | PROBE_TEST_DATA);
1599}
1600
1601void
1602unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops)
1603{
1604 __unregister_ftrace_function_probe(glob, ops, NULL, PROBE_TEST_FUNC);
1605}
1606
1607void unregister_ftrace_function_probe_all(char *glob)
1608{
1609 __unregister_ftrace_function_probe(glob, NULL, NULL, 0);
1610}
1611
1612static LIST_HEAD(ftrace_commands);
1613static DEFINE_MUTEX(ftrace_cmd_mutex);
1614
1615int register_ftrace_command(struct ftrace_func_command *cmd)
1616{
1617 struct ftrace_func_command *p;
1618 int ret = 0;
1619
1620 mutex_lock(&ftrace_cmd_mutex);
1621 list_for_each_entry(p, &ftrace_commands, list) {
1622 if (strcmp(cmd->name, p->name) == 0) {
1623 ret = -EBUSY;
1624 goto out_unlock;
1625 }
1626 }
1627 list_add(&cmd->list, &ftrace_commands);
1628 out_unlock:
1629 mutex_unlock(&ftrace_cmd_mutex);
1630
1631 return ret;
1632}
1633
1634int unregister_ftrace_command(struct ftrace_func_command *cmd)
1635{
1636 struct ftrace_func_command *p, *n;
1637 int ret = -ENODEV;
1638
1639 mutex_lock(&ftrace_cmd_mutex);
1640 list_for_each_entry_safe(p, n, &ftrace_commands, list) {
1641 if (strcmp(cmd->name, p->name) == 0) {
1642 ret = 0;
1643 list_del_init(&p->list);
1644 goto out_unlock;
1645 }
1646 }
1647 out_unlock:
1648 mutex_unlock(&ftrace_cmd_mutex);
1649
1650 return ret;
1651}
1652
1653static int ftrace_process_regex(char *buff, int len, int enable)
1654{
1655 char *func, *command, *next = buff;
1656 struct ftrace_func_command *p;
1657 int ret = -EINVAL;
1658
1659 func = strsep(&next, ":");
1660
1661 if (!next) {
1662 ftrace_match_records(func, len, enable);
1663 return 0;
1664 }
1665
1666 /* command found */
1667
1668 command = strsep(&next, ":");
1669
1670 mutex_lock(&ftrace_cmd_mutex);
1671 list_for_each_entry(p, &ftrace_commands, list) {
1672 if (strcmp(p->name, command) == 0) {
1673 ret = p->func(func, command, next, enable);
1674 goto out_unlock;
1117 } 1675 }
1118 pg = pg->next;
1119 } 1676 }
1120 spin_unlock(&ftrace_lock); 1677 out_unlock:
1678 mutex_unlock(&ftrace_cmd_mutex);
1679
1680 return ret;
1121} 1681}
1122 1682
1123static ssize_t 1683static ssize_t
@@ -1187,7 +1747,10 @@ ftrace_regex_write(struct file *file, const char __user *ubuf,
1187 if (isspace(ch)) { 1747 if (isspace(ch)) {
1188 iter->filtered++; 1748 iter->filtered++;
1189 iter->buffer[iter->buffer_idx] = 0; 1749 iter->buffer[iter->buffer_idx] = 0;
1190 ftrace_match(iter->buffer, iter->buffer_idx, enable); 1750 ret = ftrace_process_regex(iter->buffer,
1751 iter->buffer_idx, enable);
1752 if (ret)
1753 goto out;
1191 iter->buffer_idx = 0; 1754 iter->buffer_idx = 0;
1192 } else 1755 } else
1193 iter->flags |= FTRACE_ITER_CONT; 1756 iter->flags |= FTRACE_ITER_CONT;
@@ -1226,7 +1789,7 @@ ftrace_set_regex(unsigned char *buf, int len, int reset, int enable)
1226 if (reset) 1789 if (reset)
1227 ftrace_filter_reset(enable); 1790 ftrace_filter_reset(enable);
1228 if (buf) 1791 if (buf)
1229 ftrace_match(buf, len, enable); 1792 ftrace_match_records(buf, len, enable);
1230 mutex_unlock(&ftrace_regex_lock); 1793 mutex_unlock(&ftrace_regex_lock);
1231} 1794}
1232 1795
@@ -1276,15 +1839,13 @@ ftrace_regex_release(struct inode *inode, struct file *file, int enable)
1276 if (iter->buffer_idx) { 1839 if (iter->buffer_idx) {
1277 iter->filtered++; 1840 iter->filtered++;
1278 iter->buffer[iter->buffer_idx] = 0; 1841 iter->buffer[iter->buffer_idx] = 0;
1279 ftrace_match(iter->buffer, iter->buffer_idx, enable); 1842 ftrace_match_records(iter->buffer, iter->buffer_idx, enable);
1280 } 1843 }
1281 1844
1282 mutex_lock(&ftrace_sysctl_lock); 1845 mutex_lock(&ftrace_lock);
1283 mutex_lock(&ftrace_start_lock);
1284 if (ftrace_start_up && ftrace_enabled) 1846 if (ftrace_start_up && ftrace_enabled)
1285 ftrace_run_update_code(FTRACE_ENABLE_CALLS); 1847 ftrace_run_update_code(FTRACE_ENABLE_CALLS);
1286 mutex_unlock(&ftrace_start_lock); 1848 mutex_unlock(&ftrace_lock);
1287 mutex_unlock(&ftrace_sysctl_lock);
1288 1849
1289 kfree(iter); 1850 kfree(iter);
1290 mutex_unlock(&ftrace_regex_lock); 1851 mutex_unlock(&ftrace_regex_lock);
@@ -1303,21 +1864,21 @@ ftrace_notrace_release(struct inode *inode, struct file *file)
1303 return ftrace_regex_release(inode, file, 0); 1864 return ftrace_regex_release(inode, file, 0);
1304} 1865}
1305 1866
1306static struct file_operations ftrace_avail_fops = { 1867static const struct file_operations ftrace_avail_fops = {
1307 .open = ftrace_avail_open, 1868 .open = ftrace_avail_open,
1308 .read = seq_read, 1869 .read = seq_read,
1309 .llseek = seq_lseek, 1870 .llseek = seq_lseek,
1310 .release = ftrace_avail_release, 1871 .release = ftrace_avail_release,
1311}; 1872};
1312 1873
1313static struct file_operations ftrace_failures_fops = { 1874static const struct file_operations ftrace_failures_fops = {
1314 .open = ftrace_failures_open, 1875 .open = ftrace_failures_open,
1315 .read = seq_read, 1876 .read = seq_read,
1316 .llseek = seq_lseek, 1877 .llseek = seq_lseek,
1317 .release = ftrace_avail_release, 1878 .release = ftrace_avail_release,
1318}; 1879};
1319 1880
1320static struct file_operations ftrace_filter_fops = { 1881static const struct file_operations ftrace_filter_fops = {
1321 .open = ftrace_filter_open, 1882 .open = ftrace_filter_open,
1322 .read = ftrace_regex_read, 1883 .read = ftrace_regex_read,
1323 .write = ftrace_filter_write, 1884 .write = ftrace_filter_write,
@@ -1325,7 +1886,7 @@ static struct file_operations ftrace_filter_fops = {
1325 .release = ftrace_filter_release, 1886 .release = ftrace_filter_release,
1326}; 1887};
1327 1888
1328static struct file_operations ftrace_notrace_fops = { 1889static const struct file_operations ftrace_notrace_fops = {
1329 .open = ftrace_notrace_open, 1890 .open = ftrace_notrace_open,
1330 .read = ftrace_regex_read, 1891 .read = ftrace_regex_read,
1331 .write = ftrace_notrace_write, 1892 .write = ftrace_notrace_write,
@@ -1360,6 +1921,10 @@ static void *g_start(struct seq_file *m, loff_t *pos)
1360 1921
1361 mutex_lock(&graph_lock); 1922 mutex_lock(&graph_lock);
1362 1923
1924 /* Nothing, tell g_show to print all functions are enabled */
1925 if (!ftrace_graph_count && !*pos)
1926 return (void *)1;
1927
1363 p = g_next(m, p, pos); 1928 p = g_next(m, p, pos);
1364 1929
1365 return p; 1930 return p;
@@ -1378,6 +1943,11 @@ static int g_show(struct seq_file *m, void *v)
1378 if (!ptr) 1943 if (!ptr)
1379 return 0; 1944 return 0;
1380 1945
1946 if (ptr == (unsigned long *)1) {
1947 seq_printf(m, "#### all functions enabled ####\n");
1948 return 0;
1949 }
1950
1381 kallsyms_lookup(*ptr, NULL, NULL, NULL, str); 1951 kallsyms_lookup(*ptr, NULL, NULL, NULL, str);
1382 1952
1383 seq_printf(m, "%s\n", str); 1953 seq_printf(m, "%s\n", str);
@@ -1431,42 +2001,52 @@ ftrace_graph_read(struct file *file, char __user *ubuf,
1431} 2001}
1432 2002
1433static int 2003static int
1434ftrace_set_func(unsigned long *array, int idx, char *buffer) 2004ftrace_set_func(unsigned long *array, int *idx, char *buffer)
1435{ 2005{
1436 char str[KSYM_SYMBOL_LEN];
1437 struct dyn_ftrace *rec; 2006 struct dyn_ftrace *rec;
1438 struct ftrace_page *pg; 2007 struct ftrace_page *pg;
2008 int search_len;
1439 int found = 0; 2009 int found = 0;
1440 int i, j; 2010 int type, not;
2011 char *search;
2012 bool exists;
2013 int i;
1441 2014
1442 if (ftrace_disabled) 2015 if (ftrace_disabled)
1443 return -ENODEV; 2016 return -ENODEV;
1444 2017
1445 /* should not be called from interrupt context */ 2018 /* decode regex */
1446 spin_lock(&ftrace_lock); 2019 type = ftrace_setup_glob(buffer, strlen(buffer), &search, &not);
2020 if (not)
2021 return -EINVAL;
1447 2022
1448 for (pg = ftrace_pages_start; pg; pg = pg->next) { 2023 search_len = strlen(search);
1449 for (i = 0; i < pg->index; i++) {
1450 rec = &pg->records[i];
1451 2024
1452 if (rec->flags & (FTRACE_FL_FAILED | FTRACE_FL_FREE)) 2025 mutex_lock(&ftrace_lock);
1453 continue; 2026 do_for_each_ftrace_rec(pg, rec) {
2027
2028 if (*idx >= FTRACE_GRAPH_MAX_FUNCS)
2029 break;
2030
2031 if (rec->flags & (FTRACE_FL_FAILED | FTRACE_FL_FREE))
2032 continue;
1454 2033
1455 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str); 2034 if (ftrace_match_record(rec, search, search_len, type)) {
1456 if (strcmp(str, buffer) == 0) { 2035 /* ensure it is not already in the array */
2036 exists = false;
2037 for (i = 0; i < *idx; i++)
2038 if (array[i] == rec->ip) {
2039 exists = true;
2040 break;
2041 }
2042 if (!exists) {
2043 array[(*idx)++] = rec->ip;
1457 found = 1; 2044 found = 1;
1458 for (j = 0; j < idx; j++)
1459 if (array[j] == rec->ip) {
1460 found = 0;
1461 break;
1462 }
1463 if (found)
1464 array[idx] = rec->ip;
1465 break;
1466 } 2045 }
1467 } 2046 }
1468 } 2047 } while_for_each_ftrace_rec();
1469 spin_unlock(&ftrace_lock); 2048
2049 mutex_unlock(&ftrace_lock);
1470 2050
1471 return found ? 0 : -EINVAL; 2051 return found ? 0 : -EINVAL;
1472} 2052}
@@ -1534,13 +2114,11 @@ ftrace_graph_write(struct file *file, const char __user *ubuf,
1534 } 2114 }
1535 buffer[index] = 0; 2115 buffer[index] = 0;
1536 2116
1537 /* we allow only one at a time */ 2117 /* we allow only one expression at a time */
1538 ret = ftrace_set_func(array, ftrace_graph_count, buffer); 2118 ret = ftrace_set_func(array, &ftrace_graph_count, buffer);
1539 if (ret) 2119 if (ret)
1540 goto out; 2120 goto out;
1541 2121
1542 ftrace_graph_count++;
1543
1544 file->f_pos += read; 2122 file->f_pos += read;
1545 2123
1546 ret = read; 2124 ret = read;
@@ -1604,7 +2182,7 @@ static int ftrace_convert_nops(struct module *mod,
1604 unsigned long addr; 2182 unsigned long addr;
1605 unsigned long flags; 2183 unsigned long flags;
1606 2184
1607 mutex_lock(&ftrace_start_lock); 2185 mutex_lock(&ftrace_lock);
1608 p = start; 2186 p = start;
1609 while (p < end) { 2187 while (p < end) {
1610 addr = ftrace_call_adjust(*p++); 2188 addr = ftrace_call_adjust(*p++);
@@ -1623,7 +2201,7 @@ static int ftrace_convert_nops(struct module *mod,
1623 local_irq_save(flags); 2201 local_irq_save(flags);
1624 ftrace_update_code(mod); 2202 ftrace_update_code(mod);
1625 local_irq_restore(flags); 2203 local_irq_restore(flags);
1626 mutex_unlock(&ftrace_start_lock); 2204 mutex_unlock(&ftrace_lock);
1627 2205
1628 return 0; 2206 return 0;
1629} 2207}
@@ -1796,7 +2374,7 @@ ftrace_pid_write(struct file *filp, const char __user *ubuf,
1796 if (ret < 0) 2374 if (ret < 0)
1797 return ret; 2375 return ret;
1798 2376
1799 mutex_lock(&ftrace_start_lock); 2377 mutex_lock(&ftrace_lock);
1800 if (val < 0) { 2378 if (val < 0) {
1801 /* disable pid tracing */ 2379 /* disable pid tracing */
1802 if (!ftrace_pid_trace) 2380 if (!ftrace_pid_trace)
@@ -1835,12 +2413,12 @@ ftrace_pid_write(struct file *filp, const char __user *ubuf,
1835 ftrace_startup_enable(0); 2413 ftrace_startup_enable(0);
1836 2414
1837 out: 2415 out:
1838 mutex_unlock(&ftrace_start_lock); 2416 mutex_unlock(&ftrace_lock);
1839 2417
1840 return cnt; 2418 return cnt;
1841} 2419}
1842 2420
1843static struct file_operations ftrace_pid_fops = { 2421static const struct file_operations ftrace_pid_fops = {
1844 .read = ftrace_pid_read, 2422 .read = ftrace_pid_read,
1845 .write = ftrace_pid_write, 2423 .write = ftrace_pid_write,
1846}; 2424};
@@ -1863,7 +2441,6 @@ static __init int ftrace_init_debugfs(void)
1863 "'set_ftrace_pid' entry\n"); 2441 "'set_ftrace_pid' entry\n");
1864 return 0; 2442 return 0;
1865} 2443}
1866
1867fs_initcall(ftrace_init_debugfs); 2444fs_initcall(ftrace_init_debugfs);
1868 2445
1869/** 2446/**
@@ -1898,17 +2475,17 @@ int register_ftrace_function(struct ftrace_ops *ops)
1898 if (unlikely(ftrace_disabled)) 2475 if (unlikely(ftrace_disabled))
1899 return -1; 2476 return -1;
1900 2477
1901 mutex_lock(&ftrace_sysctl_lock); 2478 mutex_lock(&ftrace_lock);
1902 2479
1903 ret = __register_ftrace_function(ops); 2480 ret = __register_ftrace_function(ops);
1904 ftrace_startup(0); 2481 ftrace_startup(0);
1905 2482
1906 mutex_unlock(&ftrace_sysctl_lock); 2483 mutex_unlock(&ftrace_lock);
1907 return ret; 2484 return ret;
1908} 2485}
1909 2486
1910/** 2487/**
1911 * unregister_ftrace_function - unresgister a function for profiling. 2488 * unregister_ftrace_function - unregister a function for profiling.
1912 * @ops - ops structure that holds the function to unregister 2489 * @ops - ops structure that holds the function to unregister
1913 * 2490 *
1914 * Unregister a function that was added to be called by ftrace profiling. 2491 * Unregister a function that was added to be called by ftrace profiling.
@@ -1917,10 +2494,10 @@ int unregister_ftrace_function(struct ftrace_ops *ops)
1917{ 2494{
1918 int ret; 2495 int ret;
1919 2496
1920 mutex_lock(&ftrace_sysctl_lock); 2497 mutex_lock(&ftrace_lock);
1921 ret = __unregister_ftrace_function(ops); 2498 ret = __unregister_ftrace_function(ops);
1922 ftrace_shutdown(0); 2499 ftrace_shutdown(0);
1923 mutex_unlock(&ftrace_sysctl_lock); 2500 mutex_unlock(&ftrace_lock);
1924 2501
1925 return ret; 2502 return ret;
1926} 2503}
@@ -1935,7 +2512,7 @@ ftrace_enable_sysctl(struct ctl_table *table, int write,
1935 if (unlikely(ftrace_disabled)) 2512 if (unlikely(ftrace_disabled))
1936 return -ENODEV; 2513 return -ENODEV;
1937 2514
1938 mutex_lock(&ftrace_sysctl_lock); 2515 mutex_lock(&ftrace_lock);
1939 2516
1940 ret = proc_dointvec(table, write, file, buffer, lenp, ppos); 2517 ret = proc_dointvec(table, write, file, buffer, lenp, ppos);
1941 2518
@@ -1964,7 +2541,7 @@ ftrace_enable_sysctl(struct ctl_table *table, int write,
1964 } 2541 }
1965 2542
1966 out: 2543 out:
1967 mutex_unlock(&ftrace_sysctl_lock); 2544 mutex_unlock(&ftrace_lock);
1968 return ret; 2545 return ret;
1969} 2546}
1970 2547
@@ -2080,7 +2657,7 @@ int register_ftrace_graph(trace_func_graph_ret_t retfunc,
2080{ 2657{
2081 int ret = 0; 2658 int ret = 0;
2082 2659
2083 mutex_lock(&ftrace_sysctl_lock); 2660 mutex_lock(&ftrace_lock);
2084 2661
2085 ftrace_suspend_notifier.notifier_call = ftrace_suspend_notifier_call; 2662 ftrace_suspend_notifier.notifier_call = ftrace_suspend_notifier_call;
2086 register_pm_notifier(&ftrace_suspend_notifier); 2663 register_pm_notifier(&ftrace_suspend_notifier);
@@ -2098,13 +2675,13 @@ int register_ftrace_graph(trace_func_graph_ret_t retfunc,
2098 ftrace_startup(FTRACE_START_FUNC_RET); 2675 ftrace_startup(FTRACE_START_FUNC_RET);
2099 2676
2100out: 2677out:
2101 mutex_unlock(&ftrace_sysctl_lock); 2678 mutex_unlock(&ftrace_lock);
2102 return ret; 2679 return ret;
2103} 2680}
2104 2681
2105void unregister_ftrace_graph(void) 2682void unregister_ftrace_graph(void)
2106{ 2683{
2107 mutex_lock(&ftrace_sysctl_lock); 2684 mutex_lock(&ftrace_lock);
2108 2685
2109 atomic_dec(&ftrace_graph_active); 2686 atomic_dec(&ftrace_graph_active);
2110 ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub; 2687 ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
@@ -2112,7 +2689,7 @@ void unregister_ftrace_graph(void)
2112 ftrace_shutdown(FTRACE_STOP_FUNC_RET); 2689 ftrace_shutdown(FTRACE_STOP_FUNC_RET);
2113 unregister_pm_notifier(&ftrace_suspend_notifier); 2690 unregister_pm_notifier(&ftrace_suspend_notifier);
2114 2691
2115 mutex_unlock(&ftrace_sysctl_lock); 2692 mutex_unlock(&ftrace_lock);
2116} 2693}
2117 2694
2118/* Allocate a return stack for newly created task */ 2695/* Allocate a return stack for newly created task */
diff --git a/kernel/trace/kmemtrace.c b/kernel/trace/kmemtrace.c
new file mode 100644
index 000000000000..ae201b3eda89
--- /dev/null
+++ b/kernel/trace/kmemtrace.c
@@ -0,0 +1,339 @@
1/*
2 * Memory allocator tracing
3 *
4 * Copyright (C) 2008 Eduard - Gabriel Munteanu
5 * Copyright (C) 2008 Pekka Enberg <penberg@cs.helsinki.fi>
6 * Copyright (C) 2008 Frederic Weisbecker <fweisbec@gmail.com>
7 */
8
9#include <linux/dcache.h>
10#include <linux/debugfs.h>
11#include <linux/fs.h>
12#include <linux/seq_file.h>
13#include <trace/kmemtrace.h>
14
15#include "trace.h"
16#include "trace_output.h"
17
18/* Select an alternative, minimalistic output than the original one */
19#define TRACE_KMEM_OPT_MINIMAL 0x1
20
21static struct tracer_opt kmem_opts[] = {
22 /* Default disable the minimalistic output */
23 { TRACER_OPT(kmem_minimalistic, TRACE_KMEM_OPT_MINIMAL) },
24 { }
25};
26
27static struct tracer_flags kmem_tracer_flags = {
28 .val = 0,
29 .opts = kmem_opts
30};
31
32
33static bool kmem_tracing_enabled __read_mostly;
34static struct trace_array *kmemtrace_array;
35
36static int kmem_trace_init(struct trace_array *tr)
37{
38 int cpu;
39 kmemtrace_array = tr;
40
41 for_each_cpu_mask(cpu, cpu_possible_map)
42 tracing_reset(tr, cpu);
43
44 kmem_tracing_enabled = true;
45
46 return 0;
47}
48
49static void kmem_trace_reset(struct trace_array *tr)
50{
51 kmem_tracing_enabled = false;
52}
53
54static void kmemtrace_headers(struct seq_file *s)
55{
56 /* Don't need headers for the original kmemtrace output */
57 if (!(kmem_tracer_flags.val & TRACE_KMEM_OPT_MINIMAL))
58 return;
59
60 seq_printf(s, "#\n");
61 seq_printf(s, "# ALLOC TYPE REQ GIVEN FLAGS "
62 " POINTER NODE CALLER\n");
63 seq_printf(s, "# FREE | | | | "
64 " | | | |\n");
65 seq_printf(s, "# |\n\n");
66}
67
68/*
69 * The two following functions give the original output from kmemtrace,
70 * or something close to....perhaps they need some missing things
71 */
72static enum print_line_t
73kmemtrace_print_alloc_original(struct trace_iterator *iter,
74 struct kmemtrace_alloc_entry *entry)
75{
76 struct trace_seq *s = &iter->seq;
77 int ret;
78
79 /* Taken from the old linux/kmemtrace.h */
80 ret = trace_seq_printf(s, "type_id %d call_site %lu ptr %lu "
81 "bytes_req %lu bytes_alloc %lu gfp_flags %lu node %d\n",
82 entry->type_id, entry->call_site, (unsigned long) entry->ptr,
83 (unsigned long) entry->bytes_req, (unsigned long) entry->bytes_alloc,
84 (unsigned long) entry->gfp_flags, entry->node);
85
86 if (!ret)
87 return TRACE_TYPE_PARTIAL_LINE;
88
89 return TRACE_TYPE_HANDLED;
90}
91
92static enum print_line_t
93kmemtrace_print_free_original(struct trace_iterator *iter,
94 struct kmemtrace_free_entry *entry)
95{
96 struct trace_seq *s = &iter->seq;
97 int ret;
98
99 /* Taken from the old linux/kmemtrace.h */
100 ret = trace_seq_printf(s, "type_id %d call_site %lu ptr %lu\n",
101 entry->type_id, entry->call_site, (unsigned long) entry->ptr);
102
103 if (!ret)
104 return TRACE_TYPE_PARTIAL_LINE;
105
106 return TRACE_TYPE_HANDLED;
107}
108
109
110/* The two other following provide a more minimalistic output */
111static enum print_line_t
112kmemtrace_print_alloc_compress(struct trace_iterator *iter,
113 struct kmemtrace_alloc_entry *entry)
114{
115 struct trace_seq *s = &iter->seq;
116 int ret;
117
118 /* Alloc entry */
119 ret = trace_seq_printf(s, " + ");
120 if (!ret)
121 return TRACE_TYPE_PARTIAL_LINE;
122
123 /* Type */
124 switch (entry->type_id) {
125 case KMEMTRACE_TYPE_KMALLOC:
126 ret = trace_seq_printf(s, "K ");
127 break;
128 case KMEMTRACE_TYPE_CACHE:
129 ret = trace_seq_printf(s, "C ");
130 break;
131 case KMEMTRACE_TYPE_PAGES:
132 ret = trace_seq_printf(s, "P ");
133 break;
134 default:
135 ret = trace_seq_printf(s, "? ");
136 }
137
138 if (!ret)
139 return TRACE_TYPE_PARTIAL_LINE;
140
141 /* Requested */
142 ret = trace_seq_printf(s, "%4zu ", entry->bytes_req);
143 if (!ret)
144 return TRACE_TYPE_PARTIAL_LINE;
145
146 /* Allocated */
147 ret = trace_seq_printf(s, "%4zu ", entry->bytes_alloc);
148 if (!ret)
149 return TRACE_TYPE_PARTIAL_LINE;
150
151 /* Flags
152 * TODO: would be better to see the name of the GFP flag names
153 */
154 ret = trace_seq_printf(s, "%08x ", entry->gfp_flags);
155 if (!ret)
156 return TRACE_TYPE_PARTIAL_LINE;
157
158 /* Pointer to allocated */
159 ret = trace_seq_printf(s, "0x%tx ", (ptrdiff_t)entry->ptr);
160 if (!ret)
161 return TRACE_TYPE_PARTIAL_LINE;
162
163 /* Node */
164 ret = trace_seq_printf(s, "%4d ", entry->node);
165 if (!ret)
166 return TRACE_TYPE_PARTIAL_LINE;
167
168 /* Call site */
169 ret = seq_print_ip_sym(s, entry->call_site, 0);
170 if (!ret)
171 return TRACE_TYPE_PARTIAL_LINE;
172
173 if (!trace_seq_printf(s, "\n"))
174 return TRACE_TYPE_PARTIAL_LINE;
175
176 return TRACE_TYPE_HANDLED;
177}
178
179static enum print_line_t
180kmemtrace_print_free_compress(struct trace_iterator *iter,
181 struct kmemtrace_free_entry *entry)
182{
183 struct trace_seq *s = &iter->seq;
184 int ret;
185
186 /* Free entry */
187 ret = trace_seq_printf(s, " - ");
188 if (!ret)
189 return TRACE_TYPE_PARTIAL_LINE;
190
191 /* Type */
192 switch (entry->type_id) {
193 case KMEMTRACE_TYPE_KMALLOC:
194 ret = trace_seq_printf(s, "K ");
195 break;
196 case KMEMTRACE_TYPE_CACHE:
197 ret = trace_seq_printf(s, "C ");
198 break;
199 case KMEMTRACE_TYPE_PAGES:
200 ret = trace_seq_printf(s, "P ");
201 break;
202 default:
203 ret = trace_seq_printf(s, "? ");
204 }
205
206 if (!ret)
207 return TRACE_TYPE_PARTIAL_LINE;
208
209 /* Skip requested/allocated/flags */
210 ret = trace_seq_printf(s, " ");
211 if (!ret)
212 return TRACE_TYPE_PARTIAL_LINE;
213
214 /* Pointer to allocated */
215 ret = trace_seq_printf(s, "0x%tx ", (ptrdiff_t)entry->ptr);
216 if (!ret)
217 return TRACE_TYPE_PARTIAL_LINE;
218
219 /* Skip node */
220 ret = trace_seq_printf(s, " ");
221 if (!ret)
222 return TRACE_TYPE_PARTIAL_LINE;
223
224 /* Call site */
225 ret = seq_print_ip_sym(s, entry->call_site, 0);
226 if (!ret)
227 return TRACE_TYPE_PARTIAL_LINE;
228
229 if (!trace_seq_printf(s, "\n"))
230 return TRACE_TYPE_PARTIAL_LINE;
231
232 return TRACE_TYPE_HANDLED;
233}
234
235static enum print_line_t kmemtrace_print_line(struct trace_iterator *iter)
236{
237 struct trace_entry *entry = iter->ent;
238
239 switch (entry->type) {
240 case TRACE_KMEM_ALLOC: {
241 struct kmemtrace_alloc_entry *field;
242 trace_assign_type(field, entry);
243 if (kmem_tracer_flags.val & TRACE_KMEM_OPT_MINIMAL)
244 return kmemtrace_print_alloc_compress(iter, field);
245 else
246 return kmemtrace_print_alloc_original(iter, field);
247 }
248
249 case TRACE_KMEM_FREE: {
250 struct kmemtrace_free_entry *field;
251 trace_assign_type(field, entry);
252 if (kmem_tracer_flags.val & TRACE_KMEM_OPT_MINIMAL)
253 return kmemtrace_print_free_compress(iter, field);
254 else
255 return kmemtrace_print_free_original(iter, field);
256 }
257
258 default:
259 return TRACE_TYPE_UNHANDLED;
260 }
261}
262
263/* Trace allocations */
264void kmemtrace_mark_alloc_node(enum kmemtrace_type_id type_id,
265 unsigned long call_site,
266 const void *ptr,
267 size_t bytes_req,
268 size_t bytes_alloc,
269 gfp_t gfp_flags,
270 int node)
271{
272 struct ring_buffer_event *event;
273 struct kmemtrace_alloc_entry *entry;
274 struct trace_array *tr = kmemtrace_array;
275
276 if (!kmem_tracing_enabled)
277 return;
278
279 event = trace_buffer_lock_reserve(tr, TRACE_KMEM_ALLOC,
280 sizeof(*entry), 0, 0);
281 if (!event)
282 return;
283 entry = ring_buffer_event_data(event);
284
285 entry->call_site = call_site;
286 entry->ptr = ptr;
287 entry->bytes_req = bytes_req;
288 entry->bytes_alloc = bytes_alloc;
289 entry->gfp_flags = gfp_flags;
290 entry->node = node;
291
292 trace_buffer_unlock_commit(tr, event, 0, 0);
293}
294EXPORT_SYMBOL(kmemtrace_mark_alloc_node);
295
296void kmemtrace_mark_free(enum kmemtrace_type_id type_id,
297 unsigned long call_site,
298 const void *ptr)
299{
300 struct ring_buffer_event *event;
301 struct kmemtrace_free_entry *entry;
302 struct trace_array *tr = kmemtrace_array;
303
304 if (!kmem_tracing_enabled)
305 return;
306
307 event = trace_buffer_lock_reserve(tr, TRACE_KMEM_FREE,
308 sizeof(*entry), 0, 0);
309 if (!event)
310 return;
311 entry = ring_buffer_event_data(event);
312 entry->type_id = type_id;
313 entry->call_site = call_site;
314 entry->ptr = ptr;
315
316 trace_buffer_unlock_commit(tr, event, 0, 0);
317}
318EXPORT_SYMBOL(kmemtrace_mark_free);
319
320static struct tracer kmem_tracer __read_mostly = {
321 .name = "kmemtrace",
322 .init = kmem_trace_init,
323 .reset = kmem_trace_reset,
324 .print_line = kmemtrace_print_line,
325 .print_header = kmemtrace_headers,
326 .flags = &kmem_tracer_flags
327};
328
329void kmemtrace_init(void)
330{
331 /* earliest opportunity to start kmem tracing */
332}
333
334static int __init init_kmem_tracer(void)
335{
336 return register_tracer(&kmem_tracer);
337}
338
339device_initcall(init_kmem_tracer);
diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c
index bd38c5cfd8ad..178858492a89 100644
--- a/kernel/trace/ring_buffer.c
+++ b/kernel/trace/ring_buffer.c
@@ -4,13 +4,15 @@
4 * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com> 4 * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com>
5 */ 5 */
6#include <linux/ring_buffer.h> 6#include <linux/ring_buffer.h>
7#include <linux/trace_clock.h>
8#include <linux/ftrace_irq.h>
7#include <linux/spinlock.h> 9#include <linux/spinlock.h>
8#include <linux/debugfs.h> 10#include <linux/debugfs.h>
9#include <linux/uaccess.h> 11#include <linux/uaccess.h>
12#include <linux/hardirq.h>
10#include <linux/module.h> 13#include <linux/module.h>
11#include <linux/percpu.h> 14#include <linux/percpu.h>
12#include <linux/mutex.h> 15#include <linux/mutex.h>
13#include <linux/sched.h> /* used for sched_clock() (for now) */
14#include <linux/init.h> 16#include <linux/init.h>
15#include <linux/hash.h> 17#include <linux/hash.h>
16#include <linux/list.h> 18#include <linux/list.h>
@@ -57,7 +59,9 @@ enum {
57 RB_BUFFERS_DISABLED = 1 << RB_BUFFERS_DISABLED_BIT, 59 RB_BUFFERS_DISABLED = 1 << RB_BUFFERS_DISABLED_BIT,
58}; 60};
59 61
60static long ring_buffer_flags __read_mostly = RB_BUFFERS_ON; 62static unsigned long ring_buffer_flags __read_mostly = RB_BUFFERS_ON;
63
64#define BUF_PAGE_HDR_SIZE offsetof(struct buffer_data_page, data)
61 65
62/** 66/**
63 * tracing_on - enable all tracing buffers 67 * tracing_on - enable all tracing buffers
@@ -89,26 +93,34 @@ EXPORT_SYMBOL_GPL(tracing_off);
89 * tracing_off_permanent - permanently disable ring buffers 93 * tracing_off_permanent - permanently disable ring buffers
90 * 94 *
91 * This function, once called, will disable all ring buffers 95 * This function, once called, will disable all ring buffers
92 * permanenty. 96 * permanently.
93 */ 97 */
94void tracing_off_permanent(void) 98void tracing_off_permanent(void)
95{ 99{
96 set_bit(RB_BUFFERS_DISABLED_BIT, &ring_buffer_flags); 100 set_bit(RB_BUFFERS_DISABLED_BIT, &ring_buffer_flags);
97} 101}
98 102
103/**
104 * tracing_is_on - show state of ring buffers enabled
105 */
106int tracing_is_on(void)
107{
108 return ring_buffer_flags == RB_BUFFERS_ON;
109}
110EXPORT_SYMBOL_GPL(tracing_is_on);
111
99#include "trace.h" 112#include "trace.h"
100 113
101/* Up this if you want to test the TIME_EXTENTS and normalization */ 114/* Up this if you want to test the TIME_EXTENTS and normalization */
102#define DEBUG_SHIFT 0 115#define DEBUG_SHIFT 0
103 116
104/* FIXME!!! */
105u64 ring_buffer_time_stamp(int cpu) 117u64 ring_buffer_time_stamp(int cpu)
106{ 118{
107 u64 time; 119 u64 time;
108 120
109 preempt_disable_notrace(); 121 preempt_disable_notrace();
110 /* shift to debug/test normalization and TIME_EXTENTS */ 122 /* shift to debug/test normalization and TIME_EXTENTS */
111 time = sched_clock() << DEBUG_SHIFT; 123 time = trace_clock_local() << DEBUG_SHIFT;
112 preempt_enable_no_resched_notrace(); 124 preempt_enable_no_resched_notrace();
113 125
114 return time; 126 return time;
@@ -122,9 +134,8 @@ void ring_buffer_normalize_time_stamp(int cpu, u64 *ts)
122} 134}
123EXPORT_SYMBOL_GPL(ring_buffer_normalize_time_stamp); 135EXPORT_SYMBOL_GPL(ring_buffer_normalize_time_stamp);
124 136
125#define RB_EVNT_HDR_SIZE (sizeof(struct ring_buffer_event)) 137#define RB_EVNT_HDR_SIZE (offsetof(struct ring_buffer_event, array))
126#define RB_ALIGNMENT_SHIFT 2 138#define RB_ALIGNMENT 4U
127#define RB_ALIGNMENT (1 << RB_ALIGNMENT_SHIFT)
128#define RB_MAX_SMALL_DATA 28 139#define RB_MAX_SMALL_DATA 28
129 140
130enum { 141enum {
@@ -133,7 +144,7 @@ enum {
133}; 144};
134 145
135/* inline for ring buffer fast paths */ 146/* inline for ring buffer fast paths */
136static inline unsigned 147static unsigned
137rb_event_length(struct ring_buffer_event *event) 148rb_event_length(struct ring_buffer_event *event)
138{ 149{
139 unsigned length; 150 unsigned length;
@@ -151,7 +162,7 @@ rb_event_length(struct ring_buffer_event *event)
151 162
152 case RINGBUF_TYPE_DATA: 163 case RINGBUF_TYPE_DATA:
153 if (event->len) 164 if (event->len)
154 length = event->len << RB_ALIGNMENT_SHIFT; 165 length = event->len * RB_ALIGNMENT;
155 else 166 else
156 length = event->array[0]; 167 length = event->array[0];
157 return length + RB_EVNT_HDR_SIZE; 168 return length + RB_EVNT_HDR_SIZE;
@@ -179,7 +190,7 @@ unsigned ring_buffer_event_length(struct ring_buffer_event *event)
179EXPORT_SYMBOL_GPL(ring_buffer_event_length); 190EXPORT_SYMBOL_GPL(ring_buffer_event_length);
180 191
181/* inline for ring buffer fast paths */ 192/* inline for ring buffer fast paths */
182static inline void * 193static void *
183rb_event_data(struct ring_buffer_event *event) 194rb_event_data(struct ring_buffer_event *event)
184{ 195{
185 BUG_ON(event->type != RINGBUF_TYPE_DATA); 196 BUG_ON(event->type != RINGBUF_TYPE_DATA);
@@ -209,7 +220,7 @@ EXPORT_SYMBOL_GPL(ring_buffer_event_data);
209 220
210struct buffer_data_page { 221struct buffer_data_page {
211 u64 time_stamp; /* page time stamp */ 222 u64 time_stamp; /* page time stamp */
212 local_t commit; /* write commited index */ 223 local_t commit; /* write committed index */
213 unsigned char data[]; /* data of buffer page */ 224 unsigned char data[]; /* data of buffer page */
214}; 225};
215 226
@@ -225,14 +236,25 @@ static void rb_init_page(struct buffer_data_page *bpage)
225 local_set(&bpage->commit, 0); 236 local_set(&bpage->commit, 0);
226} 237}
227 238
239/**
240 * ring_buffer_page_len - the size of data on the page.
241 * @page: The page to read
242 *
243 * Returns the amount of data on the page, including buffer page header.
244 */
245size_t ring_buffer_page_len(void *page)
246{
247 return local_read(&((struct buffer_data_page *)page)->commit)
248 + BUF_PAGE_HDR_SIZE;
249}
250
228/* 251/*
229 * Also stolen from mm/slob.c. Thanks to Mathieu Desnoyers for pointing 252 * Also stolen from mm/slob.c. Thanks to Mathieu Desnoyers for pointing
230 * this issue out. 253 * this issue out.
231 */ 254 */
232static inline void free_buffer_page(struct buffer_page *bpage) 255static void free_buffer_page(struct buffer_page *bpage)
233{ 256{
234 if (bpage->page) 257 free_page((unsigned long)bpage->page);
235 free_page((unsigned long)bpage->page);
236 kfree(bpage); 258 kfree(bpage);
237} 259}
238 260
@@ -246,7 +268,7 @@ static inline int test_time_stamp(u64 delta)
246 return 0; 268 return 0;
247} 269}
248 270
249#define BUF_PAGE_SIZE (PAGE_SIZE - offsetof(struct buffer_data_page, data)) 271#define BUF_PAGE_SIZE (PAGE_SIZE - BUF_PAGE_HDR_SIZE)
250 272
251/* 273/*
252 * head_page == tail_page && head == tail then buffer is empty. 274 * head_page == tail_page && head == tail then buffer is empty.
@@ -260,7 +282,7 @@ struct ring_buffer_per_cpu {
260 struct list_head pages; 282 struct list_head pages;
261 struct buffer_page *head_page; /* read from head */ 283 struct buffer_page *head_page; /* read from head */
262 struct buffer_page *tail_page; /* write to tail */ 284 struct buffer_page *tail_page; /* write to tail */
263 struct buffer_page *commit_page; /* commited pages */ 285 struct buffer_page *commit_page; /* committed pages */
264 struct buffer_page *reader_page; 286 struct buffer_page *reader_page;
265 unsigned long overrun; 287 unsigned long overrun;
266 unsigned long entries; 288 unsigned long entries;
@@ -273,8 +295,8 @@ struct ring_buffer {
273 unsigned pages; 295 unsigned pages;
274 unsigned flags; 296 unsigned flags;
275 int cpus; 297 int cpus;
276 cpumask_var_t cpumask;
277 atomic_t record_disabled; 298 atomic_t record_disabled;
299 cpumask_var_t cpumask;
278 300
279 struct mutex mutex; 301 struct mutex mutex;
280 302
@@ -303,7 +325,7 @@ struct ring_buffer_iter {
303 * check_pages - integrity check of buffer pages 325 * check_pages - integrity check of buffer pages
304 * @cpu_buffer: CPU buffer with pages to test 326 * @cpu_buffer: CPU buffer with pages to test
305 * 327 *
306 * As a safty measure we check to make sure the data pages have not 328 * As a safety measure we check to make sure the data pages have not
307 * been corrupted. 329 * been corrupted.
308 */ 330 */
309static int rb_check_pages(struct ring_buffer_per_cpu *cpu_buffer) 331static int rb_check_pages(struct ring_buffer_per_cpu *cpu_buffer)
@@ -811,7 +833,7 @@ rb_event_index(struct ring_buffer_event *event)
811 return (addr & ~PAGE_MASK) - (PAGE_SIZE - BUF_PAGE_SIZE); 833 return (addr & ~PAGE_MASK) - (PAGE_SIZE - BUF_PAGE_SIZE);
812} 834}
813 835
814static inline int 836static int
815rb_is_commit(struct ring_buffer_per_cpu *cpu_buffer, 837rb_is_commit(struct ring_buffer_per_cpu *cpu_buffer,
816 struct ring_buffer_event *event) 838 struct ring_buffer_event *event)
817{ 839{
@@ -825,7 +847,7 @@ rb_is_commit(struct ring_buffer_per_cpu *cpu_buffer,
825 rb_commit_index(cpu_buffer) == index; 847 rb_commit_index(cpu_buffer) == index;
826} 848}
827 849
828static inline void 850static void
829rb_set_commit_event(struct ring_buffer_per_cpu *cpu_buffer, 851rb_set_commit_event(struct ring_buffer_per_cpu *cpu_buffer,
830 struct ring_buffer_event *event) 852 struct ring_buffer_event *event)
831{ 853{
@@ -850,7 +872,7 @@ rb_set_commit_event(struct ring_buffer_per_cpu *cpu_buffer,
850 local_set(&cpu_buffer->commit_page->page->commit, index); 872 local_set(&cpu_buffer->commit_page->page->commit, index);
851} 873}
852 874
853static inline void 875static void
854rb_set_commit_to_write(struct ring_buffer_per_cpu *cpu_buffer) 876rb_set_commit_to_write(struct ring_buffer_per_cpu *cpu_buffer)
855{ 877{
856 /* 878 /*
@@ -896,7 +918,7 @@ static void rb_reset_reader_page(struct ring_buffer_per_cpu *cpu_buffer)
896 cpu_buffer->reader_page->read = 0; 918 cpu_buffer->reader_page->read = 0;
897} 919}
898 920
899static inline void rb_inc_iter(struct ring_buffer_iter *iter) 921static void rb_inc_iter(struct ring_buffer_iter *iter)
900{ 922{
901 struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer; 923 struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
902 924
@@ -926,7 +948,7 @@ static inline void rb_inc_iter(struct ring_buffer_iter *iter)
926 * and with this, we can determine what to place into the 948 * and with this, we can determine what to place into the
927 * data field. 949 * data field.
928 */ 950 */
929static inline void 951static void
930rb_update_event(struct ring_buffer_event *event, 952rb_update_event(struct ring_buffer_event *event,
931 unsigned type, unsigned length) 953 unsigned type, unsigned length)
932{ 954{
@@ -938,15 +960,11 @@ rb_update_event(struct ring_buffer_event *event,
938 break; 960 break;
939 961
940 case RINGBUF_TYPE_TIME_EXTEND: 962 case RINGBUF_TYPE_TIME_EXTEND:
941 event->len = 963 event->len = DIV_ROUND_UP(RB_LEN_TIME_EXTEND, RB_ALIGNMENT);
942 (RB_LEN_TIME_EXTEND + (RB_ALIGNMENT-1))
943 >> RB_ALIGNMENT_SHIFT;
944 break; 964 break;
945 965
946 case RINGBUF_TYPE_TIME_STAMP: 966 case RINGBUF_TYPE_TIME_STAMP:
947 event->len = 967 event->len = DIV_ROUND_UP(RB_LEN_TIME_STAMP, RB_ALIGNMENT);
948 (RB_LEN_TIME_STAMP + (RB_ALIGNMENT-1))
949 >> RB_ALIGNMENT_SHIFT;
950 break; 968 break;
951 969
952 case RINGBUF_TYPE_DATA: 970 case RINGBUF_TYPE_DATA:
@@ -955,16 +973,14 @@ rb_update_event(struct ring_buffer_event *event,
955 event->len = 0; 973 event->len = 0;
956 event->array[0] = length; 974 event->array[0] = length;
957 } else 975 } else
958 event->len = 976 event->len = DIV_ROUND_UP(length, RB_ALIGNMENT);
959 (length + (RB_ALIGNMENT-1))
960 >> RB_ALIGNMENT_SHIFT;
961 break; 977 break;
962 default: 978 default:
963 BUG(); 979 BUG();
964 } 980 }
965} 981}
966 982
967static inline unsigned rb_calculate_event_length(unsigned length) 983static unsigned rb_calculate_event_length(unsigned length)
968{ 984{
969 struct ring_buffer_event event; /* Used only for sizeof array */ 985 struct ring_buffer_event event; /* Used only for sizeof array */
970 986
@@ -990,6 +1006,7 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer,
990 struct ring_buffer *buffer = cpu_buffer->buffer; 1006 struct ring_buffer *buffer = cpu_buffer->buffer;
991 struct ring_buffer_event *event; 1007 struct ring_buffer_event *event;
992 unsigned long flags; 1008 unsigned long flags;
1009 bool lock_taken = false;
993 1010
994 commit_page = cpu_buffer->commit_page; 1011 commit_page = cpu_buffer->commit_page;
995 /* we just need to protect against interrupts */ 1012 /* we just need to protect against interrupts */
@@ -1003,7 +1020,30 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer,
1003 struct buffer_page *next_page = tail_page; 1020 struct buffer_page *next_page = tail_page;
1004 1021
1005 local_irq_save(flags); 1022 local_irq_save(flags);
1006 __raw_spin_lock(&cpu_buffer->lock); 1023 /*
1024 * Since the write to the buffer is still not
1025 * fully lockless, we must be careful with NMIs.
1026 * The locks in the writers are taken when a write
1027 * crosses to a new page. The locks protect against
1028 * races with the readers (this will soon be fixed
1029 * with a lockless solution).
1030 *
1031 * Because we can not protect against NMIs, and we
1032 * want to keep traces reentrant, we need to manage
1033 * what happens when we are in an NMI.
1034 *
1035 * NMIs can happen after we take the lock.
1036 * If we are in an NMI, only take the lock
1037 * if it is not already taken. Otherwise
1038 * simply fail.
1039 */
1040 if (unlikely(in_nmi())) {
1041 if (!__raw_spin_trylock(&cpu_buffer->lock))
1042 goto out_reset;
1043 } else
1044 __raw_spin_lock(&cpu_buffer->lock);
1045
1046 lock_taken = true;
1007 1047
1008 rb_inc_page(cpu_buffer, &next_page); 1048 rb_inc_page(cpu_buffer, &next_page);
1009 1049
@@ -1012,7 +1052,7 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer,
1012 1052
1013 /* we grabbed the lock before incrementing */ 1053 /* we grabbed the lock before incrementing */
1014 if (RB_WARN_ON(cpu_buffer, next_page == reader_page)) 1054 if (RB_WARN_ON(cpu_buffer, next_page == reader_page))
1015 goto out_unlock; 1055 goto out_reset;
1016 1056
1017 /* 1057 /*
1018 * If for some reason, we had an interrupt storm that made 1058 * If for some reason, we had an interrupt storm that made
@@ -1021,12 +1061,12 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer,
1021 */ 1061 */
1022 if (unlikely(next_page == commit_page)) { 1062 if (unlikely(next_page == commit_page)) {
1023 WARN_ON_ONCE(1); 1063 WARN_ON_ONCE(1);
1024 goto out_unlock; 1064 goto out_reset;
1025 } 1065 }
1026 1066
1027 if (next_page == head_page) { 1067 if (next_page == head_page) {
1028 if (!(buffer->flags & RB_FL_OVERWRITE)) 1068 if (!(buffer->flags & RB_FL_OVERWRITE))
1029 goto out_unlock; 1069 goto out_reset;
1030 1070
1031 /* tail_page has not moved yet? */ 1071 /* tail_page has not moved yet? */
1032 if (tail_page == cpu_buffer->tail_page) { 1072 if (tail_page == cpu_buffer->tail_page) {
@@ -1100,12 +1140,13 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer,
1100 1140
1101 return event; 1141 return event;
1102 1142
1103 out_unlock: 1143 out_reset:
1104 /* reset write */ 1144 /* reset write */
1105 if (tail <= BUF_PAGE_SIZE) 1145 if (tail <= BUF_PAGE_SIZE)
1106 local_set(&tail_page->write, tail); 1146 local_set(&tail_page->write, tail);
1107 1147
1108 __raw_spin_unlock(&cpu_buffer->lock); 1148 if (likely(lock_taken))
1149 __raw_spin_unlock(&cpu_buffer->lock);
1109 local_irq_restore(flags); 1150 local_irq_restore(flags);
1110 return NULL; 1151 return NULL;
1111} 1152}
@@ -1265,7 +1306,6 @@ static DEFINE_PER_CPU(int, rb_need_resched);
1265 * ring_buffer_lock_reserve - reserve a part of the buffer 1306 * ring_buffer_lock_reserve - reserve a part of the buffer
1266 * @buffer: the ring buffer to reserve from 1307 * @buffer: the ring buffer to reserve from
1267 * @length: the length of the data to reserve (excluding event header) 1308 * @length: the length of the data to reserve (excluding event header)
1268 * @flags: a pointer to save the interrupt flags
1269 * 1309 *
1270 * Returns a reseverd event on the ring buffer to copy directly to. 1310 * Returns a reseverd event on the ring buffer to copy directly to.
1271 * The user of this interface will need to get the body to write into 1311 * The user of this interface will need to get the body to write into
@@ -1278,9 +1318,7 @@ static DEFINE_PER_CPU(int, rb_need_resched);
1278 * If NULL is returned, then nothing has been allocated or locked. 1318 * If NULL is returned, then nothing has been allocated or locked.
1279 */ 1319 */
1280struct ring_buffer_event * 1320struct ring_buffer_event *
1281ring_buffer_lock_reserve(struct ring_buffer *buffer, 1321ring_buffer_lock_reserve(struct ring_buffer *buffer, unsigned long length)
1282 unsigned long length,
1283 unsigned long *flags)
1284{ 1322{
1285 struct ring_buffer_per_cpu *cpu_buffer; 1323 struct ring_buffer_per_cpu *cpu_buffer;
1286 struct ring_buffer_event *event; 1324 struct ring_buffer_event *event;
@@ -1347,15 +1385,13 @@ static void rb_commit(struct ring_buffer_per_cpu *cpu_buffer,
1347 * ring_buffer_unlock_commit - commit a reserved 1385 * ring_buffer_unlock_commit - commit a reserved
1348 * @buffer: The buffer to commit to 1386 * @buffer: The buffer to commit to
1349 * @event: The event pointer to commit. 1387 * @event: The event pointer to commit.
1350 * @flags: the interrupt flags received from ring_buffer_lock_reserve.
1351 * 1388 *
1352 * This commits the data to the ring buffer, and releases any locks held. 1389 * This commits the data to the ring buffer, and releases any locks held.
1353 * 1390 *
1354 * Must be paired with ring_buffer_lock_reserve. 1391 * Must be paired with ring_buffer_lock_reserve.
1355 */ 1392 */
1356int ring_buffer_unlock_commit(struct ring_buffer *buffer, 1393int ring_buffer_unlock_commit(struct ring_buffer *buffer,
1357 struct ring_buffer_event *event, 1394 struct ring_buffer_event *event)
1358 unsigned long flags)
1359{ 1395{
1360 struct ring_buffer_per_cpu *cpu_buffer; 1396 struct ring_buffer_per_cpu *cpu_buffer;
1361 int cpu = raw_smp_processor_id(); 1397 int cpu = raw_smp_processor_id();
@@ -1438,7 +1474,7 @@ int ring_buffer_write(struct ring_buffer *buffer,
1438} 1474}
1439EXPORT_SYMBOL_GPL(ring_buffer_write); 1475EXPORT_SYMBOL_GPL(ring_buffer_write);
1440 1476
1441static inline int rb_per_cpu_empty(struct ring_buffer_per_cpu *cpu_buffer) 1477static int rb_per_cpu_empty(struct ring_buffer_per_cpu *cpu_buffer)
1442{ 1478{
1443 struct buffer_page *reader = cpu_buffer->reader_page; 1479 struct buffer_page *reader = cpu_buffer->reader_page;
1444 struct buffer_page *head = cpu_buffer->head_page; 1480 struct buffer_page *head = cpu_buffer->head_page;
@@ -2277,9 +2313,24 @@ int ring_buffer_swap_cpu(struct ring_buffer *buffer_a,
2277 if (buffer_a->pages != buffer_b->pages) 2313 if (buffer_a->pages != buffer_b->pages)
2278 return -EINVAL; 2314 return -EINVAL;
2279 2315
2316 if (ring_buffer_flags != RB_BUFFERS_ON)
2317 return -EAGAIN;
2318
2319 if (atomic_read(&buffer_a->record_disabled))
2320 return -EAGAIN;
2321
2322 if (atomic_read(&buffer_b->record_disabled))
2323 return -EAGAIN;
2324
2280 cpu_buffer_a = buffer_a->buffers[cpu]; 2325 cpu_buffer_a = buffer_a->buffers[cpu];
2281 cpu_buffer_b = buffer_b->buffers[cpu]; 2326 cpu_buffer_b = buffer_b->buffers[cpu];
2282 2327
2328 if (atomic_read(&cpu_buffer_a->record_disabled))
2329 return -EAGAIN;
2330
2331 if (atomic_read(&cpu_buffer_b->record_disabled))
2332 return -EAGAIN;
2333
2283 /* 2334 /*
2284 * We can't do a synchronize_sched here because this 2335 * We can't do a synchronize_sched here because this
2285 * function can be called in atomic context. 2336 * function can be called in atomic context.
@@ -2303,13 +2354,14 @@ int ring_buffer_swap_cpu(struct ring_buffer *buffer_a,
2303EXPORT_SYMBOL_GPL(ring_buffer_swap_cpu); 2354EXPORT_SYMBOL_GPL(ring_buffer_swap_cpu);
2304 2355
2305static void rb_remove_entries(struct ring_buffer_per_cpu *cpu_buffer, 2356static void rb_remove_entries(struct ring_buffer_per_cpu *cpu_buffer,
2306 struct buffer_data_page *bpage) 2357 struct buffer_data_page *bpage,
2358 unsigned int offset)
2307{ 2359{
2308 struct ring_buffer_event *event; 2360 struct ring_buffer_event *event;
2309 unsigned long head; 2361 unsigned long head;
2310 2362
2311 __raw_spin_lock(&cpu_buffer->lock); 2363 __raw_spin_lock(&cpu_buffer->lock);
2312 for (head = 0; head < local_read(&bpage->commit); 2364 for (head = offset; head < local_read(&bpage->commit);
2313 head += rb_event_length(event)) { 2365 head += rb_event_length(event)) {
2314 2366
2315 event = __rb_data_page_index(bpage, head); 2367 event = __rb_data_page_index(bpage, head);
@@ -2340,8 +2392,8 @@ static void rb_remove_entries(struct ring_buffer_per_cpu *cpu_buffer,
2340 */ 2392 */
2341void *ring_buffer_alloc_read_page(struct ring_buffer *buffer) 2393void *ring_buffer_alloc_read_page(struct ring_buffer *buffer)
2342{ 2394{
2343 unsigned long addr;
2344 struct buffer_data_page *bpage; 2395 struct buffer_data_page *bpage;
2396 unsigned long addr;
2345 2397
2346 addr = __get_free_page(GFP_KERNEL); 2398 addr = __get_free_page(GFP_KERNEL);
2347 if (!addr) 2399 if (!addr)
@@ -2349,6 +2401,8 @@ void *ring_buffer_alloc_read_page(struct ring_buffer *buffer)
2349 2401
2350 bpage = (void *)addr; 2402 bpage = (void *)addr;
2351 2403
2404 rb_init_page(bpage);
2405
2352 return bpage; 2406 return bpage;
2353} 2407}
2354 2408
@@ -2368,6 +2422,7 @@ void ring_buffer_free_read_page(struct ring_buffer *buffer, void *data)
2368 * ring_buffer_read_page - extract a page from the ring buffer 2422 * ring_buffer_read_page - extract a page from the ring buffer
2369 * @buffer: buffer to extract from 2423 * @buffer: buffer to extract from
2370 * @data_page: the page to use allocated from ring_buffer_alloc_read_page 2424 * @data_page: the page to use allocated from ring_buffer_alloc_read_page
2425 * @len: amount to extract
2371 * @cpu: the cpu of the buffer to extract 2426 * @cpu: the cpu of the buffer to extract
2372 * @full: should the extraction only happen when the page is full. 2427 * @full: should the extraction only happen when the page is full.
2373 * 2428 *
@@ -2377,12 +2432,12 @@ void ring_buffer_free_read_page(struct ring_buffer *buffer, void *data)
2377 * to swap with a page in the ring buffer. 2432 * to swap with a page in the ring buffer.
2378 * 2433 *
2379 * for example: 2434 * for example:
2380 * rpage = ring_buffer_alloc_page(buffer); 2435 * rpage = ring_buffer_alloc_read_page(buffer);
2381 * if (!rpage) 2436 * if (!rpage)
2382 * return error; 2437 * return error;
2383 * ret = ring_buffer_read_page(buffer, &rpage, cpu, 0); 2438 * ret = ring_buffer_read_page(buffer, &rpage, len, cpu, 0);
2384 * if (ret) 2439 * if (ret >= 0)
2385 * process_page(rpage); 2440 * process_page(rpage, ret);
2386 * 2441 *
2387 * When @full is set, the function will not return true unless 2442 * When @full is set, the function will not return true unless
2388 * the writer is off the reader page. 2443 * the writer is off the reader page.
@@ -2393,69 +2448,111 @@ void ring_buffer_free_read_page(struct ring_buffer *buffer, void *data)
2393 * responsible for that. 2448 * responsible for that.
2394 * 2449 *
2395 * Returns: 2450 * Returns:
2396 * 1 if data has been transferred 2451 * >=0 if data has been transferred, returns the offset of consumed data.
2397 * 0 if no data has been transferred. 2452 * <0 if no data has been transferred.
2398 */ 2453 */
2399int ring_buffer_read_page(struct ring_buffer *buffer, 2454int ring_buffer_read_page(struct ring_buffer *buffer,
2400 void **data_page, int cpu, int full) 2455 void **data_page, size_t len, int cpu, int full)
2401{ 2456{
2402 struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu]; 2457 struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];
2403 struct ring_buffer_event *event; 2458 struct ring_buffer_event *event;
2404 struct buffer_data_page *bpage; 2459 struct buffer_data_page *bpage;
2460 struct buffer_page *reader;
2405 unsigned long flags; 2461 unsigned long flags;
2406 int ret = 0; 2462 unsigned int commit;
2463 unsigned int read;
2464 u64 save_timestamp;
2465 int ret = -1;
2466
2467 /*
2468 * If len is not big enough to hold the page header, then
2469 * we can not copy anything.
2470 */
2471 if (len <= BUF_PAGE_HDR_SIZE)
2472 return -1;
2473
2474 len -= BUF_PAGE_HDR_SIZE;
2407 2475
2408 if (!data_page) 2476 if (!data_page)
2409 return 0; 2477 return -1;
2410 2478
2411 bpage = *data_page; 2479 bpage = *data_page;
2412 if (!bpage) 2480 if (!bpage)
2413 return 0; 2481 return -1;
2414 2482
2415 spin_lock_irqsave(&cpu_buffer->reader_lock, flags); 2483 spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
2416 2484
2417 /* 2485 reader = rb_get_reader_page(cpu_buffer);
2418 * rb_buffer_peek will get the next ring buffer if 2486 if (!reader)
2419 * the current reader page is empty.
2420 */
2421 event = rb_buffer_peek(buffer, cpu, NULL);
2422 if (!event)
2423 goto out; 2487 goto out;
2424 2488
2425 /* check for data */ 2489 event = rb_reader_event(cpu_buffer);
2426 if (!local_read(&cpu_buffer->reader_page->page->commit)) 2490
2427 goto out; 2491 read = reader->read;
2492 commit = rb_page_commit(reader);
2493
2428 /* 2494 /*
2429 * If the writer is already off of the read page, then simply 2495 * If this page has been partially read or
2430 * switch the read page with the given page. Otherwise 2496 * if len is not big enough to read the rest of the page or
2431 * we need to copy the data from the reader to the writer. 2497 * a writer is still on the page, then
2498 * we must copy the data from the page to the buffer.
2499 * Otherwise, we can simply swap the page with the one passed in.
2432 */ 2500 */
2433 if (cpu_buffer->reader_page == cpu_buffer->commit_page) { 2501 if (read || (len < (commit - read)) ||
2434 unsigned int read = cpu_buffer->reader_page->read; 2502 cpu_buffer->reader_page == cpu_buffer->commit_page) {
2503 struct buffer_data_page *rpage = cpu_buffer->reader_page->page;
2504 unsigned int rpos = read;
2505 unsigned int pos = 0;
2506 unsigned int size;
2435 2507
2436 if (full) 2508 if (full)
2437 goto out; 2509 goto out;
2438 /* The writer is still on the reader page, we must copy */
2439 bpage = cpu_buffer->reader_page->page;
2440 memcpy(bpage->data,
2441 cpu_buffer->reader_page->page->data + read,
2442 local_read(&bpage->commit) - read);
2443 2510
2444 /* consume what was read */ 2511 if (len > (commit - read))
2445 cpu_buffer->reader_page += read; 2512 len = (commit - read);
2446 2513
2514 size = rb_event_length(event);
2515
2516 if (len < size)
2517 goto out;
2518
2519 /* save the current timestamp, since the user will need it */
2520 save_timestamp = cpu_buffer->read_stamp;
2521
2522 /* Need to copy one event at a time */
2523 do {
2524 memcpy(bpage->data + pos, rpage->data + rpos, size);
2525
2526 len -= size;
2527
2528 rb_advance_reader(cpu_buffer);
2529 rpos = reader->read;
2530 pos += size;
2531
2532 event = rb_reader_event(cpu_buffer);
2533 size = rb_event_length(event);
2534 } while (len > size);
2535
2536 /* update bpage */
2537 local_set(&bpage->commit, pos);
2538 bpage->time_stamp = save_timestamp;
2539
2540 /* we copied everything to the beginning */
2541 read = 0;
2447 } else { 2542 } else {
2448 /* swap the pages */ 2543 /* swap the pages */
2449 rb_init_page(bpage); 2544 rb_init_page(bpage);
2450 bpage = cpu_buffer->reader_page->page; 2545 bpage = reader->page;
2451 cpu_buffer->reader_page->page = *data_page; 2546 reader->page = *data_page;
2452 cpu_buffer->reader_page->read = 0; 2547 local_set(&reader->write, 0);
2548 reader->read = 0;
2453 *data_page = bpage; 2549 *data_page = bpage;
2550
2551 /* update the entry counter */
2552 rb_remove_entries(cpu_buffer, bpage, read);
2454 } 2553 }
2455 ret = 1; 2554 ret = read;
2456 2555
2457 /* update the entry counter */
2458 rb_remove_entries(cpu_buffer, bpage);
2459 out: 2556 out:
2460 spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); 2557 spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
2461 2558
@@ -2466,7 +2563,7 @@ static ssize_t
2466rb_simple_read(struct file *filp, char __user *ubuf, 2563rb_simple_read(struct file *filp, char __user *ubuf,
2467 size_t cnt, loff_t *ppos) 2564 size_t cnt, loff_t *ppos)
2468{ 2565{
2469 long *p = filp->private_data; 2566 unsigned long *p = filp->private_data;
2470 char buf[64]; 2567 char buf[64];
2471 int r; 2568 int r;
2472 2569
@@ -2482,9 +2579,9 @@ static ssize_t
2482rb_simple_write(struct file *filp, const char __user *ubuf, 2579rb_simple_write(struct file *filp, const char __user *ubuf,
2483 size_t cnt, loff_t *ppos) 2580 size_t cnt, loff_t *ppos)
2484{ 2581{
2485 long *p = filp->private_data; 2582 unsigned long *p = filp->private_data;
2486 char buf[64]; 2583 char buf[64];
2487 long val; 2584 unsigned long val;
2488 int ret; 2585 int ret;
2489 2586
2490 if (cnt >= sizeof(buf)) 2587 if (cnt >= sizeof(buf))
@@ -2509,7 +2606,7 @@ rb_simple_write(struct file *filp, const char __user *ubuf,
2509 return cnt; 2606 return cnt;
2510} 2607}
2511 2608
2512static struct file_operations rb_simple_fops = { 2609static const struct file_operations rb_simple_fops = {
2513 .open = tracing_open_generic, 2610 .open = tracing_open_generic,
2514 .read = rb_simple_read, 2611 .read = rb_simple_read,
2515 .write = rb_simple_write, 2612 .write = rb_simple_write,
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index 17bb88d86ac2..e5b56199e5e0 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -11,32 +11,33 @@
11 * Copyright (C) 2004-2006 Ingo Molnar 11 * Copyright (C) 2004-2006 Ingo Molnar
12 * Copyright (C) 2004 William Lee Irwin III 12 * Copyright (C) 2004 William Lee Irwin III
13 */ 13 */
14#include <linux/ring_buffer.h>
14#include <linux/utsrelease.h> 15#include <linux/utsrelease.h>
16#include <linux/stacktrace.h>
17#include <linux/writeback.h>
15#include <linux/kallsyms.h> 18#include <linux/kallsyms.h>
16#include <linux/seq_file.h> 19#include <linux/seq_file.h>
17#include <linux/notifier.h> 20#include <linux/notifier.h>
21#include <linux/irqflags.h>
18#include <linux/debugfs.h> 22#include <linux/debugfs.h>
19#include <linux/pagemap.h> 23#include <linux/pagemap.h>
20#include <linux/hardirq.h> 24#include <linux/hardirq.h>
21#include <linux/linkage.h> 25#include <linux/linkage.h>
22#include <linux/uaccess.h> 26#include <linux/uaccess.h>
27#include <linux/kprobes.h>
23#include <linux/ftrace.h> 28#include <linux/ftrace.h>
24#include <linux/module.h> 29#include <linux/module.h>
25#include <linux/percpu.h> 30#include <linux/percpu.h>
31#include <linux/splice.h>
26#include <linux/kdebug.h> 32#include <linux/kdebug.h>
27#include <linux/ctype.h> 33#include <linux/ctype.h>
28#include <linux/init.h> 34#include <linux/init.h>
29#include <linux/poll.h> 35#include <linux/poll.h>
30#include <linux/gfp.h> 36#include <linux/gfp.h>
31#include <linux/fs.h> 37#include <linux/fs.h>
32#include <linux/kprobes.h>
33#include <linux/writeback.h>
34
35#include <linux/stacktrace.h>
36#include <linux/ring_buffer.h>
37#include <linux/irqflags.h>
38 38
39#include "trace.h" 39#include "trace.h"
40#include "trace_output.h"
40 41
41#define TRACE_BUFFER_FLAGS (RB_FL_OVERWRITE) 42#define TRACE_BUFFER_FLAGS (RB_FL_OVERWRITE)
42 43
@@ -47,11 +48,16 @@ unsigned long __read_mostly tracing_thresh;
47 * We need to change this state when a selftest is running. 48 * We need to change this state when a selftest is running.
48 * A selftest will lurk into the ring-buffer to count the 49 * A selftest will lurk into the ring-buffer to count the
49 * entries inserted during the selftest although some concurrent 50 * entries inserted during the selftest although some concurrent
50 * insertions into the ring-buffer such as ftrace_printk could occurred 51 * insertions into the ring-buffer such as trace_printk could occurred
51 * at the same time, giving false positive or negative results. 52 * at the same time, giving false positive or negative results.
52 */ 53 */
53static bool __read_mostly tracing_selftest_running; 54static bool __read_mostly tracing_selftest_running;
54 55
56/*
57 * If a tracer is running, we do not want to run SELFTEST.
58 */
59static bool __read_mostly tracing_selftest_disabled;
60
55/* For tracers that don't implement custom flags */ 61/* For tracers that don't implement custom flags */
56static struct tracer_opt dummy_tracer_opt[] = { 62static struct tracer_opt dummy_tracer_opt[] = {
57 { } 63 { }
@@ -73,7 +79,7 @@ static int dummy_set_flag(u32 old_flags, u32 bit, int set)
73 * of the tracer is successful. But that is the only place that sets 79 * of the tracer is successful. But that is the only place that sets
74 * this back to zero. 80 * this back to zero.
75 */ 81 */
76int tracing_disabled = 1; 82static int tracing_disabled = 1;
77 83
78static DEFINE_PER_CPU(local_t, ftrace_cpu_disabled); 84static DEFINE_PER_CPU(local_t, ftrace_cpu_disabled);
79 85
@@ -91,6 +97,9 @@ static inline void ftrace_enable_cpu(void)
91 97
92static cpumask_var_t __read_mostly tracing_buffer_mask; 98static cpumask_var_t __read_mostly tracing_buffer_mask;
93 99
100/* Define which cpu buffers are currently read in trace_pipe */
101static cpumask_var_t tracing_reader_cpumask;
102
94#define for_each_tracing_cpu(cpu) \ 103#define for_each_tracing_cpu(cpu) \
95 for_each_cpu(cpu, tracing_buffer_mask) 104 for_each_cpu(cpu, tracing_buffer_mask)
96 105
@@ -109,14 +118,19 @@ static cpumask_var_t __read_mostly tracing_buffer_mask;
109 */ 118 */
110int ftrace_dump_on_oops; 119int ftrace_dump_on_oops;
111 120
112static int tracing_set_tracer(char *buf); 121static int tracing_set_tracer(const char *buf);
122
123#define BOOTUP_TRACER_SIZE 100
124static char bootup_tracer_buf[BOOTUP_TRACER_SIZE] __initdata;
125static char *default_bootup_tracer;
113 126
114static int __init set_ftrace(char *str) 127static int __init set_ftrace(char *str)
115{ 128{
116 tracing_set_tracer(str); 129 strncpy(bootup_tracer_buf, str, BOOTUP_TRACER_SIZE);
130 default_bootup_tracer = bootup_tracer_buf;
117 return 1; 131 return 1;
118} 132}
119__setup("ftrace", set_ftrace); 133__setup("ftrace=", set_ftrace);
120 134
121static int __init set_ftrace_dump_on_oops(char *str) 135static int __init set_ftrace_dump_on_oops(char *str)
122{ 136{
@@ -186,9 +200,6 @@ int tracing_is_enabled(void)
186 return tracer_enabled; 200 return tracer_enabled;
187} 201}
188 202
189/* function tracing enabled */
190int ftrace_function_enabled;
191
192/* 203/*
193 * trace_buf_size is the size in bytes that is allocated 204 * trace_buf_size is the size in bytes that is allocated
194 * for a buffer. Note, the number of bytes is always rounded 205 * for a buffer. Note, the number of bytes is always rounded
@@ -229,7 +240,7 @@ static DECLARE_WAIT_QUEUE_HEAD(trace_wait);
229 240
230/* trace_flags holds trace_options default values */ 241/* trace_flags holds trace_options default values */
231unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK | 242unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
232 TRACE_ITER_ANNOTATE; 243 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO;
233 244
234/** 245/**
235 * trace_wake_up - wake up tasks waiting for trace input 246 * trace_wake_up - wake up tasks waiting for trace input
@@ -280,13 +291,15 @@ static const char *trace_options[] = {
280 "block", 291 "block",
281 "stacktrace", 292 "stacktrace",
282 "sched-tree", 293 "sched-tree",
283 "ftrace_printk", 294 "trace_printk",
284 "ftrace_preempt", 295 "ftrace_preempt",
285 "branch", 296 "branch",
286 "annotate", 297 "annotate",
287 "userstacktrace", 298 "userstacktrace",
288 "sym-userobj", 299 "sym-userobj",
289 "printk-msg-only", 300 "printk-msg-only",
301 "context-info",
302 "latency-format",
290 NULL 303 NULL
291}; 304};
292 305
@@ -326,146 +339,37 @@ __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
326 data->rt_priority = tsk->rt_priority; 339 data->rt_priority = tsk->rt_priority;
327 340
328 /* record this tasks comm */ 341 /* record this tasks comm */
329 tracing_record_cmdline(current); 342 tracing_record_cmdline(tsk);
330} 343}
331 344
332/** 345ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
333 * trace_seq_printf - sequence printing of trace information
334 * @s: trace sequence descriptor
335 * @fmt: printf format string
336 *
337 * The tracer may use either sequence operations or its own
338 * copy to user routines. To simplify formating of a trace
339 * trace_seq_printf is used to store strings into a special
340 * buffer (@s). Then the output may be either used by
341 * the sequencer or pulled into another buffer.
342 */
343int
344trace_seq_printf(struct trace_seq *s, const char *fmt, ...)
345{ 346{
346 int len = (PAGE_SIZE - 1) - s->len; 347 int len;
347 va_list ap;
348 int ret; 348 int ret;
349 349
350 if (!len) 350 if (!cnt)
351 return 0;
352
353 va_start(ap, fmt);
354 ret = vsnprintf(s->buffer + s->len, len, fmt, ap);
355 va_end(ap);
356
357 /* If we can't write it all, don't bother writing anything */
358 if (ret >= len)
359 return 0;
360
361 s->len += ret;
362
363 return len;
364}
365
366/**
367 * trace_seq_puts - trace sequence printing of simple string
368 * @s: trace sequence descriptor
369 * @str: simple string to record
370 *
371 * The tracer may use either the sequence operations or its own
372 * copy to user routines. This function records a simple string
373 * into a special buffer (@s) for later retrieval by a sequencer
374 * or other mechanism.
375 */
376static int
377trace_seq_puts(struct trace_seq *s, const char *str)
378{
379 int len = strlen(str);
380
381 if (len > ((PAGE_SIZE - 1) - s->len))
382 return 0;
383
384 memcpy(s->buffer + s->len, str, len);
385 s->len += len;
386
387 return len;
388}
389
390static int
391trace_seq_putc(struct trace_seq *s, unsigned char c)
392{
393 if (s->len >= (PAGE_SIZE - 1))
394 return 0;
395
396 s->buffer[s->len++] = c;
397
398 return 1;
399}
400
401static int
402trace_seq_putmem(struct trace_seq *s, void *mem, size_t len)
403{
404 if (len > ((PAGE_SIZE - 1) - s->len))
405 return 0; 351 return 0;
406 352
407 memcpy(s->buffer + s->len, mem, len); 353 if (s->len <= s->readpos)
408 s->len += len; 354 return -EBUSY;
409
410 return len;
411}
412
413#define MAX_MEMHEX_BYTES 8
414#define HEX_CHARS (MAX_MEMHEX_BYTES*2 + 1)
415
416static int
417trace_seq_putmem_hex(struct trace_seq *s, void *mem, size_t len)
418{
419 unsigned char hex[HEX_CHARS];
420 unsigned char *data = mem;
421 int i, j;
422
423#ifdef __BIG_ENDIAN
424 for (i = 0, j = 0; i < len; i++) {
425#else
426 for (i = len-1, j = 0; i >= 0; i--) {
427#endif
428 hex[j++] = hex_asc_hi(data[i]);
429 hex[j++] = hex_asc_lo(data[i]);
430 }
431 hex[j++] = ' ';
432
433 return trace_seq_putmem(s, hex, j);
434}
435
436static int
437trace_seq_path(struct trace_seq *s, struct path *path)
438{
439 unsigned char *p;
440 355
441 if (s->len >= (PAGE_SIZE - 1)) 356 len = s->len - s->readpos;
442 return 0; 357 if (cnt > len)
443 p = d_path(path, s->buffer + s->len, PAGE_SIZE - s->len); 358 cnt = len;
444 if (!IS_ERR(p)) { 359 ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
445 p = mangle_path(s->buffer + s->len, p, "\n"); 360 if (ret == cnt)
446 if (p) { 361 return -EFAULT;
447 s->len = p - s->buffer;
448 return 1;
449 }
450 } else {
451 s->buffer[s->len++] = '?';
452 return 1;
453 }
454 362
455 return 0; 363 cnt -= ret;
456}
457 364
458static void 365 s->readpos += cnt;
459trace_seq_reset(struct trace_seq *s) 366 return cnt;
460{
461 s->len = 0;
462 s->readpos = 0;
463} 367}
464 368
465ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt) 369ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
466{ 370{
467 int len; 371 int len;
468 int ret; 372 void *ret;
469 373
470 if (s->len <= s->readpos) 374 if (s->len <= s->readpos)
471 return -EBUSY; 375 return -EBUSY;
@@ -473,11 +377,11 @@ ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
473 len = s->len - s->readpos; 377 len = s->len - s->readpos;
474 if (cnt > len) 378 if (cnt > len)
475 cnt = len; 379 cnt = len;
476 ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt); 380 ret = memcpy(buf, s->buffer + s->readpos, cnt);
477 if (ret) 381 if (!ret)
478 return -EFAULT; 382 return -EFAULT;
479 383
480 s->readpos += len; 384 s->readpos += cnt;
481 return cnt; 385 return cnt;
482} 386}
483 387
@@ -489,7 +393,7 @@ trace_print_seq(struct seq_file *m, struct trace_seq *s)
489 s->buffer[len] = 0; 393 s->buffer[len] = 0;
490 seq_puts(m, s->buffer); 394 seq_puts(m, s->buffer);
491 395
492 trace_seq_reset(s); 396 trace_seq_init(s);
493} 397}
494 398
495/** 399/**
@@ -543,7 +447,7 @@ update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
543 447
544 ftrace_enable_cpu(); 448 ftrace_enable_cpu();
545 449
546 WARN_ON_ONCE(ret); 450 WARN_ON_ONCE(ret && ret != -EAGAIN);
547 451
548 __update_max_tr(tr, tsk, cpu); 452 __update_max_tr(tr, tsk, cpu);
549 __raw_spin_unlock(&ftrace_max_lock); 453 __raw_spin_unlock(&ftrace_max_lock);
@@ -556,6 +460,8 @@ update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
556 * Register a new plugin tracer. 460 * Register a new plugin tracer.
557 */ 461 */
558int register_tracer(struct tracer *type) 462int register_tracer(struct tracer *type)
463__releases(kernel_lock)
464__acquires(kernel_lock)
559{ 465{
560 struct tracer *t; 466 struct tracer *t;
561 int len; 467 int len;
@@ -594,9 +500,12 @@ int register_tracer(struct tracer *type)
594 else 500 else
595 if (!type->flags->opts) 501 if (!type->flags->opts)
596 type->flags->opts = dummy_tracer_opt; 502 type->flags->opts = dummy_tracer_opt;
503 if (!type->wait_pipe)
504 type->wait_pipe = default_wait_pipe;
505
597 506
598#ifdef CONFIG_FTRACE_STARTUP_TEST 507#ifdef CONFIG_FTRACE_STARTUP_TEST
599 if (type->selftest) { 508 if (type->selftest && !tracing_selftest_disabled) {
600 struct tracer *saved_tracer = current_trace; 509 struct tracer *saved_tracer = current_trace;
601 struct trace_array *tr = &global_trace; 510 struct trace_array *tr = &global_trace;
602 int i; 511 int i;
@@ -638,8 +547,26 @@ int register_tracer(struct tracer *type)
638 out: 547 out:
639 tracing_selftest_running = false; 548 tracing_selftest_running = false;
640 mutex_unlock(&trace_types_lock); 549 mutex_unlock(&trace_types_lock);
641 lock_kernel();
642 550
551 if (ret || !default_bootup_tracer)
552 goto out_unlock;
553
554 if (strncmp(default_bootup_tracer, type->name, BOOTUP_TRACER_SIZE))
555 goto out_unlock;
556
557 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
558 /* Do we want this tracer to start on bootup? */
559 tracing_set_tracer(type->name);
560 default_bootup_tracer = NULL;
561 /* disable other selftests, since this will break it. */
562 tracing_selftest_disabled = 1;
563#ifdef CONFIG_FTRACE_STARTUP_TEST
564 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
565 type->name);
566#endif
567
568 out_unlock:
569 lock_kernel();
643 return ret; 570 return ret;
644} 571}
645 572
@@ -658,6 +585,15 @@ void unregister_tracer(struct tracer *type)
658 585
659 found: 586 found:
660 *t = (*t)->next; 587 *t = (*t)->next;
588
589 if (type == current_trace && tracer_enabled) {
590 tracer_enabled = 0;
591 tracing_stop();
592 if (current_trace->stop)
593 current_trace->stop(&global_trace);
594 current_trace = &nop_trace;
595 }
596
661 if (strlen(type->name) != max_tracer_type_len) 597 if (strlen(type->name) != max_tracer_type_len)
662 goto out; 598 goto out;
663 599
@@ -693,10 +629,10 @@ static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
693static unsigned map_cmdline_to_pid[SAVED_CMDLINES]; 629static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
694static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN]; 630static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
695static int cmdline_idx; 631static int cmdline_idx;
696static DEFINE_SPINLOCK(trace_cmdline_lock); 632static raw_spinlock_t trace_cmdline_lock = __RAW_SPIN_LOCK_UNLOCKED;
697 633
698/* temporary disable recording */ 634/* temporary disable recording */
699atomic_t trace_record_cmdline_disabled __read_mostly; 635static atomic_t trace_record_cmdline_disabled __read_mostly;
700 636
701static void trace_init_cmdlines(void) 637static void trace_init_cmdlines(void)
702{ 638{
@@ -738,13 +674,12 @@ void tracing_start(void)
738 return; 674 return;
739 675
740 spin_lock_irqsave(&tracing_start_lock, flags); 676 spin_lock_irqsave(&tracing_start_lock, flags);
741 if (--trace_stop_count) 677 if (--trace_stop_count) {
742 goto out; 678 if (trace_stop_count < 0) {
743 679 /* Someone screwed up their debugging */
744 if (trace_stop_count < 0) { 680 WARN_ON_ONCE(1);
745 /* Someone screwed up their debugging */ 681 trace_stop_count = 0;
746 WARN_ON_ONCE(1); 682 }
747 trace_stop_count = 0;
748 goto out; 683 goto out;
749 } 684 }
750 685
@@ -806,7 +741,7 @@ static void trace_save_cmdline(struct task_struct *tsk)
806 * nor do we want to disable interrupts, 741 * nor do we want to disable interrupts,
807 * so if we miss here, then better luck next time. 742 * so if we miss here, then better luck next time.
808 */ 743 */
809 if (!spin_trylock(&trace_cmdline_lock)) 744 if (!__raw_spin_trylock(&trace_cmdline_lock))
810 return; 745 return;
811 746
812 idx = map_pid_to_cmdline[tsk->pid]; 747 idx = map_pid_to_cmdline[tsk->pid];
@@ -824,7 +759,7 @@ static void trace_save_cmdline(struct task_struct *tsk)
824 759
825 memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN); 760 memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
826 761
827 spin_unlock(&trace_cmdline_lock); 762 __raw_spin_unlock(&trace_cmdline_lock);
828} 763}
829 764
830char *trace_find_cmdline(int pid) 765char *trace_find_cmdline(int pid)
@@ -876,78 +811,114 @@ tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
876 (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0); 811 (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
877} 812}
878 813
814struct ring_buffer_event *trace_buffer_lock_reserve(struct trace_array *tr,
815 unsigned char type,
816 unsigned long len,
817 unsigned long flags, int pc)
818{
819 struct ring_buffer_event *event;
820
821 event = ring_buffer_lock_reserve(tr->buffer, len);
822 if (event != NULL) {
823 struct trace_entry *ent = ring_buffer_event_data(event);
824
825 tracing_generic_entry_update(ent, flags, pc);
826 ent->type = type;
827 }
828
829 return event;
830}
831static void ftrace_trace_stack(struct trace_array *tr,
832 unsigned long flags, int skip, int pc);
833static void ftrace_trace_userstack(struct trace_array *tr,
834 unsigned long flags, int pc);
835
836void trace_buffer_unlock_commit(struct trace_array *tr,
837 struct ring_buffer_event *event,
838 unsigned long flags, int pc)
839{
840 ring_buffer_unlock_commit(tr->buffer, event);
841
842 ftrace_trace_stack(tr, flags, 6, pc);
843 ftrace_trace_userstack(tr, flags, pc);
844 trace_wake_up();
845}
846
847struct ring_buffer_event *
848trace_current_buffer_lock_reserve(unsigned char type, unsigned long len,
849 unsigned long flags, int pc)
850{
851 return trace_buffer_lock_reserve(&global_trace,
852 type, len, flags, pc);
853}
854
855void trace_current_buffer_unlock_commit(struct ring_buffer_event *event,
856 unsigned long flags, int pc)
857{
858 return trace_buffer_unlock_commit(&global_trace, event, flags, pc);
859}
860
879void 861void
880trace_function(struct trace_array *tr, struct trace_array_cpu *data, 862trace_function(struct trace_array *tr,
881 unsigned long ip, unsigned long parent_ip, unsigned long flags, 863 unsigned long ip, unsigned long parent_ip, unsigned long flags,
882 int pc) 864 int pc)
883{ 865{
884 struct ring_buffer_event *event; 866 struct ring_buffer_event *event;
885 struct ftrace_entry *entry; 867 struct ftrace_entry *entry;
886 unsigned long irq_flags;
887 868
888 /* If we are reading the ring buffer, don't trace */ 869 /* If we are reading the ring buffer, don't trace */
889 if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled)))) 870 if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled))))
890 return; 871 return;
891 872
892 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry), 873 event = trace_buffer_lock_reserve(tr, TRACE_FN, sizeof(*entry),
893 &irq_flags); 874 flags, pc);
894 if (!event) 875 if (!event)
895 return; 876 return;
896 entry = ring_buffer_event_data(event); 877 entry = ring_buffer_event_data(event);
897 tracing_generic_entry_update(&entry->ent, flags, pc);
898 entry->ent.type = TRACE_FN;
899 entry->ip = ip; 878 entry->ip = ip;
900 entry->parent_ip = parent_ip; 879 entry->parent_ip = parent_ip;
901 ring_buffer_unlock_commit(tr->buffer, event, irq_flags); 880 ring_buffer_unlock_commit(tr->buffer, event);
902} 881}
903 882
904#ifdef CONFIG_FUNCTION_GRAPH_TRACER 883#ifdef CONFIG_FUNCTION_GRAPH_TRACER
905static void __trace_graph_entry(struct trace_array *tr, 884static void __trace_graph_entry(struct trace_array *tr,
906 struct trace_array_cpu *data,
907 struct ftrace_graph_ent *trace, 885 struct ftrace_graph_ent *trace,
908 unsigned long flags, 886 unsigned long flags,
909 int pc) 887 int pc)
910{ 888{
911 struct ring_buffer_event *event; 889 struct ring_buffer_event *event;
912 struct ftrace_graph_ent_entry *entry; 890 struct ftrace_graph_ent_entry *entry;
913 unsigned long irq_flags;
914 891
915 if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled)))) 892 if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled))))
916 return; 893 return;
917 894
918 event = ring_buffer_lock_reserve(global_trace.buffer, sizeof(*entry), 895 event = trace_buffer_lock_reserve(&global_trace, TRACE_GRAPH_ENT,
919 &irq_flags); 896 sizeof(*entry), flags, pc);
920 if (!event) 897 if (!event)
921 return; 898 return;
922 entry = ring_buffer_event_data(event); 899 entry = ring_buffer_event_data(event);
923 tracing_generic_entry_update(&entry->ent, flags, pc);
924 entry->ent.type = TRACE_GRAPH_ENT;
925 entry->graph_ent = *trace; 900 entry->graph_ent = *trace;
926 ring_buffer_unlock_commit(global_trace.buffer, event, irq_flags); 901 ring_buffer_unlock_commit(global_trace.buffer, event);
927} 902}
928 903
929static void __trace_graph_return(struct trace_array *tr, 904static void __trace_graph_return(struct trace_array *tr,
930 struct trace_array_cpu *data,
931 struct ftrace_graph_ret *trace, 905 struct ftrace_graph_ret *trace,
932 unsigned long flags, 906 unsigned long flags,
933 int pc) 907 int pc)
934{ 908{
935 struct ring_buffer_event *event; 909 struct ring_buffer_event *event;
936 struct ftrace_graph_ret_entry *entry; 910 struct ftrace_graph_ret_entry *entry;
937 unsigned long irq_flags;
938 911
939 if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled)))) 912 if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled))))
940 return; 913 return;
941 914
942 event = ring_buffer_lock_reserve(global_trace.buffer, sizeof(*entry), 915 event = trace_buffer_lock_reserve(&global_trace, TRACE_GRAPH_RET,
943 &irq_flags); 916 sizeof(*entry), flags, pc);
944 if (!event) 917 if (!event)
945 return; 918 return;
946 entry = ring_buffer_event_data(event); 919 entry = ring_buffer_event_data(event);
947 tracing_generic_entry_update(&entry->ent, flags, pc);
948 entry->ent.type = TRACE_GRAPH_RET;
949 entry->ret = *trace; 920 entry->ret = *trace;
950 ring_buffer_unlock_commit(global_trace.buffer, event, irq_flags); 921 ring_buffer_unlock_commit(global_trace.buffer, event);
951} 922}
952#endif 923#endif
953 924
@@ -957,31 +928,23 @@ ftrace(struct trace_array *tr, struct trace_array_cpu *data,
957 int pc) 928 int pc)
958{ 929{
959 if (likely(!atomic_read(&data->disabled))) 930 if (likely(!atomic_read(&data->disabled)))
960 trace_function(tr, data, ip, parent_ip, flags, pc); 931 trace_function(tr, ip, parent_ip, flags, pc);
961} 932}
962 933
963static void ftrace_trace_stack(struct trace_array *tr, 934static void __ftrace_trace_stack(struct trace_array *tr,
964 struct trace_array_cpu *data, 935 unsigned long flags,
965 unsigned long flags, 936 int skip, int pc)
966 int skip, int pc)
967{ 937{
968#ifdef CONFIG_STACKTRACE 938#ifdef CONFIG_STACKTRACE
969 struct ring_buffer_event *event; 939 struct ring_buffer_event *event;
970 struct stack_entry *entry; 940 struct stack_entry *entry;
971 struct stack_trace trace; 941 struct stack_trace trace;
972 unsigned long irq_flags;
973 942
974 if (!(trace_flags & TRACE_ITER_STACKTRACE)) 943 event = trace_buffer_lock_reserve(tr, TRACE_STACK,
975 return; 944 sizeof(*entry), flags, pc);
976
977 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
978 &irq_flags);
979 if (!event) 945 if (!event)
980 return; 946 return;
981 entry = ring_buffer_event_data(event); 947 entry = ring_buffer_event_data(event);
982 tracing_generic_entry_update(&entry->ent, flags, pc);
983 entry->ent.type = TRACE_STACK;
984
985 memset(&entry->caller, 0, sizeof(entry->caller)); 948 memset(&entry->caller, 0, sizeof(entry->caller));
986 949
987 trace.nr_entries = 0; 950 trace.nr_entries = 0;
@@ -990,38 +953,43 @@ static void ftrace_trace_stack(struct trace_array *tr,
990 trace.entries = entry->caller; 953 trace.entries = entry->caller;
991 954
992 save_stack_trace(&trace); 955 save_stack_trace(&trace);
993 ring_buffer_unlock_commit(tr->buffer, event, irq_flags); 956 ring_buffer_unlock_commit(tr->buffer, event);
994#endif 957#endif
995} 958}
996 959
960static void ftrace_trace_stack(struct trace_array *tr,
961 unsigned long flags,
962 int skip, int pc)
963{
964 if (!(trace_flags & TRACE_ITER_STACKTRACE))
965 return;
966
967 __ftrace_trace_stack(tr, flags, skip, pc);
968}
969
997void __trace_stack(struct trace_array *tr, 970void __trace_stack(struct trace_array *tr,
998 struct trace_array_cpu *data,
999 unsigned long flags, 971 unsigned long flags,
1000 int skip) 972 int skip, int pc)
1001{ 973{
1002 ftrace_trace_stack(tr, data, flags, skip, preempt_count()); 974 __ftrace_trace_stack(tr, flags, skip, pc);
1003} 975}
1004 976
1005static void ftrace_trace_userstack(struct trace_array *tr, 977static void ftrace_trace_userstack(struct trace_array *tr,
1006 struct trace_array_cpu *data, 978 unsigned long flags, int pc)
1007 unsigned long flags, int pc)
1008{ 979{
1009#ifdef CONFIG_STACKTRACE 980#ifdef CONFIG_STACKTRACE
1010 struct ring_buffer_event *event; 981 struct ring_buffer_event *event;
1011 struct userstack_entry *entry; 982 struct userstack_entry *entry;
1012 struct stack_trace trace; 983 struct stack_trace trace;
1013 unsigned long irq_flags;
1014 984
1015 if (!(trace_flags & TRACE_ITER_USERSTACKTRACE)) 985 if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1016 return; 986 return;
1017 987
1018 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry), 988 event = trace_buffer_lock_reserve(tr, TRACE_USER_STACK,
1019 &irq_flags); 989 sizeof(*entry), flags, pc);
1020 if (!event) 990 if (!event)
1021 return; 991 return;
1022 entry = ring_buffer_event_data(event); 992 entry = ring_buffer_event_data(event);
1023 tracing_generic_entry_update(&entry->ent, flags, pc);
1024 entry->ent.type = TRACE_USER_STACK;
1025 993
1026 memset(&entry->caller, 0, sizeof(entry->caller)); 994 memset(&entry->caller, 0, sizeof(entry->caller));
1027 995
@@ -1031,70 +999,58 @@ static void ftrace_trace_userstack(struct trace_array *tr,
1031 trace.entries = entry->caller; 999 trace.entries = entry->caller;
1032 1000
1033 save_stack_trace_user(&trace); 1001 save_stack_trace_user(&trace);
1034 ring_buffer_unlock_commit(tr->buffer, event, irq_flags); 1002 ring_buffer_unlock_commit(tr->buffer, event);
1035#endif 1003#endif
1036} 1004}
1037 1005
1038void __trace_userstack(struct trace_array *tr, 1006#ifdef UNUSED
1039 struct trace_array_cpu *data, 1007static void __trace_userstack(struct trace_array *tr, unsigned long flags)
1040 unsigned long flags)
1041{ 1008{
1042 ftrace_trace_userstack(tr, data, flags, preempt_count()); 1009 ftrace_trace_userstack(tr, flags, preempt_count());
1043} 1010}
1011#endif /* UNUSED */
1044 1012
1045static void 1013static void
1046ftrace_trace_special(void *__tr, void *__data, 1014ftrace_trace_special(void *__tr,
1047 unsigned long arg1, unsigned long arg2, unsigned long arg3, 1015 unsigned long arg1, unsigned long arg2, unsigned long arg3,
1048 int pc) 1016 int pc)
1049{ 1017{
1050 struct ring_buffer_event *event; 1018 struct ring_buffer_event *event;
1051 struct trace_array_cpu *data = __data;
1052 struct trace_array *tr = __tr; 1019 struct trace_array *tr = __tr;
1053 struct special_entry *entry; 1020 struct special_entry *entry;
1054 unsigned long irq_flags;
1055 1021
1056 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry), 1022 event = trace_buffer_lock_reserve(tr, TRACE_SPECIAL,
1057 &irq_flags); 1023 sizeof(*entry), 0, pc);
1058 if (!event) 1024 if (!event)
1059 return; 1025 return;
1060 entry = ring_buffer_event_data(event); 1026 entry = ring_buffer_event_data(event);
1061 tracing_generic_entry_update(&entry->ent, 0, pc);
1062 entry->ent.type = TRACE_SPECIAL;
1063 entry->arg1 = arg1; 1027 entry->arg1 = arg1;
1064 entry->arg2 = arg2; 1028 entry->arg2 = arg2;
1065 entry->arg3 = arg3; 1029 entry->arg3 = arg3;
1066 ring_buffer_unlock_commit(tr->buffer, event, irq_flags); 1030 trace_buffer_unlock_commit(tr, event, 0, pc);
1067 ftrace_trace_stack(tr, data, irq_flags, 4, pc);
1068 ftrace_trace_userstack(tr, data, irq_flags, pc);
1069
1070 trace_wake_up();
1071} 1031}
1072 1032
1073void 1033void
1074__trace_special(void *__tr, void *__data, 1034__trace_special(void *__tr, void *__data,
1075 unsigned long arg1, unsigned long arg2, unsigned long arg3) 1035 unsigned long arg1, unsigned long arg2, unsigned long arg3)
1076{ 1036{
1077 ftrace_trace_special(__tr, __data, arg1, arg2, arg3, preempt_count()); 1037 ftrace_trace_special(__tr, arg1, arg2, arg3, preempt_count());
1078} 1038}
1079 1039
1080void 1040void
1081tracing_sched_switch_trace(struct trace_array *tr, 1041tracing_sched_switch_trace(struct trace_array *tr,
1082 struct trace_array_cpu *data,
1083 struct task_struct *prev, 1042 struct task_struct *prev,
1084 struct task_struct *next, 1043 struct task_struct *next,
1085 unsigned long flags, int pc) 1044 unsigned long flags, int pc)
1086{ 1045{
1087 struct ring_buffer_event *event; 1046 struct ring_buffer_event *event;
1088 struct ctx_switch_entry *entry; 1047 struct ctx_switch_entry *entry;
1089 unsigned long irq_flags;
1090 1048
1091 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry), 1049 event = trace_buffer_lock_reserve(tr, TRACE_CTX,
1092 &irq_flags); 1050 sizeof(*entry), flags, pc);
1093 if (!event) 1051 if (!event)
1094 return; 1052 return;
1095 entry = ring_buffer_event_data(event); 1053 entry = ring_buffer_event_data(event);
1096 tracing_generic_entry_update(&entry->ent, flags, pc);
1097 entry->ent.type = TRACE_CTX;
1098 entry->prev_pid = prev->pid; 1054 entry->prev_pid = prev->pid;
1099 entry->prev_prio = prev->prio; 1055 entry->prev_prio = prev->prio;
1100 entry->prev_state = prev->state; 1056 entry->prev_state = prev->state;
@@ -1102,29 +1058,23 @@ tracing_sched_switch_trace(struct trace_array *tr,
1102 entry->next_prio = next->prio; 1058 entry->next_prio = next->prio;
1103 entry->next_state = next->state; 1059 entry->next_state = next->state;
1104 entry->next_cpu = task_cpu(next); 1060 entry->next_cpu = task_cpu(next);
1105 ring_buffer_unlock_commit(tr->buffer, event, irq_flags); 1061 trace_buffer_unlock_commit(tr, event, flags, pc);
1106 ftrace_trace_stack(tr, data, flags, 5, pc);
1107 ftrace_trace_userstack(tr, data, flags, pc);
1108} 1062}
1109 1063
1110void 1064void
1111tracing_sched_wakeup_trace(struct trace_array *tr, 1065tracing_sched_wakeup_trace(struct trace_array *tr,
1112 struct trace_array_cpu *data,
1113 struct task_struct *wakee, 1066 struct task_struct *wakee,
1114 struct task_struct *curr, 1067 struct task_struct *curr,
1115 unsigned long flags, int pc) 1068 unsigned long flags, int pc)
1116{ 1069{
1117 struct ring_buffer_event *event; 1070 struct ring_buffer_event *event;
1118 struct ctx_switch_entry *entry; 1071 struct ctx_switch_entry *entry;
1119 unsigned long irq_flags;
1120 1072
1121 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry), 1073 event = trace_buffer_lock_reserve(tr, TRACE_WAKE,
1122 &irq_flags); 1074 sizeof(*entry), flags, pc);
1123 if (!event) 1075 if (!event)
1124 return; 1076 return;
1125 entry = ring_buffer_event_data(event); 1077 entry = ring_buffer_event_data(event);
1126 tracing_generic_entry_update(&entry->ent, flags, pc);
1127 entry->ent.type = TRACE_WAKE;
1128 entry->prev_pid = curr->pid; 1078 entry->prev_pid = curr->pid;
1129 entry->prev_prio = curr->prio; 1079 entry->prev_prio = curr->prio;
1130 entry->prev_state = curr->state; 1080 entry->prev_state = curr->state;
@@ -1132,11 +1082,10 @@ tracing_sched_wakeup_trace(struct trace_array *tr,
1132 entry->next_prio = wakee->prio; 1082 entry->next_prio = wakee->prio;
1133 entry->next_state = wakee->state; 1083 entry->next_state = wakee->state;
1134 entry->next_cpu = task_cpu(wakee); 1084 entry->next_cpu = task_cpu(wakee);
1135 ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
1136 ftrace_trace_stack(tr, data, flags, 6, pc);
1137 ftrace_trace_userstack(tr, data, flags, pc);
1138 1085
1139 trace_wake_up(); 1086 ring_buffer_unlock_commit(tr->buffer, event);
1087 ftrace_trace_stack(tr, flags, 6, pc);
1088 ftrace_trace_userstack(tr, flags, pc);
1140} 1089}
1141 1090
1142void 1091void
@@ -1157,66 +1106,7 @@ ftrace_special(unsigned long arg1, unsigned long arg2, unsigned long arg3)
1157 data = tr->data[cpu]; 1106 data = tr->data[cpu];
1158 1107
1159 if (likely(atomic_inc_return(&data->disabled) == 1)) 1108 if (likely(atomic_inc_return(&data->disabled) == 1))
1160 ftrace_trace_special(tr, data, arg1, arg2, arg3, pc); 1109 ftrace_trace_special(tr, arg1, arg2, arg3, pc);
1161
1162 atomic_dec(&data->disabled);
1163 local_irq_restore(flags);
1164}
1165
1166#ifdef CONFIG_FUNCTION_TRACER
1167static void
1168function_trace_call_preempt_only(unsigned long ip, unsigned long parent_ip)
1169{
1170 struct trace_array *tr = &global_trace;
1171 struct trace_array_cpu *data;
1172 unsigned long flags;
1173 long disabled;
1174 int cpu, resched;
1175 int pc;
1176
1177 if (unlikely(!ftrace_function_enabled))
1178 return;
1179
1180 pc = preempt_count();
1181 resched = ftrace_preempt_disable();
1182 local_save_flags(flags);
1183 cpu = raw_smp_processor_id();
1184 data = tr->data[cpu];
1185 disabled = atomic_inc_return(&data->disabled);
1186
1187 if (likely(disabled == 1))
1188 trace_function(tr, data, ip, parent_ip, flags, pc);
1189
1190 atomic_dec(&data->disabled);
1191 ftrace_preempt_enable(resched);
1192}
1193
1194static void
1195function_trace_call(unsigned long ip, unsigned long parent_ip)
1196{
1197 struct trace_array *tr = &global_trace;
1198 struct trace_array_cpu *data;
1199 unsigned long flags;
1200 long disabled;
1201 int cpu;
1202 int pc;
1203
1204 if (unlikely(!ftrace_function_enabled))
1205 return;
1206
1207 /*
1208 * Need to use raw, since this must be called before the
1209 * recursive protection is performed.
1210 */
1211 local_irq_save(flags);
1212 cpu = raw_smp_processor_id();
1213 data = tr->data[cpu];
1214 disabled = atomic_inc_return(&data->disabled);
1215
1216 if (likely(disabled == 1)) {
1217 pc = preempt_count();
1218 trace_function(tr, data, ip, parent_ip, flags, pc);
1219 }
1220 1110
1221 atomic_dec(&data->disabled); 1111 atomic_dec(&data->disabled);
1222 local_irq_restore(flags); 1112 local_irq_restore(flags);
@@ -1244,7 +1134,7 @@ int trace_graph_entry(struct ftrace_graph_ent *trace)
1244 disabled = atomic_inc_return(&data->disabled); 1134 disabled = atomic_inc_return(&data->disabled);
1245 if (likely(disabled == 1)) { 1135 if (likely(disabled == 1)) {
1246 pc = preempt_count(); 1136 pc = preempt_count();
1247 __trace_graph_entry(tr, data, trace, flags, pc); 1137 __trace_graph_entry(tr, trace, flags, pc);
1248 } 1138 }
1249 /* Only do the atomic if it is not already set */ 1139 /* Only do the atomic if it is not already set */
1250 if (!test_tsk_trace_graph(current)) 1140 if (!test_tsk_trace_graph(current))
@@ -1270,7 +1160,7 @@ void trace_graph_return(struct ftrace_graph_ret *trace)
1270 disabled = atomic_inc_return(&data->disabled); 1160 disabled = atomic_inc_return(&data->disabled);
1271 if (likely(disabled == 1)) { 1161 if (likely(disabled == 1)) {
1272 pc = preempt_count(); 1162 pc = preempt_count();
1273 __trace_graph_return(tr, data, trace, flags, pc); 1163 __trace_graph_return(tr, trace, flags, pc);
1274 } 1164 }
1275 if (!trace->depth) 1165 if (!trace->depth)
1276 clear_tsk_trace_graph(current); 1166 clear_tsk_trace_graph(current);
@@ -1279,30 +1169,66 @@ void trace_graph_return(struct ftrace_graph_ret *trace)
1279} 1169}
1280#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 1170#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
1281 1171
1282static struct ftrace_ops trace_ops __read_mostly =
1283{
1284 .func = function_trace_call,
1285};
1286 1172
1287void tracing_start_function_trace(void) 1173/**
1174 * trace_vprintk - write binary msg to tracing buffer
1175 *
1176 */
1177int trace_vprintk(unsigned long ip, int depth, const char *fmt, va_list args)
1288{ 1178{
1289 ftrace_function_enabled = 0; 1179 static DEFINE_SPINLOCK(trace_buf_lock);
1180 static u32 trace_buf[TRACE_BUF_SIZE];
1290 1181
1291 if (trace_flags & TRACE_ITER_PREEMPTONLY) 1182 struct ring_buffer_event *event;
1292 trace_ops.func = function_trace_call_preempt_only; 1183 struct trace_array *tr = &global_trace;
1293 else 1184 struct trace_array_cpu *data;
1294 trace_ops.func = function_trace_call; 1185 struct print_entry *entry;
1186 unsigned long flags;
1187 int resched;
1188 int cpu, len = 0, size, pc;
1295 1189
1296 register_ftrace_function(&trace_ops); 1190 if (unlikely(tracing_selftest_running || tracing_disabled))
1297 ftrace_function_enabled = 1; 1191 return 0;
1298}
1299 1192
1300void tracing_stop_function_trace(void) 1193 /* Don't pollute graph traces with trace_vprintk internals */
1301{ 1194 pause_graph_tracing();
1302 ftrace_function_enabled = 0; 1195
1303 unregister_ftrace_function(&trace_ops); 1196 pc = preempt_count();
1197 resched = ftrace_preempt_disable();
1198 cpu = raw_smp_processor_id();
1199 data = tr->data[cpu];
1200
1201 if (unlikely(atomic_read(&data->disabled)))
1202 goto out;
1203
1204 spin_lock_irqsave(&trace_buf_lock, flags);
1205 len = vbin_printf(trace_buf, TRACE_BUF_SIZE, fmt, args);
1206
1207 if (len > TRACE_BUF_SIZE || len < 0)
1208 goto out_unlock;
1209
1210 size = sizeof(*entry) + sizeof(u32) * len;
1211 event = trace_buffer_lock_reserve(tr, TRACE_PRINT, size, flags, pc);
1212 if (!event)
1213 goto out_unlock;
1214 entry = ring_buffer_event_data(event);
1215 entry->ip = ip;
1216 entry->depth = depth;
1217 entry->fmt = fmt;
1218
1219 memcpy(entry->buf, trace_buf, sizeof(u32) * len);
1220 ring_buffer_unlock_commit(tr->buffer, event);
1221
1222out_unlock:
1223 spin_unlock_irqrestore(&trace_buf_lock, flags);
1224
1225out:
1226 ftrace_preempt_enable(resched);
1227 unpause_graph_tracing();
1228
1229 return len;
1304} 1230}
1305#endif 1231EXPORT_SYMBOL_GPL(trace_vprintk);
1306 1232
1307enum trace_file_type { 1233enum trace_file_type {
1308 TRACE_FILE_LAT_FMT = 1, 1234 TRACE_FILE_LAT_FMT = 1,
@@ -1345,10 +1271,25 @@ __find_next_entry(struct trace_iterator *iter, int *ent_cpu, u64 *ent_ts)
1345{ 1271{
1346 struct ring_buffer *buffer = iter->tr->buffer; 1272 struct ring_buffer *buffer = iter->tr->buffer;
1347 struct trace_entry *ent, *next = NULL; 1273 struct trace_entry *ent, *next = NULL;
1274 int cpu_file = iter->cpu_file;
1348 u64 next_ts = 0, ts; 1275 u64 next_ts = 0, ts;
1349 int next_cpu = -1; 1276 int next_cpu = -1;
1350 int cpu; 1277 int cpu;
1351 1278
1279 /*
1280 * If we are in a per_cpu trace file, don't bother by iterating over
1281 * all cpu and peek directly.
1282 */
1283 if (cpu_file > TRACE_PIPE_ALL_CPU) {
1284 if (ring_buffer_empty_cpu(buffer, cpu_file))
1285 return NULL;
1286 ent = peek_next_entry(iter, cpu_file, ent_ts);
1287 if (ent_cpu)
1288 *ent_cpu = cpu_file;
1289
1290 return ent;
1291 }
1292
1352 for_each_tracing_cpu(cpu) { 1293 for_each_tracing_cpu(cpu) {
1353 1294
1354 if (ring_buffer_empty_cpu(buffer, cpu)) 1295 if (ring_buffer_empty_cpu(buffer, cpu))
@@ -1376,8 +1317,8 @@ __find_next_entry(struct trace_iterator *iter, int *ent_cpu, u64 *ent_ts)
1376} 1317}
1377 1318
1378/* Find the next real entry, without updating the iterator itself */ 1319/* Find the next real entry, without updating the iterator itself */
1379static struct trace_entry * 1320struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
1380find_next_entry(struct trace_iterator *iter, int *ent_cpu, u64 *ent_ts) 1321 int *ent_cpu, u64 *ent_ts)
1381{ 1322{
1382 return __find_next_entry(iter, ent_cpu, ent_ts); 1323 return __find_next_entry(iter, ent_cpu, ent_ts);
1383} 1324}
@@ -1426,19 +1367,32 @@ static void *s_next(struct seq_file *m, void *v, loff_t *pos)
1426 return ent; 1367 return ent;
1427} 1368}
1428 1369
1370/*
1371 * No necessary locking here. The worst thing which can
1372 * happen is loosing events consumed at the same time
1373 * by a trace_pipe reader.
1374 * Other than that, we don't risk to crash the ring buffer
1375 * because it serializes the readers.
1376 *
1377 * The current tracer is copied to avoid a global locking
1378 * all around.
1379 */
1429static void *s_start(struct seq_file *m, loff_t *pos) 1380static void *s_start(struct seq_file *m, loff_t *pos)
1430{ 1381{
1431 struct trace_iterator *iter = m->private; 1382 struct trace_iterator *iter = m->private;
1383 static struct tracer *old_tracer;
1384 int cpu_file = iter->cpu_file;
1432 void *p = NULL; 1385 void *p = NULL;
1433 loff_t l = 0; 1386 loff_t l = 0;
1434 int cpu; 1387 int cpu;
1435 1388
1389 /* copy the tracer to avoid using a global lock all around */
1436 mutex_lock(&trace_types_lock); 1390 mutex_lock(&trace_types_lock);
1437 1391 if (unlikely(old_tracer != current_trace && current_trace)) {
1438 if (!current_trace || current_trace != iter->trace) { 1392 old_tracer = current_trace;
1439 mutex_unlock(&trace_types_lock); 1393 *iter->trace = *current_trace;
1440 return NULL;
1441 } 1394 }
1395 mutex_unlock(&trace_types_lock);
1442 1396
1443 atomic_inc(&trace_record_cmdline_disabled); 1397 atomic_inc(&trace_record_cmdline_disabled);
1444 1398
@@ -1449,9 +1403,12 @@ static void *s_start(struct seq_file *m, loff_t *pos)
1449 1403
1450 ftrace_disable_cpu(); 1404 ftrace_disable_cpu();
1451 1405
1452 for_each_tracing_cpu(cpu) { 1406 if (cpu_file == TRACE_PIPE_ALL_CPU) {
1453 ring_buffer_iter_reset(iter->buffer_iter[cpu]); 1407 for_each_tracing_cpu(cpu)
1454 } 1408 ring_buffer_iter_reset(iter->buffer_iter[cpu]);
1409 } else
1410 ring_buffer_iter_reset(iter->buffer_iter[cpu_file]);
1411
1455 1412
1456 ftrace_enable_cpu(); 1413 ftrace_enable_cpu();
1457 1414
@@ -1469,155 +1426,6 @@ static void *s_start(struct seq_file *m, loff_t *pos)
1469static void s_stop(struct seq_file *m, void *p) 1426static void s_stop(struct seq_file *m, void *p)
1470{ 1427{
1471 atomic_dec(&trace_record_cmdline_disabled); 1428 atomic_dec(&trace_record_cmdline_disabled);
1472 mutex_unlock(&trace_types_lock);
1473}
1474
1475#ifdef CONFIG_KRETPROBES
1476static inline const char *kretprobed(const char *name)
1477{
1478 static const char tramp_name[] = "kretprobe_trampoline";
1479 int size = sizeof(tramp_name);
1480
1481 if (strncmp(tramp_name, name, size) == 0)
1482 return "[unknown/kretprobe'd]";
1483 return name;
1484}
1485#else
1486static inline const char *kretprobed(const char *name)
1487{
1488 return name;
1489}
1490#endif /* CONFIG_KRETPROBES */
1491
1492static int
1493seq_print_sym_short(struct trace_seq *s, const char *fmt, unsigned long address)
1494{
1495#ifdef CONFIG_KALLSYMS
1496 char str[KSYM_SYMBOL_LEN];
1497 const char *name;
1498
1499 kallsyms_lookup(address, NULL, NULL, NULL, str);
1500
1501 name = kretprobed(str);
1502
1503 return trace_seq_printf(s, fmt, name);
1504#endif
1505 return 1;
1506}
1507
1508static int
1509seq_print_sym_offset(struct trace_seq *s, const char *fmt,
1510 unsigned long address)
1511{
1512#ifdef CONFIG_KALLSYMS
1513 char str[KSYM_SYMBOL_LEN];
1514 const char *name;
1515
1516 sprint_symbol(str, address);
1517 name = kretprobed(str);
1518
1519 return trace_seq_printf(s, fmt, name);
1520#endif
1521 return 1;
1522}
1523
1524#ifndef CONFIG_64BIT
1525# define IP_FMT "%08lx"
1526#else
1527# define IP_FMT "%016lx"
1528#endif
1529
1530int
1531seq_print_ip_sym(struct trace_seq *s, unsigned long ip, unsigned long sym_flags)
1532{
1533 int ret;
1534
1535 if (!ip)
1536 return trace_seq_printf(s, "0");
1537
1538 if (sym_flags & TRACE_ITER_SYM_OFFSET)
1539 ret = seq_print_sym_offset(s, "%s", ip);
1540 else
1541 ret = seq_print_sym_short(s, "%s", ip);
1542
1543 if (!ret)
1544 return 0;
1545
1546 if (sym_flags & TRACE_ITER_SYM_ADDR)
1547 ret = trace_seq_printf(s, " <" IP_FMT ">", ip);
1548 return ret;
1549}
1550
1551static inline int seq_print_user_ip(struct trace_seq *s, struct mm_struct *mm,
1552 unsigned long ip, unsigned long sym_flags)
1553{
1554 struct file *file = NULL;
1555 unsigned long vmstart = 0;
1556 int ret = 1;
1557
1558 if (mm) {
1559 const struct vm_area_struct *vma;
1560
1561 down_read(&mm->mmap_sem);
1562 vma = find_vma(mm, ip);
1563 if (vma) {
1564 file = vma->vm_file;
1565 vmstart = vma->vm_start;
1566 }
1567 if (file) {
1568 ret = trace_seq_path(s, &file->f_path);
1569 if (ret)
1570 ret = trace_seq_printf(s, "[+0x%lx]", ip - vmstart);
1571 }
1572 up_read(&mm->mmap_sem);
1573 }
1574 if (ret && ((sym_flags & TRACE_ITER_SYM_ADDR) || !file))
1575 ret = trace_seq_printf(s, " <" IP_FMT ">", ip);
1576 return ret;
1577}
1578
1579static int
1580seq_print_userip_objs(const struct userstack_entry *entry, struct trace_seq *s,
1581 unsigned long sym_flags)
1582{
1583 struct mm_struct *mm = NULL;
1584 int ret = 1;
1585 unsigned int i;
1586
1587 if (trace_flags & TRACE_ITER_SYM_USEROBJ) {
1588 struct task_struct *task;
1589 /*
1590 * we do the lookup on the thread group leader,
1591 * since individual threads might have already quit!
1592 */
1593 rcu_read_lock();
1594 task = find_task_by_vpid(entry->ent.tgid);
1595 if (task)
1596 mm = get_task_mm(task);
1597 rcu_read_unlock();
1598 }
1599
1600 for (i = 0; i < FTRACE_STACK_ENTRIES; i++) {
1601 unsigned long ip = entry->caller[i];
1602
1603 if (ip == ULONG_MAX || !ret)
1604 break;
1605 if (i && ret)
1606 ret = trace_seq_puts(s, " <- ");
1607 if (!ip) {
1608 if (ret)
1609 ret = trace_seq_puts(s, "??");
1610 continue;
1611 }
1612 if (!ret)
1613 break;
1614 if (ret)
1615 ret = seq_print_user_ip(s, mm, ip, sym_flags);
1616 }
1617
1618 if (mm)
1619 mmput(mm);
1620 return ret;
1621} 1429}
1622 1430
1623static void print_lat_help_header(struct seq_file *m) 1431static void print_lat_help_header(struct seq_file *m)
@@ -1658,11 +1466,11 @@ print_trace_header(struct seq_file *m, struct trace_iterator *iter)
1658 total = entries + 1466 total = entries +
1659 ring_buffer_overruns(iter->tr->buffer); 1467 ring_buffer_overruns(iter->tr->buffer);
1660 1468
1661 seq_printf(m, "%s latency trace v1.1.5 on %s\n", 1469 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
1662 name, UTS_RELEASE); 1470 name, UTS_RELEASE);
1663 seq_puts(m, "-----------------------------------" 1471 seq_puts(m, "# -----------------------------------"
1664 "---------------------------------\n"); 1472 "---------------------------------\n");
1665 seq_printf(m, " latency: %lu us, #%lu/%lu, CPU#%d |" 1473 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
1666 " (M:%s VP:%d, KP:%d, SP:%d HP:%d", 1474 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
1667 nsecs_to_usecs(data->saved_latency), 1475 nsecs_to_usecs(data->saved_latency),
1668 entries, 1476 entries,
@@ -1684,121 +1492,24 @@ print_trace_header(struct seq_file *m, struct trace_iterator *iter)
1684#else 1492#else
1685 seq_puts(m, ")\n"); 1493 seq_puts(m, ")\n");
1686#endif 1494#endif
1687 seq_puts(m, " -----------------\n"); 1495 seq_puts(m, "# -----------------\n");
1688 seq_printf(m, " | task: %.16s-%d " 1496 seq_printf(m, "# | task: %.16s-%d "
1689 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n", 1497 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
1690 data->comm, data->pid, data->uid, data->nice, 1498 data->comm, data->pid, data->uid, data->nice,
1691 data->policy, data->rt_priority); 1499 data->policy, data->rt_priority);
1692 seq_puts(m, " -----------------\n"); 1500 seq_puts(m, "# -----------------\n");
1693 1501
1694 if (data->critical_start) { 1502 if (data->critical_start) {
1695 seq_puts(m, " => started at: "); 1503 seq_puts(m, "# => started at: ");
1696 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags); 1504 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
1697 trace_print_seq(m, &iter->seq); 1505 trace_print_seq(m, &iter->seq);
1698 seq_puts(m, "\n => ended at: "); 1506 seq_puts(m, "\n# => ended at: ");
1699 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags); 1507 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
1700 trace_print_seq(m, &iter->seq); 1508 trace_print_seq(m, &iter->seq);
1701 seq_puts(m, "\n"); 1509 seq_puts(m, "#\n");
1702 }
1703
1704 seq_puts(m, "\n");
1705}
1706
1707static void
1708lat_print_generic(struct trace_seq *s, struct trace_entry *entry, int cpu)
1709{
1710 int hardirq, softirq;
1711 char *comm;
1712
1713 comm = trace_find_cmdline(entry->pid);
1714
1715 trace_seq_printf(s, "%8.8s-%-5d ", comm, entry->pid);
1716 trace_seq_printf(s, "%3d", cpu);
1717 trace_seq_printf(s, "%c%c",
1718 (entry->flags & TRACE_FLAG_IRQS_OFF) ? 'd' :
1719 (entry->flags & TRACE_FLAG_IRQS_NOSUPPORT) ? 'X' : '.',
1720 ((entry->flags & TRACE_FLAG_NEED_RESCHED) ? 'N' : '.'));
1721
1722 hardirq = entry->flags & TRACE_FLAG_HARDIRQ;
1723 softirq = entry->flags & TRACE_FLAG_SOFTIRQ;
1724 if (hardirq && softirq) {
1725 trace_seq_putc(s, 'H');
1726 } else {
1727 if (hardirq) {
1728 trace_seq_putc(s, 'h');
1729 } else {
1730 if (softirq)
1731 trace_seq_putc(s, 's');
1732 else
1733 trace_seq_putc(s, '.');
1734 }
1735 } 1510 }
1736 1511
1737 if (entry->preempt_count) 1512 seq_puts(m, "#\n");
1738 trace_seq_printf(s, "%x", entry->preempt_count);
1739 else
1740 trace_seq_puts(s, ".");
1741}
1742
1743unsigned long preempt_mark_thresh = 100;
1744
1745static void
1746lat_print_timestamp(struct trace_seq *s, u64 abs_usecs,
1747 unsigned long rel_usecs)
1748{
1749 trace_seq_printf(s, " %4lldus", abs_usecs);
1750 if (rel_usecs > preempt_mark_thresh)
1751 trace_seq_puts(s, "!: ");
1752 else if (rel_usecs > 1)
1753 trace_seq_puts(s, "+: ");
1754 else
1755 trace_seq_puts(s, " : ");
1756}
1757
1758static const char state_to_char[] = TASK_STATE_TO_CHAR_STR;
1759
1760static int task_state_char(unsigned long state)
1761{
1762 int bit = state ? __ffs(state) + 1 : 0;
1763
1764 return bit < sizeof(state_to_char) - 1 ? state_to_char[bit] : '?';
1765}
1766
1767/*
1768 * The message is supposed to contain an ending newline.
1769 * If the printing stops prematurely, try to add a newline of our own.
1770 */
1771void trace_seq_print_cont(struct trace_seq *s, struct trace_iterator *iter)
1772{
1773 struct trace_entry *ent;
1774 struct trace_field_cont *cont;
1775 bool ok = true;
1776
1777 ent = peek_next_entry(iter, iter->cpu, NULL);
1778 if (!ent || ent->type != TRACE_CONT) {
1779 trace_seq_putc(s, '\n');
1780 return;
1781 }
1782
1783 do {
1784 cont = (struct trace_field_cont *)ent;
1785 if (ok)
1786 ok = (trace_seq_printf(s, "%s", cont->buf) > 0);
1787
1788 ftrace_disable_cpu();
1789
1790 if (iter->buffer_iter[iter->cpu])
1791 ring_buffer_read(iter->buffer_iter[iter->cpu], NULL);
1792 else
1793 ring_buffer_consume(iter->tr->buffer, iter->cpu, NULL);
1794
1795 ftrace_enable_cpu();
1796
1797 ent = peek_next_entry(iter, iter->cpu, NULL);
1798 } while (ent && ent->type == TRACE_CONT);
1799
1800 if (!ok)
1801 trace_seq_putc(s, '\n');
1802} 1513}
1803 1514
1804static void test_cpu_buff_start(struct trace_iterator *iter) 1515static void test_cpu_buff_start(struct trace_iterator *iter)
@@ -1818,452 +1529,88 @@ static void test_cpu_buff_start(struct trace_iterator *iter)
1818 trace_seq_printf(s, "##### CPU %u buffer started ####\n", iter->cpu); 1529 trace_seq_printf(s, "##### CPU %u buffer started ####\n", iter->cpu);
1819} 1530}
1820 1531
1821static enum print_line_t
1822print_lat_fmt(struct trace_iterator *iter, unsigned int trace_idx, int cpu)
1823{
1824 struct trace_seq *s = &iter->seq;
1825 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1826 struct trace_entry *next_entry;
1827 unsigned long verbose = (trace_flags & TRACE_ITER_VERBOSE);
1828 struct trace_entry *entry = iter->ent;
1829 unsigned long abs_usecs;
1830 unsigned long rel_usecs;
1831 u64 next_ts;
1832 char *comm;
1833 int S, T;
1834 int i;
1835
1836 if (entry->type == TRACE_CONT)
1837 return TRACE_TYPE_HANDLED;
1838
1839 test_cpu_buff_start(iter);
1840
1841 next_entry = find_next_entry(iter, NULL, &next_ts);
1842 if (!next_entry)
1843 next_ts = iter->ts;
1844 rel_usecs = ns2usecs(next_ts - iter->ts);
1845 abs_usecs = ns2usecs(iter->ts - iter->tr->time_start);
1846
1847 if (verbose) {
1848 comm = trace_find_cmdline(entry->pid);
1849 trace_seq_printf(s, "%16s %5d %3d %d %08x %08x [%08lx]"
1850 " %ld.%03ldms (+%ld.%03ldms): ",
1851 comm,
1852 entry->pid, cpu, entry->flags,
1853 entry->preempt_count, trace_idx,
1854 ns2usecs(iter->ts),
1855 abs_usecs/1000,
1856 abs_usecs % 1000, rel_usecs/1000,
1857 rel_usecs % 1000);
1858 } else {
1859 lat_print_generic(s, entry, cpu);
1860 lat_print_timestamp(s, abs_usecs, rel_usecs);
1861 }
1862 switch (entry->type) {
1863 case TRACE_FN: {
1864 struct ftrace_entry *field;
1865
1866 trace_assign_type(field, entry);
1867
1868 seq_print_ip_sym(s, field->ip, sym_flags);
1869 trace_seq_puts(s, " (");
1870 seq_print_ip_sym(s, field->parent_ip, sym_flags);
1871 trace_seq_puts(s, ")\n");
1872 break;
1873 }
1874 case TRACE_CTX:
1875 case TRACE_WAKE: {
1876 struct ctx_switch_entry *field;
1877
1878 trace_assign_type(field, entry);
1879
1880 T = task_state_char(field->next_state);
1881 S = task_state_char(field->prev_state);
1882 comm = trace_find_cmdline(field->next_pid);
1883 trace_seq_printf(s, " %5d:%3d:%c %s [%03d] %5d:%3d:%c %s\n",
1884 field->prev_pid,
1885 field->prev_prio,
1886 S, entry->type == TRACE_CTX ? "==>" : " +",
1887 field->next_cpu,
1888 field->next_pid,
1889 field->next_prio,
1890 T, comm);
1891 break;
1892 }
1893 case TRACE_SPECIAL: {
1894 struct special_entry *field;
1895
1896 trace_assign_type(field, entry);
1897
1898 trace_seq_printf(s, "# %ld %ld %ld\n",
1899 field->arg1,
1900 field->arg2,
1901 field->arg3);
1902 break;
1903 }
1904 case TRACE_STACK: {
1905 struct stack_entry *field;
1906
1907 trace_assign_type(field, entry);
1908
1909 for (i = 0; i < FTRACE_STACK_ENTRIES; i++) {
1910 if (i)
1911 trace_seq_puts(s, " <= ");
1912 seq_print_ip_sym(s, field->caller[i], sym_flags);
1913 }
1914 trace_seq_puts(s, "\n");
1915 break;
1916 }
1917 case TRACE_PRINT: {
1918 struct print_entry *field;
1919
1920 trace_assign_type(field, entry);
1921
1922 seq_print_ip_sym(s, field->ip, sym_flags);
1923 trace_seq_printf(s, ": %s", field->buf);
1924 if (entry->flags & TRACE_FLAG_CONT)
1925 trace_seq_print_cont(s, iter);
1926 break;
1927 }
1928 case TRACE_BRANCH: {
1929 struct trace_branch *field;
1930
1931 trace_assign_type(field, entry);
1932
1933 trace_seq_printf(s, "[%s] %s:%s:%d\n",
1934 field->correct ? " ok " : " MISS ",
1935 field->func,
1936 field->file,
1937 field->line);
1938 break;
1939 }
1940 case TRACE_USER_STACK: {
1941 struct userstack_entry *field;
1942
1943 trace_assign_type(field, entry);
1944
1945 seq_print_userip_objs(field, s, sym_flags);
1946 trace_seq_putc(s, '\n');
1947 break;
1948 }
1949 default:
1950 trace_seq_printf(s, "Unknown type %d\n", entry->type);
1951 }
1952 return TRACE_TYPE_HANDLED;
1953}
1954
1955static enum print_line_t print_trace_fmt(struct trace_iterator *iter) 1532static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
1956{ 1533{
1957 struct trace_seq *s = &iter->seq; 1534 struct trace_seq *s = &iter->seq;
1958 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK); 1535 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1959 struct trace_entry *entry; 1536 struct trace_entry *entry;
1960 unsigned long usec_rem; 1537 struct trace_event *event;
1961 unsigned long long t;
1962 unsigned long secs;
1963 char *comm;
1964 int ret;
1965 int S, T;
1966 int i;
1967 1538
1968 entry = iter->ent; 1539 entry = iter->ent;
1969 1540
1970 if (entry->type == TRACE_CONT)
1971 return TRACE_TYPE_HANDLED;
1972
1973 test_cpu_buff_start(iter); 1541 test_cpu_buff_start(iter);
1974 1542
1975 comm = trace_find_cmdline(iter->ent->pid); 1543 event = ftrace_find_event(entry->type);
1976
1977 t = ns2usecs(iter->ts);
1978 usec_rem = do_div(t, 1000000ULL);
1979 secs = (unsigned long)t;
1980
1981 ret = trace_seq_printf(s, "%16s-%-5d ", comm, entry->pid);
1982 if (!ret)
1983 return TRACE_TYPE_PARTIAL_LINE;
1984 ret = trace_seq_printf(s, "[%03d] ", iter->cpu);
1985 if (!ret)
1986 return TRACE_TYPE_PARTIAL_LINE;
1987 ret = trace_seq_printf(s, "%5lu.%06lu: ", secs, usec_rem);
1988 if (!ret)
1989 return TRACE_TYPE_PARTIAL_LINE;
1990
1991 switch (entry->type) {
1992 case TRACE_FN: {
1993 struct ftrace_entry *field;
1994
1995 trace_assign_type(field, entry);
1996
1997 ret = seq_print_ip_sym(s, field->ip, sym_flags);
1998 if (!ret)
1999 return TRACE_TYPE_PARTIAL_LINE;
2000 if ((sym_flags & TRACE_ITER_PRINT_PARENT) &&
2001 field->parent_ip) {
2002 ret = trace_seq_printf(s, " <-");
2003 if (!ret)
2004 return TRACE_TYPE_PARTIAL_LINE;
2005 ret = seq_print_ip_sym(s,
2006 field->parent_ip,
2007 sym_flags);
2008 if (!ret)
2009 return TRACE_TYPE_PARTIAL_LINE;
2010 }
2011 ret = trace_seq_printf(s, "\n");
2012 if (!ret)
2013 return TRACE_TYPE_PARTIAL_LINE;
2014 break;
2015 }
2016 case TRACE_CTX:
2017 case TRACE_WAKE: {
2018 struct ctx_switch_entry *field;
2019
2020 trace_assign_type(field, entry);
2021
2022 T = task_state_char(field->next_state);
2023 S = task_state_char(field->prev_state);
2024 ret = trace_seq_printf(s, " %5d:%3d:%c %s [%03d] %5d:%3d:%c\n",
2025 field->prev_pid,
2026 field->prev_prio,
2027 S,
2028 entry->type == TRACE_CTX ? "==>" : " +",
2029 field->next_cpu,
2030 field->next_pid,
2031 field->next_prio,
2032 T);
2033 if (!ret)
2034 return TRACE_TYPE_PARTIAL_LINE;
2035 break;
2036 }
2037 case TRACE_SPECIAL: {
2038 struct special_entry *field;
2039
2040 trace_assign_type(field, entry);
2041
2042 ret = trace_seq_printf(s, "# %ld %ld %ld\n",
2043 field->arg1,
2044 field->arg2,
2045 field->arg3);
2046 if (!ret)
2047 return TRACE_TYPE_PARTIAL_LINE;
2048 break;
2049 }
2050 case TRACE_STACK: {
2051 struct stack_entry *field;
2052 1544
2053 trace_assign_type(field, entry); 1545 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2054 1546 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2055 for (i = 0; i < FTRACE_STACK_ENTRIES; i++) { 1547 if (!trace_print_lat_context(iter))
2056 if (i) { 1548 goto partial;
2057 ret = trace_seq_puts(s, " <= "); 1549 } else {
2058 if (!ret) 1550 if (!trace_print_context(iter))
2059 return TRACE_TYPE_PARTIAL_LINE; 1551 goto partial;
2060 }
2061 ret = seq_print_ip_sym(s, field->caller[i],
2062 sym_flags);
2063 if (!ret)
2064 return TRACE_TYPE_PARTIAL_LINE;
2065 } 1552 }
2066 ret = trace_seq_puts(s, "\n");
2067 if (!ret)
2068 return TRACE_TYPE_PARTIAL_LINE;
2069 break;
2070 } 1553 }
2071 case TRACE_PRINT: {
2072 struct print_entry *field;
2073 1554
2074 trace_assign_type(field, entry); 1555 if (event)
1556 return event->trace(iter, sym_flags);
2075 1557
2076 seq_print_ip_sym(s, field->ip, sym_flags); 1558 if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
2077 trace_seq_printf(s, ": %s", field->buf); 1559 goto partial;
2078 if (entry->flags & TRACE_FLAG_CONT)
2079 trace_seq_print_cont(s, iter);
2080 break;
2081 }
2082 case TRACE_GRAPH_RET: {
2083 return print_graph_function(iter);
2084 }
2085 case TRACE_GRAPH_ENT: {
2086 return print_graph_function(iter);
2087 }
2088 case TRACE_BRANCH: {
2089 struct trace_branch *field;
2090
2091 trace_assign_type(field, entry);
2092 1560
2093 trace_seq_printf(s, "[%s] %s:%s:%d\n",
2094 field->correct ? " ok " : " MISS ",
2095 field->func,
2096 field->file,
2097 field->line);
2098 break;
2099 }
2100 case TRACE_USER_STACK: {
2101 struct userstack_entry *field;
2102
2103 trace_assign_type(field, entry);
2104
2105 ret = seq_print_userip_objs(field, s, sym_flags);
2106 if (!ret)
2107 return TRACE_TYPE_PARTIAL_LINE;
2108 ret = trace_seq_putc(s, '\n');
2109 if (!ret)
2110 return TRACE_TYPE_PARTIAL_LINE;
2111 break;
2112 }
2113 }
2114 return TRACE_TYPE_HANDLED; 1561 return TRACE_TYPE_HANDLED;
1562partial:
1563 return TRACE_TYPE_PARTIAL_LINE;
2115} 1564}
2116 1565
2117static enum print_line_t print_raw_fmt(struct trace_iterator *iter) 1566static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
2118{ 1567{
2119 struct trace_seq *s = &iter->seq; 1568 struct trace_seq *s = &iter->seq;
2120 struct trace_entry *entry; 1569 struct trace_entry *entry;
2121 int ret; 1570 struct trace_event *event;
2122 int S, T;
2123 1571
2124 entry = iter->ent; 1572 entry = iter->ent;
2125 1573
2126 if (entry->type == TRACE_CONT) 1574 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2127 return TRACE_TYPE_HANDLED; 1575 if (!trace_seq_printf(s, "%d %d %llu ",
2128 1576 entry->pid, iter->cpu, iter->ts))
2129 ret = trace_seq_printf(s, "%d %d %llu ", 1577 goto partial;
2130 entry->pid, iter->cpu, iter->ts);
2131 if (!ret)
2132 return TRACE_TYPE_PARTIAL_LINE;
2133
2134 switch (entry->type) {
2135 case TRACE_FN: {
2136 struct ftrace_entry *field;
2137
2138 trace_assign_type(field, entry);
2139
2140 ret = trace_seq_printf(s, "%x %x\n",
2141 field->ip,
2142 field->parent_ip);
2143 if (!ret)
2144 return TRACE_TYPE_PARTIAL_LINE;
2145 break;
2146 } 1578 }
2147 case TRACE_CTX:
2148 case TRACE_WAKE: {
2149 struct ctx_switch_entry *field;
2150
2151 trace_assign_type(field, entry);
2152
2153 T = task_state_char(field->next_state);
2154 S = entry->type == TRACE_WAKE ? '+' :
2155 task_state_char(field->prev_state);
2156 ret = trace_seq_printf(s, "%d %d %c %d %d %d %c\n",
2157 field->prev_pid,
2158 field->prev_prio,
2159 S,
2160 field->next_cpu,
2161 field->next_pid,
2162 field->next_prio,
2163 T);
2164 if (!ret)
2165 return TRACE_TYPE_PARTIAL_LINE;
2166 break;
2167 }
2168 case TRACE_SPECIAL:
2169 case TRACE_USER_STACK:
2170 case TRACE_STACK: {
2171 struct special_entry *field;
2172 1579
2173 trace_assign_type(field, entry); 1580 event = ftrace_find_event(entry->type);
1581 if (event)
1582 return event->raw(iter, 0);
2174 1583
2175 ret = trace_seq_printf(s, "# %ld %ld %ld\n", 1584 if (!trace_seq_printf(s, "%d ?\n", entry->type))
2176 field->arg1, 1585 goto partial;
2177 field->arg2,
2178 field->arg3);
2179 if (!ret)
2180 return TRACE_TYPE_PARTIAL_LINE;
2181 break;
2182 }
2183 case TRACE_PRINT: {
2184 struct print_entry *field;
2185 1586
2186 trace_assign_type(field, entry);
2187
2188 trace_seq_printf(s, "# %lx %s", field->ip, field->buf);
2189 if (entry->flags & TRACE_FLAG_CONT)
2190 trace_seq_print_cont(s, iter);
2191 break;
2192 }
2193 }
2194 return TRACE_TYPE_HANDLED; 1587 return TRACE_TYPE_HANDLED;
1588partial:
1589 return TRACE_TYPE_PARTIAL_LINE;
2195} 1590}
2196 1591
2197#define SEQ_PUT_FIELD_RET(s, x) \
2198do { \
2199 if (!trace_seq_putmem(s, &(x), sizeof(x))) \
2200 return 0; \
2201} while (0)
2202
2203#define SEQ_PUT_HEX_FIELD_RET(s, x) \
2204do { \
2205 BUILD_BUG_ON(sizeof(x) > MAX_MEMHEX_BYTES); \
2206 if (!trace_seq_putmem_hex(s, &(x), sizeof(x))) \
2207 return 0; \
2208} while (0)
2209
2210static enum print_line_t print_hex_fmt(struct trace_iterator *iter) 1592static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
2211{ 1593{
2212 struct trace_seq *s = &iter->seq; 1594 struct trace_seq *s = &iter->seq;
2213 unsigned char newline = '\n'; 1595 unsigned char newline = '\n';
2214 struct trace_entry *entry; 1596 struct trace_entry *entry;
2215 int S, T; 1597 struct trace_event *event;
2216 1598
2217 entry = iter->ent; 1599 entry = iter->ent;
2218 1600
2219 if (entry->type == TRACE_CONT) 1601 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2220 return TRACE_TYPE_HANDLED; 1602 SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
2221 1603 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
2222 SEQ_PUT_HEX_FIELD_RET(s, entry->pid); 1604 SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
2223 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
2224 SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
2225
2226 switch (entry->type) {
2227 case TRACE_FN: {
2228 struct ftrace_entry *field;
2229
2230 trace_assign_type(field, entry);
2231
2232 SEQ_PUT_HEX_FIELD_RET(s, field->ip);
2233 SEQ_PUT_HEX_FIELD_RET(s, field->parent_ip);
2234 break;
2235 }
2236 case TRACE_CTX:
2237 case TRACE_WAKE: {
2238 struct ctx_switch_entry *field;
2239
2240 trace_assign_type(field, entry);
2241
2242 T = task_state_char(field->next_state);
2243 S = entry->type == TRACE_WAKE ? '+' :
2244 task_state_char(field->prev_state);
2245 SEQ_PUT_HEX_FIELD_RET(s, field->prev_pid);
2246 SEQ_PUT_HEX_FIELD_RET(s, field->prev_prio);
2247 SEQ_PUT_HEX_FIELD_RET(s, S);
2248 SEQ_PUT_HEX_FIELD_RET(s, field->next_cpu);
2249 SEQ_PUT_HEX_FIELD_RET(s, field->next_pid);
2250 SEQ_PUT_HEX_FIELD_RET(s, field->next_prio);
2251 SEQ_PUT_HEX_FIELD_RET(s, T);
2252 break;
2253 } 1605 }
2254 case TRACE_SPECIAL:
2255 case TRACE_USER_STACK:
2256 case TRACE_STACK: {
2257 struct special_entry *field;
2258 1606
2259 trace_assign_type(field, entry); 1607 event = ftrace_find_event(entry->type);
2260 1608 if (event) {
2261 SEQ_PUT_HEX_FIELD_RET(s, field->arg1); 1609 enum print_line_t ret = event->hex(iter, 0);
2262 SEQ_PUT_HEX_FIELD_RET(s, field->arg2); 1610 if (ret != TRACE_TYPE_HANDLED)
2263 SEQ_PUT_HEX_FIELD_RET(s, field->arg3); 1611 return ret;
2264 break;
2265 }
2266 } 1612 }
1613
2267 SEQ_PUT_FIELD_RET(s, newline); 1614 SEQ_PUT_FIELD_RET(s, newline);
2268 1615
2269 return TRACE_TYPE_HANDLED; 1616 return TRACE_TYPE_HANDLED;
@@ -2278,13 +1625,10 @@ static enum print_line_t print_printk_msg_only(struct trace_iterator *iter)
2278 1625
2279 trace_assign_type(field, entry); 1626 trace_assign_type(field, entry);
2280 1627
2281 ret = trace_seq_printf(s, field->buf); 1628 ret = trace_seq_bprintf(s, field->fmt, field->buf);
2282 if (!ret) 1629 if (!ret)
2283 return TRACE_TYPE_PARTIAL_LINE; 1630 return TRACE_TYPE_PARTIAL_LINE;
2284 1631
2285 if (entry->flags & TRACE_FLAG_CONT)
2286 trace_seq_print_cont(s, iter);
2287
2288 return TRACE_TYPE_HANDLED; 1632 return TRACE_TYPE_HANDLED;
2289} 1633}
2290 1634
@@ -2292,53 +1636,18 @@ static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
2292{ 1636{
2293 struct trace_seq *s = &iter->seq; 1637 struct trace_seq *s = &iter->seq;
2294 struct trace_entry *entry; 1638 struct trace_entry *entry;
1639 struct trace_event *event;
2295 1640
2296 entry = iter->ent; 1641 entry = iter->ent;
2297 1642
2298 if (entry->type == TRACE_CONT) 1643 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2299 return TRACE_TYPE_HANDLED; 1644 SEQ_PUT_FIELD_RET(s, entry->pid);
2300 1645 SEQ_PUT_FIELD_RET(s, iter->cpu);
2301 SEQ_PUT_FIELD_RET(s, entry->pid); 1646 SEQ_PUT_FIELD_RET(s, iter->ts);
2302 SEQ_PUT_FIELD_RET(s, entry->cpu);
2303 SEQ_PUT_FIELD_RET(s, iter->ts);
2304
2305 switch (entry->type) {
2306 case TRACE_FN: {
2307 struct ftrace_entry *field;
2308
2309 trace_assign_type(field, entry);
2310
2311 SEQ_PUT_FIELD_RET(s, field->ip);
2312 SEQ_PUT_FIELD_RET(s, field->parent_ip);
2313 break;
2314 } 1647 }
2315 case TRACE_CTX: {
2316 struct ctx_switch_entry *field;
2317
2318 trace_assign_type(field, entry);
2319 1648
2320 SEQ_PUT_FIELD_RET(s, field->prev_pid); 1649 event = ftrace_find_event(entry->type);
2321 SEQ_PUT_FIELD_RET(s, field->prev_prio); 1650 return event ? event->binary(iter, 0) : TRACE_TYPE_HANDLED;
2322 SEQ_PUT_FIELD_RET(s, field->prev_state);
2323 SEQ_PUT_FIELD_RET(s, field->next_pid);
2324 SEQ_PUT_FIELD_RET(s, field->next_prio);
2325 SEQ_PUT_FIELD_RET(s, field->next_state);
2326 break;
2327 }
2328 case TRACE_SPECIAL:
2329 case TRACE_USER_STACK:
2330 case TRACE_STACK: {
2331 struct special_entry *field;
2332
2333 trace_assign_type(field, entry);
2334
2335 SEQ_PUT_FIELD_RET(s, field->arg1);
2336 SEQ_PUT_FIELD_RET(s, field->arg2);
2337 SEQ_PUT_FIELD_RET(s, field->arg3);
2338 break;
2339 }
2340 }
2341 return 1;
2342} 1651}
2343 1652
2344static int trace_empty(struct trace_iterator *iter) 1653static int trace_empty(struct trace_iterator *iter)
@@ -2382,9 +1691,6 @@ static enum print_line_t print_trace_line(struct trace_iterator *iter)
2382 if (trace_flags & TRACE_ITER_RAW) 1691 if (trace_flags & TRACE_ITER_RAW)
2383 return print_raw_fmt(iter); 1692 return print_raw_fmt(iter);
2384 1693
2385 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2386 return print_lat_fmt(iter, iter->idx, iter->cpu);
2387
2388 return print_trace_fmt(iter); 1694 return print_trace_fmt(iter);
2389} 1695}
2390 1696
@@ -2426,30 +1732,40 @@ static struct seq_operations tracer_seq_ops = {
2426}; 1732};
2427 1733
2428static struct trace_iterator * 1734static struct trace_iterator *
2429__tracing_open(struct inode *inode, struct file *file, int *ret) 1735__tracing_open(struct inode *inode, struct file *file)
2430{ 1736{
1737 long cpu_file = (long) inode->i_private;
1738 void *fail_ret = ERR_PTR(-ENOMEM);
2431 struct trace_iterator *iter; 1739 struct trace_iterator *iter;
2432 struct seq_file *m; 1740 struct seq_file *m;
2433 int cpu; 1741 int cpu, ret;
2434 1742
2435 if (tracing_disabled) { 1743 if (tracing_disabled)
2436 *ret = -ENODEV; 1744 return ERR_PTR(-ENODEV);
2437 return NULL;
2438 }
2439 1745
2440 iter = kzalloc(sizeof(*iter), GFP_KERNEL); 1746 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
2441 if (!iter) { 1747 if (!iter)
2442 *ret = -ENOMEM; 1748 return ERR_PTR(-ENOMEM);
2443 goto out;
2444 }
2445 1749
1750 /*
1751 * We make a copy of the current tracer to avoid concurrent
1752 * changes on it while we are reading.
1753 */
2446 mutex_lock(&trace_types_lock); 1754 mutex_lock(&trace_types_lock);
1755 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
1756 if (!iter->trace)
1757 goto fail;
1758
1759 if (current_trace)
1760 *iter->trace = *current_trace;
1761
2447 if (current_trace && current_trace->print_max) 1762 if (current_trace && current_trace->print_max)
2448 iter->tr = &max_tr; 1763 iter->tr = &max_tr;
2449 else 1764 else
2450 iter->tr = inode->i_private; 1765 iter->tr = &global_trace;
2451 iter->trace = current_trace;
2452 iter->pos = -1; 1766 iter->pos = -1;
1767 mutex_init(&iter->mutex);
1768 iter->cpu_file = cpu_file;
2453 1769
2454 /* Notify the tracer early; before we stop tracing. */ 1770 /* Notify the tracer early; before we stop tracing. */
2455 if (iter->trace && iter->trace->open) 1771 if (iter->trace && iter->trace->open)
@@ -2459,20 +1775,30 @@ __tracing_open(struct inode *inode, struct file *file, int *ret)
2459 if (ring_buffer_overruns(iter->tr->buffer)) 1775 if (ring_buffer_overruns(iter->tr->buffer))
2460 iter->iter_flags |= TRACE_FILE_ANNOTATE; 1776 iter->iter_flags |= TRACE_FILE_ANNOTATE;
2461 1777
1778 if (iter->cpu_file == TRACE_PIPE_ALL_CPU) {
1779 for_each_tracing_cpu(cpu) {
2462 1780
2463 for_each_tracing_cpu(cpu) { 1781 iter->buffer_iter[cpu] =
1782 ring_buffer_read_start(iter->tr->buffer, cpu);
2464 1783
1784 if (!iter->buffer_iter[cpu])
1785 goto fail_buffer;
1786 }
1787 } else {
1788 cpu = iter->cpu_file;
2465 iter->buffer_iter[cpu] = 1789 iter->buffer_iter[cpu] =
2466 ring_buffer_read_start(iter->tr->buffer, cpu); 1790 ring_buffer_read_start(iter->tr->buffer, cpu);
2467 1791
2468 if (!iter->buffer_iter[cpu]) 1792 if (!iter->buffer_iter[cpu])
2469 goto fail_buffer; 1793 goto fail;
2470 } 1794 }
2471 1795
2472 /* TODO stop tracer */ 1796 /* TODO stop tracer */
2473 *ret = seq_open(file, &tracer_seq_ops); 1797 ret = seq_open(file, &tracer_seq_ops);
2474 if (*ret) 1798 if (ret < 0) {
1799 fail_ret = ERR_PTR(ret);
2475 goto fail_buffer; 1800 goto fail_buffer;
1801 }
2476 1802
2477 m = file->private_data; 1803 m = file->private_data;
2478 m->private = iter; 1804 m->private = iter;
@@ -2482,7 +1808,6 @@ __tracing_open(struct inode *inode, struct file *file, int *ret)
2482 1808
2483 mutex_unlock(&trace_types_lock); 1809 mutex_unlock(&trace_types_lock);
2484 1810
2485 out:
2486 return iter; 1811 return iter;
2487 1812
2488 fail_buffer: 1813 fail_buffer:
@@ -2490,10 +1815,12 @@ __tracing_open(struct inode *inode, struct file *file, int *ret)
2490 if (iter->buffer_iter[cpu]) 1815 if (iter->buffer_iter[cpu])
2491 ring_buffer_read_finish(iter->buffer_iter[cpu]); 1816 ring_buffer_read_finish(iter->buffer_iter[cpu]);
2492 } 1817 }
1818 fail:
2493 mutex_unlock(&trace_types_lock); 1819 mutex_unlock(&trace_types_lock);
1820 kfree(iter->trace);
2494 kfree(iter); 1821 kfree(iter);
2495 1822
2496 return ERR_PTR(-ENOMEM); 1823 return fail_ret;
2497} 1824}
2498 1825
2499int tracing_open_generic(struct inode *inode, struct file *filp) 1826int tracing_open_generic(struct inode *inode, struct file *filp)
@@ -2505,7 +1832,7 @@ int tracing_open_generic(struct inode *inode, struct file *filp)
2505 return 0; 1832 return 0;
2506} 1833}
2507 1834
2508int tracing_release(struct inode *inode, struct file *file) 1835static int tracing_release(struct inode *inode, struct file *file)
2509{ 1836{
2510 struct seq_file *m = (struct seq_file *)file->private_data; 1837 struct seq_file *m = (struct seq_file *)file->private_data;
2511 struct trace_iterator *iter = m->private; 1838 struct trace_iterator *iter = m->private;
@@ -2525,33 +1852,26 @@ int tracing_release(struct inode *inode, struct file *file)
2525 mutex_unlock(&trace_types_lock); 1852 mutex_unlock(&trace_types_lock);
2526 1853
2527 seq_release(inode, file); 1854 seq_release(inode, file);
1855 mutex_destroy(&iter->mutex);
1856 kfree(iter->trace);
2528 kfree(iter); 1857 kfree(iter);
2529 return 0; 1858 return 0;
2530} 1859}
2531 1860
2532static int tracing_open(struct inode *inode, struct file *file) 1861static int tracing_open(struct inode *inode, struct file *file)
2533{ 1862{
2534 int ret;
2535
2536 __tracing_open(inode, file, &ret);
2537
2538 return ret;
2539}
2540
2541static int tracing_lt_open(struct inode *inode, struct file *file)
2542{
2543 struct trace_iterator *iter; 1863 struct trace_iterator *iter;
2544 int ret; 1864 int ret = 0;
2545
2546 iter = __tracing_open(inode, file, &ret);
2547 1865
2548 if (!ret) 1866 iter = __tracing_open(inode, file);
1867 if (IS_ERR(iter))
1868 ret = PTR_ERR(iter);
1869 else if (trace_flags & TRACE_ITER_LATENCY_FMT)
2549 iter->iter_flags |= TRACE_FILE_LAT_FMT; 1870 iter->iter_flags |= TRACE_FILE_LAT_FMT;
2550 1871
2551 return ret; 1872 return ret;
2552} 1873}
2553 1874
2554
2555static void * 1875static void *
2556t_next(struct seq_file *m, void *v, loff_t *pos) 1876t_next(struct seq_file *m, void *v, loff_t *pos)
2557{ 1877{
@@ -2623,21 +1943,14 @@ static int show_traces_open(struct inode *inode, struct file *file)
2623 return ret; 1943 return ret;
2624} 1944}
2625 1945
2626static struct file_operations tracing_fops = { 1946static const struct file_operations tracing_fops = {
2627 .open = tracing_open, 1947 .open = tracing_open,
2628 .read = seq_read, 1948 .read = seq_read,
2629 .llseek = seq_lseek, 1949 .llseek = seq_lseek,
2630 .release = tracing_release, 1950 .release = tracing_release,
2631}; 1951};
2632 1952
2633static struct file_operations tracing_lt_fops = { 1953static const struct file_operations show_traces_fops = {
2634 .open = tracing_lt_open,
2635 .read = seq_read,
2636 .llseek = seq_lseek,
2637 .release = tracing_release,
2638};
2639
2640static struct file_operations show_traces_fops = {
2641 .open = show_traces_open, 1954 .open = show_traces_open,
2642 .read = seq_read, 1955 .read = seq_read,
2643 .release = seq_release, 1956 .release = seq_release,
@@ -2730,7 +2043,7 @@ err_unlock:
2730 return err; 2043 return err;
2731} 2044}
2732 2045
2733static struct file_operations tracing_cpumask_fops = { 2046static const struct file_operations tracing_cpumask_fops = {
2734 .open = tracing_open_generic, 2047 .open = tracing_open_generic,
2735 .read = tracing_cpumask_read, 2048 .read = tracing_cpumask_read,
2736 .write = tracing_cpumask_write, 2049 .write = tracing_cpumask_write,
@@ -2740,57 +2053,62 @@ static ssize_t
2740tracing_trace_options_read(struct file *filp, char __user *ubuf, 2053tracing_trace_options_read(struct file *filp, char __user *ubuf,
2741 size_t cnt, loff_t *ppos) 2054 size_t cnt, loff_t *ppos)
2742{ 2055{
2743 int i; 2056 struct tracer_opt *trace_opts;
2057 u32 tracer_flags;
2058 int len = 0;
2744 char *buf; 2059 char *buf;
2745 int r = 0; 2060 int r = 0;
2746 int len = 0; 2061 int i;
2747 u32 tracer_flags = current_trace->flags->val;
2748 struct tracer_opt *trace_opts = current_trace->flags->opts;
2749 2062
2750 2063
2751 /* calulate max size */ 2064 /* calculate max size */
2752 for (i = 0; trace_options[i]; i++) { 2065 for (i = 0; trace_options[i]; i++) {
2753 len += strlen(trace_options[i]); 2066 len += strlen(trace_options[i]);
2754 len += 3; /* "no" and space */ 2067 len += 3; /* "no" and newline */
2755 } 2068 }
2756 2069
2070 mutex_lock(&trace_types_lock);
2071 tracer_flags = current_trace->flags->val;
2072 trace_opts = current_trace->flags->opts;
2073
2757 /* 2074 /*
2758 * Increase the size with names of options specific 2075 * Increase the size with names of options specific
2759 * of the current tracer. 2076 * of the current tracer.
2760 */ 2077 */
2761 for (i = 0; trace_opts[i].name; i++) { 2078 for (i = 0; trace_opts[i].name; i++) {
2762 len += strlen(trace_opts[i].name); 2079 len += strlen(trace_opts[i].name);
2763 len += 3; /* "no" and space */ 2080 len += 3; /* "no" and newline */
2764 } 2081 }
2765 2082
2766 /* +2 for \n and \0 */ 2083 /* +2 for \n and \0 */
2767 buf = kmalloc(len + 2, GFP_KERNEL); 2084 buf = kmalloc(len + 2, GFP_KERNEL);
2768 if (!buf) 2085 if (!buf) {
2086 mutex_unlock(&trace_types_lock);
2769 return -ENOMEM; 2087 return -ENOMEM;
2088 }
2770 2089
2771 for (i = 0; trace_options[i]; i++) { 2090 for (i = 0; trace_options[i]; i++) {
2772 if (trace_flags & (1 << i)) 2091 if (trace_flags & (1 << i))
2773 r += sprintf(buf + r, "%s ", trace_options[i]); 2092 r += sprintf(buf + r, "%s\n", trace_options[i]);
2774 else 2093 else
2775 r += sprintf(buf + r, "no%s ", trace_options[i]); 2094 r += sprintf(buf + r, "no%s\n", trace_options[i]);
2776 } 2095 }
2777 2096
2778 for (i = 0; trace_opts[i].name; i++) { 2097 for (i = 0; trace_opts[i].name; i++) {
2779 if (tracer_flags & trace_opts[i].bit) 2098 if (tracer_flags & trace_opts[i].bit)
2780 r += sprintf(buf + r, "%s ", 2099 r += sprintf(buf + r, "%s\n",
2781 trace_opts[i].name); 2100 trace_opts[i].name);
2782 else 2101 else
2783 r += sprintf(buf + r, "no%s ", 2102 r += sprintf(buf + r, "no%s\n",
2784 trace_opts[i].name); 2103 trace_opts[i].name);
2785 } 2104 }
2105 mutex_unlock(&trace_types_lock);
2786 2106
2787 r += sprintf(buf + r, "\n");
2788 WARN_ON(r >= len + 2); 2107 WARN_ON(r >= len + 2);
2789 2108
2790 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 2109 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2791 2110
2792 kfree(buf); 2111 kfree(buf);
2793
2794 return r; 2112 return r;
2795} 2113}
2796 2114
@@ -2865,7 +2183,9 @@ tracing_trace_options_write(struct file *filp, const char __user *ubuf,
2865 2183
2866 /* If no option could be set, test the specific tracer options */ 2184 /* If no option could be set, test the specific tracer options */
2867 if (!trace_options[i]) { 2185 if (!trace_options[i]) {
2186 mutex_lock(&trace_types_lock);
2868 ret = set_tracer_option(current_trace, cmp, neg); 2187 ret = set_tracer_option(current_trace, cmp, neg);
2188 mutex_unlock(&trace_types_lock);
2869 if (ret) 2189 if (ret)
2870 return ret; 2190 return ret;
2871 } 2191 }
@@ -2875,7 +2195,7 @@ tracing_trace_options_write(struct file *filp, const char __user *ubuf,
2875 return cnt; 2195 return cnt;
2876} 2196}
2877 2197
2878static struct file_operations tracing_iter_fops = { 2198static const struct file_operations tracing_iter_fops = {
2879 .open = tracing_open_generic, 2199 .open = tracing_open_generic,
2880 .read = tracing_trace_options_read, 2200 .read = tracing_trace_options_read,
2881 .write = tracing_trace_options_write, 2201 .write = tracing_trace_options_write,
@@ -2908,7 +2228,7 @@ tracing_readme_read(struct file *filp, char __user *ubuf,
2908 readme_msg, strlen(readme_msg)); 2228 readme_msg, strlen(readme_msg));
2909} 2229}
2910 2230
2911static struct file_operations tracing_readme_fops = { 2231static const struct file_operations tracing_readme_fops = {
2912 .open = tracing_open_generic, 2232 .open = tracing_open_generic,
2913 .read = tracing_readme_read, 2233 .read = tracing_readme_read,
2914}; 2234};
@@ -2930,7 +2250,7 @@ tracing_ctrl_write(struct file *filp, const char __user *ubuf,
2930{ 2250{
2931 struct trace_array *tr = filp->private_data; 2251 struct trace_array *tr = filp->private_data;
2932 char buf[64]; 2252 char buf[64];
2933 long val; 2253 unsigned long val;
2934 int ret; 2254 int ret;
2935 2255
2936 if (cnt >= sizeof(buf)) 2256 if (cnt >= sizeof(buf))
@@ -2985,8 +2305,23 @@ tracing_set_trace_read(struct file *filp, char __user *ubuf,
2985 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 2305 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2986} 2306}
2987 2307
2988static int tracing_set_tracer(char *buf) 2308int tracer_init(struct tracer *t, struct trace_array *tr)
2309{
2310 tracing_reset_online_cpus(tr);
2311 return t->init(tr);
2312}
2313
2314struct trace_option_dentry;
2315
2316static struct trace_option_dentry *
2317create_trace_option_files(struct tracer *tracer);
2318
2319static void
2320destroy_trace_option_files(struct trace_option_dentry *topts);
2321
2322static int tracing_set_tracer(const char *buf)
2989{ 2323{
2324 static struct trace_option_dentry *topts;
2990 struct trace_array *tr = &global_trace; 2325 struct trace_array *tr = &global_trace;
2991 struct tracer *t; 2326 struct tracer *t;
2992 int ret = 0; 2327 int ret = 0;
@@ -3007,9 +2342,14 @@ static int tracing_set_tracer(char *buf)
3007 if (current_trace && current_trace->reset) 2342 if (current_trace && current_trace->reset)
3008 current_trace->reset(tr); 2343 current_trace->reset(tr);
3009 2344
2345 destroy_trace_option_files(topts);
2346
3010 current_trace = t; 2347 current_trace = t;
2348
2349 topts = create_trace_option_files(current_trace);
2350
3011 if (t->init) { 2351 if (t->init) {
3012 ret = t->init(tr); 2352 ret = tracer_init(t, tr);
3013 if (ret) 2353 if (ret)
3014 goto out; 2354 goto out;
3015 } 2355 }
@@ -3072,9 +2412,9 @@ static ssize_t
3072tracing_max_lat_write(struct file *filp, const char __user *ubuf, 2412tracing_max_lat_write(struct file *filp, const char __user *ubuf,
3073 size_t cnt, loff_t *ppos) 2413 size_t cnt, loff_t *ppos)
3074{ 2414{
3075 long *ptr = filp->private_data; 2415 unsigned long *ptr = filp->private_data;
3076 char buf[64]; 2416 char buf[64];
3077 long val; 2417 unsigned long val;
3078 int ret; 2418 int ret;
3079 2419
3080 if (cnt >= sizeof(buf)) 2420 if (cnt >= sizeof(buf))
@@ -3094,54 +2434,96 @@ tracing_max_lat_write(struct file *filp, const char __user *ubuf,
3094 return cnt; 2434 return cnt;
3095} 2435}
3096 2436
3097static atomic_t tracing_reader;
3098
3099static int tracing_open_pipe(struct inode *inode, struct file *filp) 2437static int tracing_open_pipe(struct inode *inode, struct file *filp)
3100{ 2438{
2439 long cpu_file = (long) inode->i_private;
3101 struct trace_iterator *iter; 2440 struct trace_iterator *iter;
2441 int ret = 0;
3102 2442
3103 if (tracing_disabled) 2443 if (tracing_disabled)
3104 return -ENODEV; 2444 return -ENODEV;
3105 2445
3106 /* We only allow for reader of the pipe */ 2446 mutex_lock(&trace_types_lock);
3107 if (atomic_inc_return(&tracing_reader) != 1) { 2447
3108 atomic_dec(&tracing_reader); 2448 /* We only allow one reader per cpu */
3109 return -EBUSY; 2449 if (cpu_file == TRACE_PIPE_ALL_CPU) {
2450 if (!cpumask_empty(tracing_reader_cpumask)) {
2451 ret = -EBUSY;
2452 goto out;
2453 }
2454 cpumask_setall(tracing_reader_cpumask);
2455 } else {
2456 if (!cpumask_test_cpu(cpu_file, tracing_reader_cpumask))
2457 cpumask_set_cpu(cpu_file, tracing_reader_cpumask);
2458 else {
2459 ret = -EBUSY;
2460 goto out;
2461 }
3110 } 2462 }
3111 2463
3112 /* create a buffer to store the information to pass to userspace */ 2464 /* create a buffer to store the information to pass to userspace */
3113 iter = kzalloc(sizeof(*iter), GFP_KERNEL); 2465 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
3114 if (!iter) 2466 if (!iter) {
3115 return -ENOMEM; 2467 ret = -ENOMEM;
2468 goto out;
2469 }
3116 2470
3117 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) { 2471 /*
3118 kfree(iter); 2472 * We make a copy of the current tracer to avoid concurrent
3119 return -ENOMEM; 2473 * changes on it while we are reading.
2474 */
2475 iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
2476 if (!iter->trace) {
2477 ret = -ENOMEM;
2478 goto fail;
3120 } 2479 }
2480 if (current_trace)
2481 *iter->trace = *current_trace;
3121 2482
3122 mutex_lock(&trace_types_lock); 2483 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
2484 ret = -ENOMEM;
2485 goto fail;
2486 }
3123 2487
3124 /* trace pipe does not show start of buffer */ 2488 /* trace pipe does not show start of buffer */
3125 cpumask_setall(iter->started); 2489 cpumask_setall(iter->started);
3126 2490
2491 iter->cpu_file = cpu_file;
3127 iter->tr = &global_trace; 2492 iter->tr = &global_trace;
3128 iter->trace = current_trace; 2493 mutex_init(&iter->mutex);
3129 filp->private_data = iter; 2494 filp->private_data = iter;
3130 2495
3131 if (iter->trace->pipe_open) 2496 if (iter->trace->pipe_open)
3132 iter->trace->pipe_open(iter); 2497 iter->trace->pipe_open(iter);
2498
2499out:
3133 mutex_unlock(&trace_types_lock); 2500 mutex_unlock(&trace_types_lock);
2501 return ret;
3134 2502
3135 return 0; 2503fail:
2504 kfree(iter->trace);
2505 kfree(iter);
2506 mutex_unlock(&trace_types_lock);
2507 return ret;
3136} 2508}
3137 2509
3138static int tracing_release_pipe(struct inode *inode, struct file *file) 2510static int tracing_release_pipe(struct inode *inode, struct file *file)
3139{ 2511{
3140 struct trace_iterator *iter = file->private_data; 2512 struct trace_iterator *iter = file->private_data;
3141 2513
2514 mutex_lock(&trace_types_lock);
2515
2516 if (iter->cpu_file == TRACE_PIPE_ALL_CPU)
2517 cpumask_clear(tracing_reader_cpumask);
2518 else
2519 cpumask_clear_cpu(iter->cpu_file, tracing_reader_cpumask);
2520
2521 mutex_unlock(&trace_types_lock);
2522
3142 free_cpumask_var(iter->started); 2523 free_cpumask_var(iter->started);
2524 mutex_destroy(&iter->mutex);
2525 kfree(iter->trace);
3143 kfree(iter); 2526 kfree(iter);
3144 atomic_dec(&tracing_reader);
3145 2527
3146 return 0; 2528 return 0;
3147} 2529}
@@ -3167,67 +2549,57 @@ tracing_poll_pipe(struct file *filp, poll_table *poll_table)
3167 } 2549 }
3168} 2550}
3169 2551
3170/* 2552
3171 * Consumer reader. 2553void default_wait_pipe(struct trace_iterator *iter)
3172 */
3173static ssize_t
3174tracing_read_pipe(struct file *filp, char __user *ubuf,
3175 size_t cnt, loff_t *ppos)
3176{ 2554{
3177 struct trace_iterator *iter = filp->private_data; 2555 DEFINE_WAIT(wait);
3178 ssize_t sret;
3179 2556
3180 /* return any leftover data */ 2557 prepare_to_wait(&trace_wait, &wait, TASK_INTERRUPTIBLE);
3181 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3182 if (sret != -EBUSY)
3183 return sret;
3184 2558
3185 trace_seq_reset(&iter->seq); 2559 if (trace_empty(iter))
2560 schedule();
3186 2561
3187 mutex_lock(&trace_types_lock); 2562 finish_wait(&trace_wait, &wait);
3188 if (iter->trace->read) { 2563}
3189 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos); 2564
3190 if (sret) 2565/*
3191 goto out; 2566 * This is a make-shift waitqueue.
3192 } 2567 * A tracer might use this callback on some rare cases:
2568 *
2569 * 1) the current tracer might hold the runqueue lock when it wakes up
2570 * a reader, hence a deadlock (sched, function, and function graph tracers)
2571 * 2) the function tracers, trace all functions, we don't want
2572 * the overhead of calling wake_up and friends
2573 * (and tracing them too)
2574 *
2575 * Anyway, this is really very primitive wakeup.
2576 */
2577void poll_wait_pipe(struct trace_iterator *iter)
2578{
2579 set_current_state(TASK_INTERRUPTIBLE);
2580 /* sleep for 100 msecs, and try again. */
2581 schedule_timeout(HZ / 10);
2582}
2583
2584/* Must be called with trace_types_lock mutex held. */
2585static int tracing_wait_pipe(struct file *filp)
2586{
2587 struct trace_iterator *iter = filp->private_data;
3193 2588
3194waitagain:
3195 sret = 0;
3196 while (trace_empty(iter)) { 2589 while (trace_empty(iter)) {
3197 2590
3198 if ((filp->f_flags & O_NONBLOCK)) { 2591 if ((filp->f_flags & O_NONBLOCK)) {
3199 sret = -EAGAIN; 2592 return -EAGAIN;
3200 goto out;
3201 } 2593 }
3202 2594
3203 /* 2595 mutex_unlock(&iter->mutex);
3204 * This is a make-shift waitqueue. The reason we don't use
3205 * an actual wait queue is because:
3206 * 1) we only ever have one waiter
3207 * 2) the tracing, traces all functions, we don't want
3208 * the overhead of calling wake_up and friends
3209 * (and tracing them too)
3210 * Anyway, this is really very primitive wakeup.
3211 */
3212 set_current_state(TASK_INTERRUPTIBLE);
3213 iter->tr->waiter = current;
3214
3215 mutex_unlock(&trace_types_lock);
3216 2596
3217 /* sleep for 100 msecs, and try again. */ 2597 iter->trace->wait_pipe(iter);
3218 schedule_timeout(HZ/10);
3219 2598
3220 mutex_lock(&trace_types_lock); 2599 mutex_lock(&iter->mutex);
3221 2600
3222 iter->tr->waiter = NULL; 2601 if (signal_pending(current))
3223 2602 return -EINTR;
3224 if (signal_pending(current)) {
3225 sret = -EINTR;
3226 goto out;
3227 }
3228
3229 if (iter->trace != current_trace)
3230 goto out;
3231 2603
3232 /* 2604 /*
3233 * We block until we read something and tracing is disabled. 2605 * We block until we read something and tracing is disabled.
@@ -3240,13 +2612,59 @@ waitagain:
3240 */ 2612 */
3241 if (!tracer_enabled && iter->pos) 2613 if (!tracer_enabled && iter->pos)
3242 break; 2614 break;
2615 }
2616
2617 return 1;
2618}
2619
2620/*
2621 * Consumer reader.
2622 */
2623static ssize_t
2624tracing_read_pipe(struct file *filp, char __user *ubuf,
2625 size_t cnt, loff_t *ppos)
2626{
2627 struct trace_iterator *iter = filp->private_data;
2628 static struct tracer *old_tracer;
2629 ssize_t sret;
2630
2631 /* return any leftover data */
2632 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
2633 if (sret != -EBUSY)
2634 return sret;
2635
2636 trace_seq_init(&iter->seq);
3243 2637
3244 continue; 2638 /* copy the tracer to avoid using a global lock all around */
2639 mutex_lock(&trace_types_lock);
2640 if (unlikely(old_tracer != current_trace && current_trace)) {
2641 old_tracer = current_trace;
2642 *iter->trace = *current_trace;
3245 } 2643 }
2644 mutex_unlock(&trace_types_lock);
2645
2646 /*
2647 * Avoid more than one consumer on a single file descriptor
2648 * This is just a matter of traces coherency, the ring buffer itself
2649 * is protected.
2650 */
2651 mutex_lock(&iter->mutex);
2652 if (iter->trace->read) {
2653 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
2654 if (sret)
2655 goto out;
2656 }
2657
2658waitagain:
2659 sret = tracing_wait_pipe(filp);
2660 if (sret <= 0)
2661 goto out;
3246 2662
3247 /* stop when tracing is finished */ 2663 /* stop when tracing is finished */
3248 if (trace_empty(iter)) 2664 if (trace_empty(iter)) {
2665 sret = 0;
3249 goto out; 2666 goto out;
2667 }
3250 2668
3251 if (cnt >= PAGE_SIZE) 2669 if (cnt >= PAGE_SIZE)
3252 cnt = PAGE_SIZE - 1; 2670 cnt = PAGE_SIZE - 1;
@@ -3267,8 +2685,8 @@ waitagain:
3267 iter->seq.len = len; 2685 iter->seq.len = len;
3268 break; 2686 break;
3269 } 2687 }
3270 2688 if (ret != TRACE_TYPE_NO_CONSUME)
3271 trace_consume(iter); 2689 trace_consume(iter);
3272 2690
3273 if (iter->seq.len >= cnt) 2691 if (iter->seq.len >= cnt)
3274 break; 2692 break;
@@ -3277,7 +2695,7 @@ waitagain:
3277 /* Now copy what we have to the user */ 2695 /* Now copy what we have to the user */
3278 sret = trace_seq_to_user(&iter->seq, ubuf, cnt); 2696 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3279 if (iter->seq.readpos >= iter->seq.len) 2697 if (iter->seq.readpos >= iter->seq.len)
3280 trace_seq_reset(&iter->seq); 2698 trace_seq_init(&iter->seq);
3281 2699
3282 /* 2700 /*
3283 * If there was nothing to send to user, inspite of consuming trace 2701 * If there was nothing to send to user, inspite of consuming trace
@@ -3287,11 +2705,148 @@ waitagain:
3287 goto waitagain; 2705 goto waitagain;
3288 2706
3289out: 2707out:
3290 mutex_unlock(&trace_types_lock); 2708 mutex_unlock(&iter->mutex);
3291 2709
3292 return sret; 2710 return sret;
3293} 2711}
3294 2712
2713static void tracing_pipe_buf_release(struct pipe_inode_info *pipe,
2714 struct pipe_buffer *buf)
2715{
2716 __free_page(buf->page);
2717}
2718
2719static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
2720 unsigned int idx)
2721{
2722 __free_page(spd->pages[idx]);
2723}
2724
2725static struct pipe_buf_operations tracing_pipe_buf_ops = {
2726 .can_merge = 0,
2727 .map = generic_pipe_buf_map,
2728 .unmap = generic_pipe_buf_unmap,
2729 .confirm = generic_pipe_buf_confirm,
2730 .release = tracing_pipe_buf_release,
2731 .steal = generic_pipe_buf_steal,
2732 .get = generic_pipe_buf_get,
2733};
2734
2735static size_t
2736tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
2737{
2738 size_t count;
2739 int ret;
2740
2741 /* Seq buffer is page-sized, exactly what we need. */
2742 for (;;) {
2743 count = iter->seq.len;
2744 ret = print_trace_line(iter);
2745 count = iter->seq.len - count;
2746 if (rem < count) {
2747 rem = 0;
2748 iter->seq.len -= count;
2749 break;
2750 }
2751 if (ret == TRACE_TYPE_PARTIAL_LINE) {
2752 iter->seq.len -= count;
2753 break;
2754 }
2755
2756 trace_consume(iter);
2757 rem -= count;
2758 if (!find_next_entry_inc(iter)) {
2759 rem = 0;
2760 iter->ent = NULL;
2761 break;
2762 }
2763 }
2764
2765 return rem;
2766}
2767
2768static ssize_t tracing_splice_read_pipe(struct file *filp,
2769 loff_t *ppos,
2770 struct pipe_inode_info *pipe,
2771 size_t len,
2772 unsigned int flags)
2773{
2774 struct page *pages[PIPE_BUFFERS];
2775 struct partial_page partial[PIPE_BUFFERS];
2776 struct trace_iterator *iter = filp->private_data;
2777 struct splice_pipe_desc spd = {
2778 .pages = pages,
2779 .partial = partial,
2780 .nr_pages = 0, /* This gets updated below. */
2781 .flags = flags,
2782 .ops = &tracing_pipe_buf_ops,
2783 .spd_release = tracing_spd_release_pipe,
2784 };
2785 static struct tracer *old_tracer;
2786 ssize_t ret;
2787 size_t rem;
2788 unsigned int i;
2789
2790 /* copy the tracer to avoid using a global lock all around */
2791 mutex_lock(&trace_types_lock);
2792 if (unlikely(old_tracer != current_trace && current_trace)) {
2793 old_tracer = current_trace;
2794 *iter->trace = *current_trace;
2795 }
2796 mutex_unlock(&trace_types_lock);
2797
2798 mutex_lock(&iter->mutex);
2799
2800 if (iter->trace->splice_read) {
2801 ret = iter->trace->splice_read(iter, filp,
2802 ppos, pipe, len, flags);
2803 if (ret)
2804 goto out_err;
2805 }
2806
2807 ret = tracing_wait_pipe(filp);
2808 if (ret <= 0)
2809 goto out_err;
2810
2811 if (!iter->ent && !find_next_entry_inc(iter)) {
2812 ret = -EFAULT;
2813 goto out_err;
2814 }
2815
2816 /* Fill as many pages as possible. */
2817 for (i = 0, rem = len; i < PIPE_BUFFERS && rem; i++) {
2818 pages[i] = alloc_page(GFP_KERNEL);
2819 if (!pages[i])
2820 break;
2821
2822 rem = tracing_fill_pipe_page(rem, iter);
2823
2824 /* Copy the data into the page, so we can start over. */
2825 ret = trace_seq_to_buffer(&iter->seq,
2826 page_address(pages[i]),
2827 iter->seq.len);
2828 if (ret < 0) {
2829 __free_page(pages[i]);
2830 break;
2831 }
2832 partial[i].offset = 0;
2833 partial[i].len = iter->seq.len;
2834
2835 trace_seq_init(&iter->seq);
2836 }
2837
2838 mutex_unlock(&iter->mutex);
2839
2840 spd.nr_pages = i;
2841
2842 return splice_to_pipe(pipe, &spd);
2843
2844out_err:
2845 mutex_unlock(&iter->mutex);
2846
2847 return ret;
2848}
2849
3295static ssize_t 2850static ssize_t
3296tracing_entries_read(struct file *filp, char __user *ubuf, 2851tracing_entries_read(struct file *filp, char __user *ubuf,
3297 size_t cnt, loff_t *ppos) 2852 size_t cnt, loff_t *ppos)
@@ -3433,42 +2988,288 @@ tracing_mark_write(struct file *filp, const char __user *ubuf,
3433 return cnt; 2988 return cnt;
3434} 2989}
3435 2990
3436static struct file_operations tracing_max_lat_fops = { 2991static const struct file_operations tracing_max_lat_fops = {
3437 .open = tracing_open_generic, 2992 .open = tracing_open_generic,
3438 .read = tracing_max_lat_read, 2993 .read = tracing_max_lat_read,
3439 .write = tracing_max_lat_write, 2994 .write = tracing_max_lat_write,
3440}; 2995};
3441 2996
3442static struct file_operations tracing_ctrl_fops = { 2997static const struct file_operations tracing_ctrl_fops = {
3443 .open = tracing_open_generic, 2998 .open = tracing_open_generic,
3444 .read = tracing_ctrl_read, 2999 .read = tracing_ctrl_read,
3445 .write = tracing_ctrl_write, 3000 .write = tracing_ctrl_write,
3446}; 3001};
3447 3002
3448static struct file_operations set_tracer_fops = { 3003static const struct file_operations set_tracer_fops = {
3449 .open = tracing_open_generic, 3004 .open = tracing_open_generic,
3450 .read = tracing_set_trace_read, 3005 .read = tracing_set_trace_read,
3451 .write = tracing_set_trace_write, 3006 .write = tracing_set_trace_write,
3452}; 3007};
3453 3008
3454static struct file_operations tracing_pipe_fops = { 3009static const struct file_operations tracing_pipe_fops = {
3455 .open = tracing_open_pipe, 3010 .open = tracing_open_pipe,
3456 .poll = tracing_poll_pipe, 3011 .poll = tracing_poll_pipe,
3457 .read = tracing_read_pipe, 3012 .read = tracing_read_pipe,
3013 .splice_read = tracing_splice_read_pipe,
3458 .release = tracing_release_pipe, 3014 .release = tracing_release_pipe,
3459}; 3015};
3460 3016
3461static struct file_operations tracing_entries_fops = { 3017static const struct file_operations tracing_entries_fops = {
3462 .open = tracing_open_generic, 3018 .open = tracing_open_generic,
3463 .read = tracing_entries_read, 3019 .read = tracing_entries_read,
3464 .write = tracing_entries_write, 3020 .write = tracing_entries_write,
3465}; 3021};
3466 3022
3467static struct file_operations tracing_mark_fops = { 3023static const struct file_operations tracing_mark_fops = {
3468 .open = tracing_open_generic, 3024 .open = tracing_open_generic,
3469 .write = tracing_mark_write, 3025 .write = tracing_mark_write,
3470}; 3026};
3471 3027
3028struct ftrace_buffer_info {
3029 struct trace_array *tr;
3030 void *spare;
3031 int cpu;
3032 unsigned int read;
3033};
3034
3035static int tracing_buffers_open(struct inode *inode, struct file *filp)
3036{
3037 int cpu = (int)(long)inode->i_private;
3038 struct ftrace_buffer_info *info;
3039
3040 if (tracing_disabled)
3041 return -ENODEV;
3042
3043 info = kzalloc(sizeof(*info), GFP_KERNEL);
3044 if (!info)
3045 return -ENOMEM;
3046
3047 info->tr = &global_trace;
3048 info->cpu = cpu;
3049 info->spare = ring_buffer_alloc_read_page(info->tr->buffer);
3050 /* Force reading ring buffer for first read */
3051 info->read = (unsigned int)-1;
3052 if (!info->spare)
3053 goto out;
3054
3055 filp->private_data = info;
3056
3057 return 0;
3058
3059 out:
3060 kfree(info);
3061 return -ENOMEM;
3062}
3063
3064static ssize_t
3065tracing_buffers_read(struct file *filp, char __user *ubuf,
3066 size_t count, loff_t *ppos)
3067{
3068 struct ftrace_buffer_info *info = filp->private_data;
3069 unsigned int pos;
3070 ssize_t ret;
3071 size_t size;
3072
3073 if (!count)
3074 return 0;
3075
3076 /* Do we have previous read data to read? */
3077 if (info->read < PAGE_SIZE)
3078 goto read;
3079
3080 info->read = 0;
3081
3082 ret = ring_buffer_read_page(info->tr->buffer,
3083 &info->spare,
3084 count,
3085 info->cpu, 0);
3086 if (ret < 0)
3087 return 0;
3088
3089 pos = ring_buffer_page_len(info->spare);
3090
3091 if (pos < PAGE_SIZE)
3092 memset(info->spare + pos, 0, PAGE_SIZE - pos);
3093
3094read:
3095 size = PAGE_SIZE - info->read;
3096 if (size > count)
3097 size = count;
3098
3099 ret = copy_to_user(ubuf, info->spare + info->read, size);
3100 if (ret == size)
3101 return -EFAULT;
3102 size -= ret;
3103
3104 *ppos += size;
3105 info->read += size;
3106
3107 return size;
3108}
3109
3110static int tracing_buffers_release(struct inode *inode, struct file *file)
3111{
3112 struct ftrace_buffer_info *info = file->private_data;
3113
3114 ring_buffer_free_read_page(info->tr->buffer, info->spare);
3115 kfree(info);
3116
3117 return 0;
3118}
3119
3120struct buffer_ref {
3121 struct ring_buffer *buffer;
3122 void *page;
3123 int ref;
3124};
3125
3126static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
3127 struct pipe_buffer *buf)
3128{
3129 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
3130
3131 if (--ref->ref)
3132 return;
3133
3134 ring_buffer_free_read_page(ref->buffer, ref->page);
3135 kfree(ref);
3136 buf->private = 0;
3137}
3138
3139static int buffer_pipe_buf_steal(struct pipe_inode_info *pipe,
3140 struct pipe_buffer *buf)
3141{
3142 return 1;
3143}
3144
3145static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
3146 struct pipe_buffer *buf)
3147{
3148 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
3149
3150 ref->ref++;
3151}
3152
3153/* Pipe buffer operations for a buffer. */
3154static struct pipe_buf_operations buffer_pipe_buf_ops = {
3155 .can_merge = 0,
3156 .map = generic_pipe_buf_map,
3157 .unmap = generic_pipe_buf_unmap,
3158 .confirm = generic_pipe_buf_confirm,
3159 .release = buffer_pipe_buf_release,
3160 .steal = buffer_pipe_buf_steal,
3161 .get = buffer_pipe_buf_get,
3162};
3163
3164/*
3165 * Callback from splice_to_pipe(), if we need to release some pages
3166 * at the end of the spd in case we error'ed out in filling the pipe.
3167 */
3168static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
3169{
3170 struct buffer_ref *ref =
3171 (struct buffer_ref *)spd->partial[i].private;
3172
3173 if (--ref->ref)
3174 return;
3175
3176 ring_buffer_free_read_page(ref->buffer, ref->page);
3177 kfree(ref);
3178 spd->partial[i].private = 0;
3179}
3180
3181static ssize_t
3182tracing_buffers_splice_read(struct file *file, loff_t *ppos,
3183 struct pipe_inode_info *pipe, size_t len,
3184 unsigned int flags)
3185{
3186 struct ftrace_buffer_info *info = file->private_data;
3187 struct partial_page partial[PIPE_BUFFERS];
3188 struct page *pages[PIPE_BUFFERS];
3189 struct splice_pipe_desc spd = {
3190 .pages = pages,
3191 .partial = partial,
3192 .flags = flags,
3193 .ops = &buffer_pipe_buf_ops,
3194 .spd_release = buffer_spd_release,
3195 };
3196 struct buffer_ref *ref;
3197 int size, i;
3198 size_t ret;
3199
3200 /*
3201 * We can't seek on a buffer input
3202 */
3203 if (unlikely(*ppos))
3204 return -ESPIPE;
3205
3206
3207 for (i = 0; i < PIPE_BUFFERS && len; i++, len -= size) {
3208 struct page *page;
3209 int r;
3210
3211 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
3212 if (!ref)
3213 break;
3214
3215 ref->buffer = info->tr->buffer;
3216 ref->page = ring_buffer_alloc_read_page(ref->buffer);
3217 if (!ref->page) {
3218 kfree(ref);
3219 break;
3220 }
3221
3222 r = ring_buffer_read_page(ref->buffer, &ref->page,
3223 len, info->cpu, 0);
3224 if (r < 0) {
3225 ring_buffer_free_read_page(ref->buffer,
3226 ref->page);
3227 kfree(ref);
3228 break;
3229 }
3230
3231 /*
3232 * zero out any left over data, this is going to
3233 * user land.
3234 */
3235 size = ring_buffer_page_len(ref->page);
3236 if (size < PAGE_SIZE)
3237 memset(ref->page + size, 0, PAGE_SIZE - size);
3238
3239 page = virt_to_page(ref->page);
3240
3241 spd.pages[i] = page;
3242 spd.partial[i].len = PAGE_SIZE;
3243 spd.partial[i].offset = 0;
3244 spd.partial[i].private = (unsigned long)ref;
3245 spd.nr_pages++;
3246 }
3247
3248 spd.nr_pages = i;
3249
3250 /* did we read anything? */
3251 if (!spd.nr_pages) {
3252 if (flags & SPLICE_F_NONBLOCK)
3253 ret = -EAGAIN;
3254 else
3255 ret = 0;
3256 /* TODO: block */
3257 return ret;
3258 }
3259
3260 ret = splice_to_pipe(pipe, &spd);
3261
3262 return ret;
3263}
3264
3265static const struct file_operations tracing_buffers_fops = {
3266 .open = tracing_buffers_open,
3267 .read = tracing_buffers_read,
3268 .release = tracing_buffers_release,
3269 .splice_read = tracing_buffers_splice_read,
3270 .llseek = no_llseek,
3271};
3272
3472#ifdef CONFIG_DYNAMIC_FTRACE 3273#ifdef CONFIG_DYNAMIC_FTRACE
3473 3274
3474int __weak ftrace_arch_read_dyn_info(char *buf, int size) 3275int __weak ftrace_arch_read_dyn_info(char *buf, int size)
@@ -3500,7 +3301,7 @@ tracing_read_dyn_info(struct file *filp, char __user *ubuf,
3500 return r; 3301 return r;
3501} 3302}
3502 3303
3503static struct file_operations tracing_dyn_info_fops = { 3304static const struct file_operations tracing_dyn_info_fops = {
3504 .open = tracing_open_generic, 3305 .open = tracing_open_generic,
3505 .read = tracing_read_dyn_info, 3306 .read = tracing_read_dyn_info,
3506}; 3307};
@@ -3526,15 +3327,346 @@ struct dentry *tracing_init_dentry(void)
3526 return d_tracer; 3327 return d_tracer;
3527} 3328}
3528 3329
3330static struct dentry *d_percpu;
3331
3332struct dentry *tracing_dentry_percpu(void)
3333{
3334 static int once;
3335 struct dentry *d_tracer;
3336
3337 if (d_percpu)
3338 return d_percpu;
3339
3340 d_tracer = tracing_init_dentry();
3341
3342 if (!d_tracer)
3343 return NULL;
3344
3345 d_percpu = debugfs_create_dir("per_cpu", d_tracer);
3346
3347 if (!d_percpu && !once) {
3348 once = 1;
3349 pr_warning("Could not create debugfs directory 'per_cpu'\n");
3350 return NULL;
3351 }
3352
3353 return d_percpu;
3354}
3355
3356static void tracing_init_debugfs_percpu(long cpu)
3357{
3358 struct dentry *d_percpu = tracing_dentry_percpu();
3359 struct dentry *entry, *d_cpu;
3360 /* strlen(cpu) + MAX(log10(cpu)) + '\0' */
3361 char cpu_dir[7];
3362
3363 if (cpu > 999 || cpu < 0)
3364 return;
3365
3366 sprintf(cpu_dir, "cpu%ld", cpu);
3367 d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
3368 if (!d_cpu) {
3369 pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
3370 return;
3371 }
3372
3373 /* per cpu trace_pipe */
3374 entry = debugfs_create_file("trace_pipe", 0444, d_cpu,
3375 (void *) cpu, &tracing_pipe_fops);
3376 if (!entry)
3377 pr_warning("Could not create debugfs 'trace_pipe' entry\n");
3378
3379 /* per cpu trace */
3380 entry = debugfs_create_file("trace", 0444, d_cpu,
3381 (void *) cpu, &tracing_fops);
3382 if (!entry)
3383 pr_warning("Could not create debugfs 'trace' entry\n");
3384}
3385
3529#ifdef CONFIG_FTRACE_SELFTEST 3386#ifdef CONFIG_FTRACE_SELFTEST
3530/* Let selftest have access to static functions in this file */ 3387/* Let selftest have access to static functions in this file */
3531#include "trace_selftest.c" 3388#include "trace_selftest.c"
3532#endif 3389#endif
3533 3390
3391struct trace_option_dentry {
3392 struct tracer_opt *opt;
3393 struct tracer_flags *flags;
3394 struct dentry *entry;
3395};
3396
3397static ssize_t
3398trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
3399 loff_t *ppos)
3400{
3401 struct trace_option_dentry *topt = filp->private_data;
3402 char *buf;
3403
3404 if (topt->flags->val & topt->opt->bit)
3405 buf = "1\n";
3406 else
3407 buf = "0\n";
3408
3409 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
3410}
3411
3412static ssize_t
3413trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
3414 loff_t *ppos)
3415{
3416 struct trace_option_dentry *topt = filp->private_data;
3417 unsigned long val;
3418 char buf[64];
3419 int ret;
3420
3421 if (cnt >= sizeof(buf))
3422 return -EINVAL;
3423
3424 if (copy_from_user(&buf, ubuf, cnt))
3425 return -EFAULT;
3426
3427 buf[cnt] = 0;
3428
3429 ret = strict_strtoul(buf, 10, &val);
3430 if (ret < 0)
3431 return ret;
3432
3433 ret = 0;
3434 switch (val) {
3435 case 0:
3436 /* do nothing if already cleared */
3437 if (!(topt->flags->val & topt->opt->bit))
3438 break;
3439
3440 mutex_lock(&trace_types_lock);
3441 if (current_trace->set_flag)
3442 ret = current_trace->set_flag(topt->flags->val,
3443 topt->opt->bit, 0);
3444 mutex_unlock(&trace_types_lock);
3445 if (ret)
3446 return ret;
3447 topt->flags->val &= ~topt->opt->bit;
3448 break;
3449 case 1:
3450 /* do nothing if already set */
3451 if (topt->flags->val & topt->opt->bit)
3452 break;
3453
3454 mutex_lock(&trace_types_lock);
3455 if (current_trace->set_flag)
3456 ret = current_trace->set_flag(topt->flags->val,
3457 topt->opt->bit, 1);
3458 mutex_unlock(&trace_types_lock);
3459 if (ret)
3460 return ret;
3461 topt->flags->val |= topt->opt->bit;
3462 break;
3463
3464 default:
3465 return -EINVAL;
3466 }
3467
3468 *ppos += cnt;
3469
3470 return cnt;
3471}
3472
3473
3474static const struct file_operations trace_options_fops = {
3475 .open = tracing_open_generic,
3476 .read = trace_options_read,
3477 .write = trace_options_write,
3478};
3479
3480static ssize_t
3481trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
3482 loff_t *ppos)
3483{
3484 long index = (long)filp->private_data;
3485 char *buf;
3486
3487 if (trace_flags & (1 << index))
3488 buf = "1\n";
3489 else
3490 buf = "0\n";
3491
3492 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
3493}
3494
3495static ssize_t
3496trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
3497 loff_t *ppos)
3498{
3499 long index = (long)filp->private_data;
3500 char buf[64];
3501 unsigned long val;
3502 int ret;
3503
3504 if (cnt >= sizeof(buf))
3505 return -EINVAL;
3506
3507 if (copy_from_user(&buf, ubuf, cnt))
3508 return -EFAULT;
3509
3510 buf[cnt] = 0;
3511
3512 ret = strict_strtoul(buf, 10, &val);
3513 if (ret < 0)
3514 return ret;
3515
3516 switch (val) {
3517 case 0:
3518 trace_flags &= ~(1 << index);
3519 break;
3520 case 1:
3521 trace_flags |= 1 << index;
3522 break;
3523
3524 default:
3525 return -EINVAL;
3526 }
3527
3528 *ppos += cnt;
3529
3530 return cnt;
3531}
3532
3533static const struct file_operations trace_options_core_fops = {
3534 .open = tracing_open_generic,
3535 .read = trace_options_core_read,
3536 .write = trace_options_core_write,
3537};
3538
3539static struct dentry *trace_options_init_dentry(void)
3540{
3541 struct dentry *d_tracer;
3542 static struct dentry *t_options;
3543
3544 if (t_options)
3545 return t_options;
3546
3547 d_tracer = tracing_init_dentry();
3548 if (!d_tracer)
3549 return NULL;
3550
3551 t_options = debugfs_create_dir("options", d_tracer);
3552 if (!t_options) {
3553 pr_warning("Could not create debugfs directory 'options'\n");
3554 return NULL;
3555 }
3556
3557 return t_options;
3558}
3559
3560static void
3561create_trace_option_file(struct trace_option_dentry *topt,
3562 struct tracer_flags *flags,
3563 struct tracer_opt *opt)
3564{
3565 struct dentry *t_options;
3566 struct dentry *entry;
3567
3568 t_options = trace_options_init_dentry();
3569 if (!t_options)
3570 return;
3571
3572 topt->flags = flags;
3573 topt->opt = opt;
3574
3575 entry = debugfs_create_file(opt->name, 0644, t_options, topt,
3576 &trace_options_fops);
3577
3578 topt->entry = entry;
3579
3580}
3581
3582static struct trace_option_dentry *
3583create_trace_option_files(struct tracer *tracer)
3584{
3585 struct trace_option_dentry *topts;
3586 struct tracer_flags *flags;
3587 struct tracer_opt *opts;
3588 int cnt;
3589
3590 if (!tracer)
3591 return NULL;
3592
3593 flags = tracer->flags;
3594
3595 if (!flags || !flags->opts)
3596 return NULL;
3597
3598 opts = flags->opts;
3599
3600 for (cnt = 0; opts[cnt].name; cnt++)
3601 ;
3602
3603 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
3604 if (!topts)
3605 return NULL;
3606
3607 for (cnt = 0; opts[cnt].name; cnt++)
3608 create_trace_option_file(&topts[cnt], flags,
3609 &opts[cnt]);
3610
3611 return topts;
3612}
3613
3614static void
3615destroy_trace_option_files(struct trace_option_dentry *topts)
3616{
3617 int cnt;
3618
3619 if (!topts)
3620 return;
3621
3622 for (cnt = 0; topts[cnt].opt; cnt++) {
3623 if (topts[cnt].entry)
3624 debugfs_remove(topts[cnt].entry);
3625 }
3626
3627 kfree(topts);
3628}
3629
3630static struct dentry *
3631create_trace_option_core_file(const char *option, long index)
3632{
3633 struct dentry *t_options;
3634 struct dentry *entry;
3635
3636 t_options = trace_options_init_dentry();
3637 if (!t_options)
3638 return NULL;
3639
3640 entry = debugfs_create_file(option, 0644, t_options, (void *)index,
3641 &trace_options_core_fops);
3642
3643 return entry;
3644}
3645
3646static __init void create_trace_options_dir(void)
3647{
3648 struct dentry *t_options;
3649 struct dentry *entry;
3650 int i;
3651
3652 t_options = trace_options_init_dentry();
3653 if (!t_options)
3654 return;
3655
3656 for (i = 0; trace_options[i]; i++) {
3657 entry = create_trace_option_core_file(trace_options[i], i);
3658 if (!entry)
3659 pr_warning("Could not create debugfs %s entry\n",
3660 trace_options[i]);
3661 }
3662}
3663
3534static __init int tracer_init_debugfs(void) 3664static __init int tracer_init_debugfs(void)
3535{ 3665{
3536 struct dentry *d_tracer; 3666 struct dentry *d_tracer;
3667 struct dentry *buffers;
3537 struct dentry *entry; 3668 struct dentry *entry;
3669 int cpu;
3538 3670
3539 d_tracer = tracing_init_dentry(); 3671 d_tracer = tracing_init_dentry();
3540 3672
@@ -3548,18 +3680,15 @@ static __init int tracer_init_debugfs(void)
3548 if (!entry) 3680 if (!entry)
3549 pr_warning("Could not create debugfs 'trace_options' entry\n"); 3681 pr_warning("Could not create debugfs 'trace_options' entry\n");
3550 3682
3683 create_trace_options_dir();
3684
3551 entry = debugfs_create_file("tracing_cpumask", 0644, d_tracer, 3685 entry = debugfs_create_file("tracing_cpumask", 0644, d_tracer,
3552 NULL, &tracing_cpumask_fops); 3686 NULL, &tracing_cpumask_fops);
3553 if (!entry) 3687 if (!entry)
3554 pr_warning("Could not create debugfs 'tracing_cpumask' entry\n"); 3688 pr_warning("Could not create debugfs 'tracing_cpumask' entry\n");
3555 3689
3556 entry = debugfs_create_file("latency_trace", 0444, d_tracer,
3557 &global_trace, &tracing_lt_fops);
3558 if (!entry)
3559 pr_warning("Could not create debugfs 'latency_trace' entry\n");
3560
3561 entry = debugfs_create_file("trace", 0444, d_tracer, 3690 entry = debugfs_create_file("trace", 0444, d_tracer,
3562 &global_trace, &tracing_fops); 3691 (void *) TRACE_PIPE_ALL_CPU, &tracing_fops);
3563 if (!entry) 3692 if (!entry)
3564 pr_warning("Could not create debugfs 'trace' entry\n"); 3693 pr_warning("Could not create debugfs 'trace' entry\n");
3565 3694
@@ -3590,8 +3719,8 @@ static __init int tracer_init_debugfs(void)
3590 if (!entry) 3719 if (!entry)
3591 pr_warning("Could not create debugfs 'README' entry\n"); 3720 pr_warning("Could not create debugfs 'README' entry\n");
3592 3721
3593 entry = debugfs_create_file("trace_pipe", 0644, d_tracer, 3722 entry = debugfs_create_file("trace_pipe", 0444, d_tracer,
3594 NULL, &tracing_pipe_fops); 3723 (void *) TRACE_PIPE_ALL_CPU, &tracing_pipe_fops);
3595 if (!entry) 3724 if (!entry)
3596 pr_warning("Could not create debugfs " 3725 pr_warning("Could not create debugfs "
3597 "'trace_pipe' entry\n"); 3726 "'trace_pipe' entry\n");
@@ -3608,6 +3737,26 @@ static __init int tracer_init_debugfs(void)
3608 pr_warning("Could not create debugfs " 3737 pr_warning("Could not create debugfs "
3609 "'trace_marker' entry\n"); 3738 "'trace_marker' entry\n");
3610 3739
3740 buffers = debugfs_create_dir("binary_buffers", d_tracer);
3741
3742 if (!buffers)
3743 pr_warning("Could not create buffers directory\n");
3744 else {
3745 int cpu;
3746 char buf[64];
3747
3748 for_each_tracing_cpu(cpu) {
3749 sprintf(buf, "%d", cpu);
3750
3751 entry = debugfs_create_file(buf, 0444, buffers,
3752 (void *)(long)cpu,
3753 &tracing_buffers_fops);
3754 if (!entry)
3755 pr_warning("Could not create debugfs buffers "
3756 "'%s' entry\n", buf);
3757 }
3758 }
3759
3611#ifdef CONFIG_DYNAMIC_FTRACE 3760#ifdef CONFIG_DYNAMIC_FTRACE
3612 entry = debugfs_create_file("dyn_ftrace_total_info", 0444, d_tracer, 3761 entry = debugfs_create_file("dyn_ftrace_total_info", 0444, d_tracer,
3613 &ftrace_update_tot_cnt, 3762 &ftrace_update_tot_cnt,
@@ -3619,77 +3768,12 @@ static __init int tracer_init_debugfs(void)
3619#ifdef CONFIG_SYSPROF_TRACER 3768#ifdef CONFIG_SYSPROF_TRACER
3620 init_tracer_sysprof_debugfs(d_tracer); 3769 init_tracer_sysprof_debugfs(d_tracer);
3621#endif 3770#endif
3622 return 0;
3623}
3624
3625int trace_vprintk(unsigned long ip, int depth, const char *fmt, va_list args)
3626{
3627 static DEFINE_SPINLOCK(trace_buf_lock);
3628 static char trace_buf[TRACE_BUF_SIZE];
3629
3630 struct ring_buffer_event *event;
3631 struct trace_array *tr = &global_trace;
3632 struct trace_array_cpu *data;
3633 int cpu, len = 0, size, pc;
3634 struct print_entry *entry;
3635 unsigned long irq_flags;
3636 3771
3637 if (tracing_disabled || tracing_selftest_running) 3772 for_each_tracing_cpu(cpu)
3638 return 0; 3773 tracing_init_debugfs_percpu(cpu);
3639 3774
3640 pc = preempt_count(); 3775 return 0;
3641 preempt_disable_notrace();
3642 cpu = raw_smp_processor_id();
3643 data = tr->data[cpu];
3644
3645 if (unlikely(atomic_read(&data->disabled)))
3646 goto out;
3647
3648 pause_graph_tracing();
3649 spin_lock_irqsave(&trace_buf_lock, irq_flags);
3650 len = vsnprintf(trace_buf, TRACE_BUF_SIZE, fmt, args);
3651
3652 len = min(len, TRACE_BUF_SIZE-1);
3653 trace_buf[len] = 0;
3654
3655 size = sizeof(*entry) + len + 1;
3656 event = ring_buffer_lock_reserve(tr->buffer, size, &irq_flags);
3657 if (!event)
3658 goto out_unlock;
3659 entry = ring_buffer_event_data(event);
3660 tracing_generic_entry_update(&entry->ent, irq_flags, pc);
3661 entry->ent.type = TRACE_PRINT;
3662 entry->ip = ip;
3663 entry->depth = depth;
3664
3665 memcpy(&entry->buf, trace_buf, len);
3666 entry->buf[len] = 0;
3667 ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
3668
3669 out_unlock:
3670 spin_unlock_irqrestore(&trace_buf_lock, irq_flags);
3671 unpause_graph_tracing();
3672 out:
3673 preempt_enable_notrace();
3674
3675 return len;
3676}
3677EXPORT_SYMBOL_GPL(trace_vprintk);
3678
3679int __ftrace_printk(unsigned long ip, const char *fmt, ...)
3680{
3681 int ret;
3682 va_list ap;
3683
3684 if (!(trace_flags & TRACE_ITER_PRINTK))
3685 return 0;
3686
3687 va_start(ap, fmt);
3688 ret = trace_vprintk(ip, task_curr_ret_stack(current), fmt, ap);
3689 va_end(ap);
3690 return ret;
3691} 3776}
3692EXPORT_SYMBOL_GPL(__ftrace_printk);
3693 3777
3694static int trace_panic_handler(struct notifier_block *this, 3778static int trace_panic_handler(struct notifier_block *this,
3695 unsigned long event, void *unused) 3779 unsigned long event, void *unused)
@@ -3750,7 +3834,7 @@ trace_printk_seq(struct trace_seq *s)
3750 3834
3751 printk(KERN_TRACE "%s", s->buffer); 3835 printk(KERN_TRACE "%s", s->buffer);
3752 3836
3753 trace_seq_reset(s); 3837 trace_seq_init(s);
3754} 3838}
3755 3839
3756void ftrace_dump(void) 3840void ftrace_dump(void)
@@ -3782,8 +3866,10 @@ void ftrace_dump(void)
3782 3866
3783 printk(KERN_TRACE "Dumping ftrace buffer:\n"); 3867 printk(KERN_TRACE "Dumping ftrace buffer:\n");
3784 3868
3869 /* Simulate the iterator */
3785 iter.tr = &global_trace; 3870 iter.tr = &global_trace;
3786 iter.trace = current_trace; 3871 iter.trace = current_trace;
3872 iter.cpu_file = TRACE_PIPE_ALL_CPU;
3787 3873
3788 /* 3874 /*
3789 * We need to stop all tracing on all CPUS to read the 3875 * We need to stop all tracing on all CPUS to read the
@@ -3835,8 +3921,12 @@ __init static int tracer_alloc_buffers(void)
3835 if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL)) 3921 if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL))
3836 goto out_free_buffer_mask; 3922 goto out_free_buffer_mask;
3837 3923
3924 if (!alloc_cpumask_var(&tracing_reader_cpumask, GFP_KERNEL))
3925 goto out_free_tracing_cpumask;
3926
3838 cpumask_copy(tracing_buffer_mask, cpu_possible_mask); 3927 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
3839 cpumask_copy(tracing_cpumask, cpu_all_mask); 3928 cpumask_copy(tracing_cpumask, cpu_all_mask);
3929 cpumask_clear(tracing_reader_cpumask);
3840 3930
3841 /* TODO: make the number of buffers hot pluggable with CPUS */ 3931 /* TODO: make the number of buffers hot pluggable with CPUS */
3842 global_trace.buffer = ring_buffer_alloc(trace_buf_size, 3932 global_trace.buffer = ring_buffer_alloc(trace_buf_size,
@@ -3871,14 +3961,10 @@ __init static int tracer_alloc_buffers(void)
3871 trace_init_cmdlines(); 3961 trace_init_cmdlines();
3872 3962
3873 register_tracer(&nop_trace); 3963 register_tracer(&nop_trace);
3964 current_trace = &nop_trace;
3874#ifdef CONFIG_BOOT_TRACER 3965#ifdef CONFIG_BOOT_TRACER
3875 register_tracer(&boot_tracer); 3966 register_tracer(&boot_tracer);
3876 current_trace = &boot_tracer;
3877 current_trace->init(&global_trace);
3878#else
3879 current_trace = &nop_trace;
3880#endif 3967#endif
3881
3882 /* All seems OK, enable tracing */ 3968 /* All seems OK, enable tracing */
3883 tracing_disabled = 0; 3969 tracing_disabled = 0;
3884 3970
@@ -3889,11 +3975,34 @@ __init static int tracer_alloc_buffers(void)
3889 ret = 0; 3975 ret = 0;
3890 3976
3891out_free_cpumask: 3977out_free_cpumask:
3978 free_cpumask_var(tracing_reader_cpumask);
3979out_free_tracing_cpumask:
3892 free_cpumask_var(tracing_cpumask); 3980 free_cpumask_var(tracing_cpumask);
3893out_free_buffer_mask: 3981out_free_buffer_mask:
3894 free_cpumask_var(tracing_buffer_mask); 3982 free_cpumask_var(tracing_buffer_mask);
3895out: 3983out:
3896 return ret; 3984 return ret;
3897} 3985}
3986
3987__init static int clear_boot_tracer(void)
3988{
3989 /*
3990 * The default tracer at boot buffer is an init section.
3991 * This function is called in lateinit. If we did not
3992 * find the boot tracer, then clear it out, to prevent
3993 * later registration from accessing the buffer that is
3994 * about to be freed.
3995 */
3996 if (!default_bootup_tracer)
3997 return 0;
3998
3999 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
4000 default_bootup_tracer);
4001 default_bootup_tracer = NULL;
4002
4003 return 0;
4004}
4005
3898early_initcall(tracer_alloc_buffers); 4006early_initcall(tracer_alloc_buffers);
3899fs_initcall(tracer_init_debugfs); 4007fs_initcall(tracer_init_debugfs);
4008late_initcall(clear_boot_tracer);
diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h
index 4d3d381bfd95..c5e1d8865fe4 100644
--- a/kernel/trace/trace.h
+++ b/kernel/trace/trace.h
@@ -9,6 +9,8 @@
9#include <linux/mmiotrace.h> 9#include <linux/mmiotrace.h>
10#include <linux/ftrace.h> 10#include <linux/ftrace.h>
11#include <trace/boot.h> 11#include <trace/boot.h>
12#include <trace/kmemtrace.h>
13#include <trace/power.h>
12 14
13enum trace_type { 15enum trace_type {
14 __TRACE_FIRST_TYPE = 0, 16 __TRACE_FIRST_TYPE = 0,
@@ -16,7 +18,6 @@ enum trace_type {
16 TRACE_FN, 18 TRACE_FN,
17 TRACE_CTX, 19 TRACE_CTX,
18 TRACE_WAKE, 20 TRACE_WAKE,
19 TRACE_CONT,
20 TRACE_STACK, 21 TRACE_STACK,
21 TRACE_PRINT, 22 TRACE_PRINT,
22 TRACE_SPECIAL, 23 TRACE_SPECIAL,
@@ -29,9 +30,12 @@ enum trace_type {
29 TRACE_GRAPH_ENT, 30 TRACE_GRAPH_ENT,
30 TRACE_USER_STACK, 31 TRACE_USER_STACK,
31 TRACE_HW_BRANCHES, 32 TRACE_HW_BRANCHES,
33 TRACE_KMEM_ALLOC,
34 TRACE_KMEM_FREE,
32 TRACE_POWER, 35 TRACE_POWER,
36 TRACE_BLK,
33 37
34 __TRACE_LAST_TYPE 38 __TRACE_LAST_TYPE,
35}; 39};
36 40
37/* 41/*
@@ -42,7 +46,6 @@ enum trace_type {
42 */ 46 */
43struct trace_entry { 47struct trace_entry {
44 unsigned char type; 48 unsigned char type;
45 unsigned char cpu;
46 unsigned char flags; 49 unsigned char flags;
47 unsigned char preempt_count; 50 unsigned char preempt_count;
48 int pid; 51 int pid;
@@ -60,13 +63,13 @@ struct ftrace_entry {
60 63
61/* Function call entry */ 64/* Function call entry */
62struct ftrace_graph_ent_entry { 65struct ftrace_graph_ent_entry {
63 struct trace_entry ent; 66 struct trace_entry ent;
64 struct ftrace_graph_ent graph_ent; 67 struct ftrace_graph_ent graph_ent;
65}; 68};
66 69
67/* Function return entry */ 70/* Function return entry */
68struct ftrace_graph_ret_entry { 71struct ftrace_graph_ret_entry {
69 struct trace_entry ent; 72 struct trace_entry ent;
70 struct ftrace_graph_ret ret; 73 struct ftrace_graph_ret ret;
71}; 74};
72extern struct tracer boot_tracer; 75extern struct tracer boot_tracer;
@@ -112,13 +115,14 @@ struct userstack_entry {
112}; 115};
113 116
114/* 117/*
115 * ftrace_printk entry: 118 * trace_printk entry:
116 */ 119 */
117struct print_entry { 120struct print_entry {
118 struct trace_entry ent; 121 struct trace_entry ent;
119 unsigned long ip; 122 unsigned long ip;
120 int depth; 123 int depth;
121 char buf[]; 124 const char *fmt;
125 u32 buf[];
122}; 126};
123 127
124#define TRACE_OLD_SIZE 88 128#define TRACE_OLD_SIZE 88
@@ -170,15 +174,32 @@ struct trace_power {
170 struct power_trace state_data; 174 struct power_trace state_data;
171}; 175};
172 176
177struct kmemtrace_alloc_entry {
178 struct trace_entry ent;
179 enum kmemtrace_type_id type_id;
180 unsigned long call_site;
181 const void *ptr;
182 size_t bytes_req;
183 size_t bytes_alloc;
184 gfp_t gfp_flags;
185 int node;
186};
187
188struct kmemtrace_free_entry {
189 struct trace_entry ent;
190 enum kmemtrace_type_id type_id;
191 unsigned long call_site;
192 const void *ptr;
193};
194
173/* 195/*
174 * trace_flag_type is an enumeration that holds different 196 * trace_flag_type is an enumeration that holds different
175 * states when a trace occurs. These are: 197 * states when a trace occurs. These are:
176 * IRQS_OFF - interrupts were disabled 198 * IRQS_OFF - interrupts were disabled
177 * IRQS_NOSUPPORT - arch does not support irqs_disabled_flags 199 * IRQS_NOSUPPORT - arch does not support irqs_disabled_flags
178 * NEED_RESCED - reschedule is requested 200 * NEED_RESCED - reschedule is requested
179 * HARDIRQ - inside an interrupt handler 201 * HARDIRQ - inside an interrupt handler
180 * SOFTIRQ - inside a softirq handler 202 * SOFTIRQ - inside a softirq handler
181 * CONT - multiple entries hold the trace item
182 */ 203 */
183enum trace_flag_type { 204enum trace_flag_type {
184 TRACE_FLAG_IRQS_OFF = 0x01, 205 TRACE_FLAG_IRQS_OFF = 0x01,
@@ -186,7 +207,6 @@ enum trace_flag_type {
186 TRACE_FLAG_NEED_RESCHED = 0x04, 207 TRACE_FLAG_NEED_RESCHED = 0x04,
187 TRACE_FLAG_HARDIRQ = 0x08, 208 TRACE_FLAG_HARDIRQ = 0x08,
188 TRACE_FLAG_SOFTIRQ = 0x10, 209 TRACE_FLAG_SOFTIRQ = 0x10,
189 TRACE_FLAG_CONT = 0x20,
190}; 210};
191 211
192#define TRACE_BUF_SIZE 1024 212#define TRACE_BUF_SIZE 1024
@@ -198,6 +218,7 @@ enum trace_flag_type {
198 */ 218 */
199struct trace_array_cpu { 219struct trace_array_cpu {
200 atomic_t disabled; 220 atomic_t disabled;
221 void *buffer_page; /* ring buffer spare */
201 222
202 /* these fields get copied into max-trace: */ 223 /* these fields get copied into max-trace: */
203 unsigned long trace_idx; 224 unsigned long trace_idx;
@@ -262,7 +283,6 @@ extern void __ftrace_bad_type(void);
262 do { \ 283 do { \
263 IF_ASSIGN(var, ent, struct ftrace_entry, TRACE_FN); \ 284 IF_ASSIGN(var, ent, struct ftrace_entry, TRACE_FN); \
264 IF_ASSIGN(var, ent, struct ctx_switch_entry, 0); \ 285 IF_ASSIGN(var, ent, struct ctx_switch_entry, 0); \
265 IF_ASSIGN(var, ent, struct trace_field_cont, TRACE_CONT); \
266 IF_ASSIGN(var, ent, struct stack_entry, TRACE_STACK); \ 286 IF_ASSIGN(var, ent, struct stack_entry, TRACE_STACK); \
267 IF_ASSIGN(var, ent, struct userstack_entry, TRACE_USER_STACK);\ 287 IF_ASSIGN(var, ent, struct userstack_entry, TRACE_USER_STACK);\
268 IF_ASSIGN(var, ent, struct print_entry, TRACE_PRINT); \ 288 IF_ASSIGN(var, ent, struct print_entry, TRACE_PRINT); \
@@ -279,7 +299,11 @@ extern void __ftrace_bad_type(void);
279 IF_ASSIGN(var, ent, struct ftrace_graph_ret_entry, \ 299 IF_ASSIGN(var, ent, struct ftrace_graph_ret_entry, \
280 TRACE_GRAPH_RET); \ 300 TRACE_GRAPH_RET); \
281 IF_ASSIGN(var, ent, struct hw_branch_entry, TRACE_HW_BRANCHES);\ 301 IF_ASSIGN(var, ent, struct hw_branch_entry, TRACE_HW_BRANCHES);\
282 IF_ASSIGN(var, ent, struct trace_power, TRACE_POWER); \ 302 IF_ASSIGN(var, ent, struct trace_power, TRACE_POWER); \
303 IF_ASSIGN(var, ent, struct kmemtrace_alloc_entry, \
304 TRACE_KMEM_ALLOC); \
305 IF_ASSIGN(var, ent, struct kmemtrace_free_entry, \
306 TRACE_KMEM_FREE); \
283 __ftrace_bad_type(); \ 307 __ftrace_bad_type(); \
284 } while (0) 308 } while (0)
285 309
@@ -287,7 +311,8 @@ extern void __ftrace_bad_type(void);
287enum print_line_t { 311enum print_line_t {
288 TRACE_TYPE_PARTIAL_LINE = 0, /* Retry after flushing the seq */ 312 TRACE_TYPE_PARTIAL_LINE = 0, /* Retry after flushing the seq */
289 TRACE_TYPE_HANDLED = 1, 313 TRACE_TYPE_HANDLED = 1,
290 TRACE_TYPE_UNHANDLED = 2 /* Relay to other output functions */ 314 TRACE_TYPE_UNHANDLED = 2, /* Relay to other output functions */
315 TRACE_TYPE_NO_CONSUME = 3 /* Handled but ask to not consume */
291}; 316};
292 317
293 318
@@ -297,8 +322,8 @@ enum print_line_t {
297 * flags value in struct tracer_flags. 322 * flags value in struct tracer_flags.
298 */ 323 */
299struct tracer_opt { 324struct tracer_opt {
300 const char *name; /* Will appear on the trace_options file */ 325 const char *name; /* Will appear on the trace_options file */
301 u32 bit; /* Mask assigned in val field in tracer_flags */ 326 u32 bit; /* Mask assigned in val field in tracer_flags */
302}; 327};
303 328
304/* 329/*
@@ -307,28 +332,51 @@ struct tracer_opt {
307 */ 332 */
308struct tracer_flags { 333struct tracer_flags {
309 u32 val; 334 u32 val;
310 struct tracer_opt *opts; 335 struct tracer_opt *opts;
311}; 336};
312 337
313/* Makes more easy to define a tracer opt */ 338/* Makes more easy to define a tracer opt */
314#define TRACER_OPT(s, b) .name = #s, .bit = b 339#define TRACER_OPT(s, b) .name = #s, .bit = b
315 340
316/* 341
317 * A specific tracer, represented by methods that operate on a trace array: 342/**
343 * struct tracer - a specific tracer and its callbacks to interact with debugfs
344 * @name: the name chosen to select it on the available_tracers file
345 * @init: called when one switches to this tracer (echo name > current_tracer)
346 * @reset: called when one switches to another tracer
347 * @start: called when tracing is unpaused (echo 1 > tracing_enabled)
348 * @stop: called when tracing is paused (echo 0 > tracing_enabled)
349 * @open: called when the trace file is opened
350 * @pipe_open: called when the trace_pipe file is opened
351 * @wait_pipe: override how the user waits for traces on trace_pipe
352 * @close: called when the trace file is released
353 * @read: override the default read callback on trace_pipe
354 * @splice_read: override the default splice_read callback on trace_pipe
355 * @selftest: selftest to run on boot (see trace_selftest.c)
356 * @print_headers: override the first lines that describe your columns
357 * @print_line: callback that prints a trace
358 * @set_flag: signals one of your private flags changed (trace_options file)
359 * @flags: your private flags
318 */ 360 */
319struct tracer { 361struct tracer {
320 const char *name; 362 const char *name;
321 /* Your tracer should raise a warning if init fails */
322 int (*init)(struct trace_array *tr); 363 int (*init)(struct trace_array *tr);
323 void (*reset)(struct trace_array *tr); 364 void (*reset)(struct trace_array *tr);
324 void (*start)(struct trace_array *tr); 365 void (*start)(struct trace_array *tr);
325 void (*stop)(struct trace_array *tr); 366 void (*stop)(struct trace_array *tr);
326 void (*open)(struct trace_iterator *iter); 367 void (*open)(struct trace_iterator *iter);
327 void (*pipe_open)(struct trace_iterator *iter); 368 void (*pipe_open)(struct trace_iterator *iter);
369 void (*wait_pipe)(struct trace_iterator *iter);
328 void (*close)(struct trace_iterator *iter); 370 void (*close)(struct trace_iterator *iter);
329 ssize_t (*read)(struct trace_iterator *iter, 371 ssize_t (*read)(struct trace_iterator *iter,
330 struct file *filp, char __user *ubuf, 372 struct file *filp, char __user *ubuf,
331 size_t cnt, loff_t *ppos); 373 size_t cnt, loff_t *ppos);
374 ssize_t (*splice_read)(struct trace_iterator *iter,
375 struct file *filp,
376 loff_t *ppos,
377 struct pipe_inode_info *pipe,
378 size_t len,
379 unsigned int flags);
332#ifdef CONFIG_FTRACE_STARTUP_TEST 380#ifdef CONFIG_FTRACE_STARTUP_TEST
333 int (*selftest)(struct tracer *trace, 381 int (*selftest)(struct tracer *trace,
334 struct trace_array *tr); 382 struct trace_array *tr);
@@ -339,7 +387,8 @@ struct tracer {
339 int (*set_flag)(u32 old_flags, u32 bit, int set); 387 int (*set_flag)(u32 old_flags, u32 bit, int set);
340 struct tracer *next; 388 struct tracer *next;
341 int print_max; 389 int print_max;
342 struct tracer_flags *flags; 390 struct tracer_flags *flags;
391 struct tracer_stat *stats;
343}; 392};
344 393
345struct trace_seq { 394struct trace_seq {
@@ -348,6 +397,16 @@ struct trace_seq {
348 unsigned int readpos; 397 unsigned int readpos;
349}; 398};
350 399
400static inline void
401trace_seq_init(struct trace_seq *s)
402{
403 s->len = 0;
404 s->readpos = 0;
405}
406
407
408#define TRACE_PIPE_ALL_CPU -1
409
351/* 410/*
352 * Trace iterator - used by printout routines who present trace 411 * Trace iterator - used by printout routines who present trace
353 * results to users and which routines might sleep, etc: 412 * results to users and which routines might sleep, etc:
@@ -356,6 +415,8 @@ struct trace_iterator {
356 struct trace_array *tr; 415 struct trace_array *tr;
357 struct tracer *trace; 416 struct tracer *trace;
358 void *private; 417 void *private;
418 int cpu_file;
419 struct mutex mutex;
359 struct ring_buffer_iter *buffer_iter[NR_CPUS]; 420 struct ring_buffer_iter *buffer_iter[NR_CPUS];
360 421
361 /* The below is zeroed out in pipe_read */ 422 /* The below is zeroed out in pipe_read */
@@ -371,6 +432,7 @@ struct trace_iterator {
371 cpumask_var_t started; 432 cpumask_var_t started;
372}; 433};
373 434
435int tracer_init(struct tracer *t, struct trace_array *tr);
374int tracing_is_enabled(void); 436int tracing_is_enabled(void);
375void trace_wake_up(void); 437void trace_wake_up(void);
376void tracing_reset(struct trace_array *tr, int cpu); 438void tracing_reset(struct trace_array *tr, int cpu);
@@ -379,26 +441,48 @@ int tracing_open_generic(struct inode *inode, struct file *filp);
379struct dentry *tracing_init_dentry(void); 441struct dentry *tracing_init_dentry(void);
380void init_tracer_sysprof_debugfs(struct dentry *d_tracer); 442void init_tracer_sysprof_debugfs(struct dentry *d_tracer);
381 443
444struct ring_buffer_event;
445
446struct ring_buffer_event *trace_buffer_lock_reserve(struct trace_array *tr,
447 unsigned char type,
448 unsigned long len,
449 unsigned long flags,
450 int pc);
451void trace_buffer_unlock_commit(struct trace_array *tr,
452 struct ring_buffer_event *event,
453 unsigned long flags, int pc);
454
455struct ring_buffer_event *
456trace_current_buffer_lock_reserve(unsigned char type, unsigned long len,
457 unsigned long flags, int pc);
458void trace_current_buffer_unlock_commit(struct ring_buffer_event *event,
459 unsigned long flags, int pc);
460
382struct trace_entry *tracing_get_trace_entry(struct trace_array *tr, 461struct trace_entry *tracing_get_trace_entry(struct trace_array *tr,
383 struct trace_array_cpu *data); 462 struct trace_array_cpu *data);
463
464struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
465 int *ent_cpu, u64 *ent_ts);
466
384void tracing_generic_entry_update(struct trace_entry *entry, 467void tracing_generic_entry_update(struct trace_entry *entry,
385 unsigned long flags, 468 unsigned long flags,
386 int pc); 469 int pc);
387 470
471void default_wait_pipe(struct trace_iterator *iter);
472void poll_wait_pipe(struct trace_iterator *iter);
473
388void ftrace(struct trace_array *tr, 474void ftrace(struct trace_array *tr,
389 struct trace_array_cpu *data, 475 struct trace_array_cpu *data,
390 unsigned long ip, 476 unsigned long ip,
391 unsigned long parent_ip, 477 unsigned long parent_ip,
392 unsigned long flags, int pc); 478 unsigned long flags, int pc);
393void tracing_sched_switch_trace(struct trace_array *tr, 479void tracing_sched_switch_trace(struct trace_array *tr,
394 struct trace_array_cpu *data,
395 struct task_struct *prev, 480 struct task_struct *prev,
396 struct task_struct *next, 481 struct task_struct *next,
397 unsigned long flags, int pc); 482 unsigned long flags, int pc);
398void tracing_record_cmdline(struct task_struct *tsk); 483void tracing_record_cmdline(struct task_struct *tsk);
399 484
400void tracing_sched_wakeup_trace(struct trace_array *tr, 485void tracing_sched_wakeup_trace(struct trace_array *tr,
401 struct trace_array_cpu *data,
402 struct task_struct *wakee, 486 struct task_struct *wakee,
403 struct task_struct *cur, 487 struct task_struct *cur,
404 unsigned long flags, int pc); 488 unsigned long flags, int pc);
@@ -408,14 +492,12 @@ void trace_special(struct trace_array *tr,
408 unsigned long arg2, 492 unsigned long arg2,
409 unsigned long arg3, int pc); 493 unsigned long arg3, int pc);
410void trace_function(struct trace_array *tr, 494void trace_function(struct trace_array *tr,
411 struct trace_array_cpu *data,
412 unsigned long ip, 495 unsigned long ip,
413 unsigned long parent_ip, 496 unsigned long parent_ip,
414 unsigned long flags, int pc); 497 unsigned long flags, int pc);
415 498
416void trace_graph_return(struct ftrace_graph_ret *trace); 499void trace_graph_return(struct ftrace_graph_ret *trace);
417int trace_graph_entry(struct ftrace_graph_ent *trace); 500int trace_graph_entry(struct ftrace_graph_ent *trace);
418void trace_hw_branch(struct trace_array *tr, u64 from, u64 to);
419 501
420void tracing_start_cmdline_record(void); 502void tracing_start_cmdline_record(void);
421void tracing_stop_cmdline_record(void); 503void tracing_stop_cmdline_record(void);
@@ -434,15 +516,11 @@ void update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu);
434void update_max_tr_single(struct trace_array *tr, 516void update_max_tr_single(struct trace_array *tr,
435 struct task_struct *tsk, int cpu); 517 struct task_struct *tsk, int cpu);
436 518
437extern cycle_t ftrace_now(int cpu); 519void __trace_stack(struct trace_array *tr,
520 unsigned long flags,
521 int skip, int pc);
438 522
439#ifdef CONFIG_FUNCTION_TRACER 523extern cycle_t ftrace_now(int cpu);
440void tracing_start_function_trace(void);
441void tracing_stop_function_trace(void);
442#else
443# define tracing_start_function_trace() do { } while (0)
444# define tracing_stop_function_trace() do { } while (0)
445#endif
446 524
447#ifdef CONFIG_CONTEXT_SWITCH_TRACER 525#ifdef CONFIG_CONTEXT_SWITCH_TRACER
448typedef void 526typedef void
@@ -456,10 +534,10 @@ struct tracer_switch_ops {
456 void *private; 534 void *private;
457 struct tracer_switch_ops *next; 535 struct tracer_switch_ops *next;
458}; 536};
459
460char *trace_find_cmdline(int pid);
461#endif /* CONFIG_CONTEXT_SWITCH_TRACER */ 537#endif /* CONFIG_CONTEXT_SWITCH_TRACER */
462 538
539extern char *trace_find_cmdline(int pid);
540
463#ifdef CONFIG_DYNAMIC_FTRACE 541#ifdef CONFIG_DYNAMIC_FTRACE
464extern unsigned long ftrace_update_tot_cnt; 542extern unsigned long ftrace_update_tot_cnt;
465#define DYN_FTRACE_TEST_NAME trace_selftest_dynamic_test_func 543#define DYN_FTRACE_TEST_NAME trace_selftest_dynamic_test_func
@@ -469,6 +547,8 @@ extern int DYN_FTRACE_TEST_NAME(void);
469#ifdef CONFIG_FTRACE_STARTUP_TEST 547#ifdef CONFIG_FTRACE_STARTUP_TEST
470extern int trace_selftest_startup_function(struct tracer *trace, 548extern int trace_selftest_startup_function(struct tracer *trace,
471 struct trace_array *tr); 549 struct trace_array *tr);
550extern int trace_selftest_startup_function_graph(struct tracer *trace,
551 struct trace_array *tr);
472extern int trace_selftest_startup_irqsoff(struct tracer *trace, 552extern int trace_selftest_startup_irqsoff(struct tracer *trace,
473 struct trace_array *tr); 553 struct trace_array *tr);
474extern int trace_selftest_startup_preemptoff(struct tracer *trace, 554extern int trace_selftest_startup_preemptoff(struct tracer *trace,
@@ -488,15 +568,6 @@ extern int trace_selftest_startup_branch(struct tracer *trace,
488#endif /* CONFIG_FTRACE_STARTUP_TEST */ 568#endif /* CONFIG_FTRACE_STARTUP_TEST */
489 569
490extern void *head_page(struct trace_array_cpu *data); 570extern void *head_page(struct trace_array_cpu *data);
491extern int trace_seq_printf(struct trace_seq *s, const char *fmt, ...);
492extern void trace_seq_print_cont(struct trace_seq *s,
493 struct trace_iterator *iter);
494
495extern int
496seq_print_ip_sym(struct trace_seq *s, unsigned long ip,
497 unsigned long sym_flags);
498extern ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf,
499 size_t cnt);
500extern long ns2usecs(cycle_t nsec); 571extern long ns2usecs(cycle_t nsec);
501extern int 572extern int
502trace_vprintk(unsigned long ip, int depth, const char *fmt, va_list args); 573trace_vprintk(unsigned long ip, int depth, const char *fmt, va_list args);
@@ -580,7 +651,9 @@ enum trace_iterator_flags {
580 TRACE_ITER_ANNOTATE = 0x2000, 651 TRACE_ITER_ANNOTATE = 0x2000,
581 TRACE_ITER_USERSTACKTRACE = 0x4000, 652 TRACE_ITER_USERSTACKTRACE = 0x4000,
582 TRACE_ITER_SYM_USEROBJ = 0x8000, 653 TRACE_ITER_SYM_USEROBJ = 0x8000,
583 TRACE_ITER_PRINTK_MSGONLY = 0x10000 654 TRACE_ITER_PRINTK_MSGONLY = 0x10000,
655 TRACE_ITER_CONTEXT_INFO = 0x20000, /* Print pid/cpu/time */
656 TRACE_ITER_LATENCY_FMT = 0x40000,
584}; 657};
585 658
586/* 659/*
@@ -601,12 +674,12 @@ extern struct tracer nop_trace;
601 * preempt_enable (after a disable), a schedule might take place 674 * preempt_enable (after a disable), a schedule might take place
602 * causing an infinite recursion. 675 * causing an infinite recursion.
603 * 676 *
604 * To prevent this, we read the need_recshed flag before 677 * To prevent this, we read the need_resched flag before
605 * disabling preemption. When we want to enable preemption we 678 * disabling preemption. When we want to enable preemption we
606 * check the flag, if it is set, then we call preempt_enable_no_resched. 679 * check the flag, if it is set, then we call preempt_enable_no_resched.
607 * Otherwise, we call preempt_enable. 680 * Otherwise, we call preempt_enable.
608 * 681 *
609 * The rational for doing the above is that if need resched is set 682 * The rational for doing the above is that if need_resched is set
610 * and we have yet to reschedule, we are either in an atomic location 683 * and we have yet to reschedule, we are either in an atomic location
611 * (where we do not need to check for scheduling) or we are inside 684 * (where we do not need to check for scheduling) or we are inside
612 * the scheduler and do not want to resched. 685 * the scheduler and do not want to resched.
@@ -627,7 +700,7 @@ static inline int ftrace_preempt_disable(void)
627 * 700 *
628 * This is a scheduler safe way to enable preemption and not miss 701 * This is a scheduler safe way to enable preemption and not miss
629 * any preemption checks. The disabled saved the state of preemption. 702 * any preemption checks. The disabled saved the state of preemption.
630 * If resched is set, then we were either inside an atomic or 703 * If resched is set, then we are either inside an atomic or
631 * are inside the scheduler (we would have already scheduled 704 * are inside the scheduler (we would have already scheduled
632 * otherwise). In this case, we do not want to call normal 705 * otherwise). In this case, we do not want to call normal
633 * preempt_enable, but preempt_enable_no_resched instead. 706 * preempt_enable, but preempt_enable_no_resched instead.
@@ -664,4 +737,26 @@ static inline void trace_branch_disable(void)
664} 737}
665#endif /* CONFIG_BRANCH_TRACER */ 738#endif /* CONFIG_BRANCH_TRACER */
666 739
740/* trace event type bit fields, not numeric */
741enum {
742 TRACE_EVENT_TYPE_PRINTF = 1,
743 TRACE_EVENT_TYPE_RAW = 2,
744};
745
746struct ftrace_event_call {
747 char *name;
748 char *system;
749 struct dentry *dir;
750 int enabled;
751 int (*regfunc)(void);
752 void (*unregfunc)(void);
753 int id;
754 int (*raw_init)(void);
755 int (*show_format)(struct trace_seq *s);
756};
757
758void event_trace_printk(unsigned long ip, const char *fmt, ...);
759extern struct ftrace_event_call __start_ftrace_events[];
760extern struct ftrace_event_call __stop_ftrace_events[];
761
667#endif /* _LINUX_KERNEL_TRACE_H */ 762#endif /* _LINUX_KERNEL_TRACE_H */
diff --git a/kernel/trace/trace_boot.c b/kernel/trace/trace_boot.c
index 366c8c333e13..7a30fc4c3642 100644
--- a/kernel/trace/trace_boot.c
+++ b/kernel/trace/trace_boot.c
@@ -11,6 +11,7 @@
11#include <linux/kallsyms.h> 11#include <linux/kallsyms.h>
12 12
13#include "trace.h" 13#include "trace.h"
14#include "trace_output.h"
14 15
15static struct trace_array *boot_trace; 16static struct trace_array *boot_trace;
16static bool pre_initcalls_finished; 17static bool pre_initcalls_finished;
@@ -27,13 +28,13 @@ void start_boot_trace(void)
27 28
28void enable_boot_trace(void) 29void enable_boot_trace(void)
29{ 30{
30 if (pre_initcalls_finished) 31 if (boot_trace && pre_initcalls_finished)
31 tracing_start_sched_switch_record(); 32 tracing_start_sched_switch_record();
32} 33}
33 34
34void disable_boot_trace(void) 35void disable_boot_trace(void)
35{ 36{
36 if (pre_initcalls_finished) 37 if (boot_trace && pre_initcalls_finished)
37 tracing_stop_sched_switch_record(); 38 tracing_stop_sched_switch_record();
38} 39}
39 40
@@ -42,6 +43,9 @@ static int boot_trace_init(struct trace_array *tr)
42 int cpu; 43 int cpu;
43 boot_trace = tr; 44 boot_trace = tr;
44 45
46 if (!tr)
47 return 0;
48
45 for_each_cpu(cpu, cpu_possible_mask) 49 for_each_cpu(cpu, cpu_possible_mask)
46 tracing_reset(tr, cpu); 50 tracing_reset(tr, cpu);
47 51
@@ -128,10 +132,9 @@ void trace_boot_call(struct boot_trace_call *bt, initcall_t fn)
128{ 132{
129 struct ring_buffer_event *event; 133 struct ring_buffer_event *event;
130 struct trace_boot_call *entry; 134 struct trace_boot_call *entry;
131 unsigned long irq_flags;
132 struct trace_array *tr = boot_trace; 135 struct trace_array *tr = boot_trace;
133 136
134 if (!pre_initcalls_finished) 137 if (!tr || !pre_initcalls_finished)
135 return; 138 return;
136 139
137 /* Get its name now since this function could 140 /* Get its name now since this function could
@@ -140,18 +143,13 @@ void trace_boot_call(struct boot_trace_call *bt, initcall_t fn)
140 sprint_symbol(bt->func, (unsigned long)fn); 143 sprint_symbol(bt->func, (unsigned long)fn);
141 preempt_disable(); 144 preempt_disable();
142 145
143 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry), 146 event = trace_buffer_lock_reserve(tr, TRACE_BOOT_CALL,
144 &irq_flags); 147 sizeof(*entry), 0, 0);
145 if (!event) 148 if (!event)
146 goto out; 149 goto out;
147 entry = ring_buffer_event_data(event); 150 entry = ring_buffer_event_data(event);
148 tracing_generic_entry_update(&entry->ent, 0, 0);
149 entry->ent.type = TRACE_BOOT_CALL;
150 entry->boot_call = *bt; 151 entry->boot_call = *bt;
151 ring_buffer_unlock_commit(tr->buffer, event, irq_flags); 152 trace_buffer_unlock_commit(tr, event, 0, 0);
152
153 trace_wake_up();
154
155 out: 153 out:
156 preempt_enable(); 154 preempt_enable();
157} 155}
@@ -160,27 +158,21 @@ void trace_boot_ret(struct boot_trace_ret *bt, initcall_t fn)
160{ 158{
161 struct ring_buffer_event *event; 159 struct ring_buffer_event *event;
162 struct trace_boot_ret *entry; 160 struct trace_boot_ret *entry;
163 unsigned long irq_flags;
164 struct trace_array *tr = boot_trace; 161 struct trace_array *tr = boot_trace;
165 162
166 if (!pre_initcalls_finished) 163 if (!tr || !pre_initcalls_finished)
167 return; 164 return;
168 165
169 sprint_symbol(bt->func, (unsigned long)fn); 166 sprint_symbol(bt->func, (unsigned long)fn);
170 preempt_disable(); 167 preempt_disable();
171 168
172 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry), 169 event = trace_buffer_lock_reserve(tr, TRACE_BOOT_RET,
173 &irq_flags); 170 sizeof(*entry), 0, 0);
174 if (!event) 171 if (!event)
175 goto out; 172 goto out;
176 entry = ring_buffer_event_data(event); 173 entry = ring_buffer_event_data(event);
177 tracing_generic_entry_update(&entry->ent, 0, 0);
178 entry->ent.type = TRACE_BOOT_RET;
179 entry->boot_ret = *bt; 174 entry->boot_ret = *bt;
180 ring_buffer_unlock_commit(tr->buffer, event, irq_flags); 175 trace_buffer_unlock_commit(tr, event, 0, 0);
181
182 trace_wake_up();
183
184 out: 176 out:
185 preempt_enable(); 177 preempt_enable();
186} 178}
diff --git a/kernel/trace/trace_branch.c b/kernel/trace/trace_branch.c
index 6c00feb3bac7..aaa0755268b9 100644
--- a/kernel/trace/trace_branch.c
+++ b/kernel/trace/trace_branch.c
@@ -14,12 +14,17 @@
14#include <linux/hash.h> 14#include <linux/hash.h>
15#include <linux/fs.h> 15#include <linux/fs.h>
16#include <asm/local.h> 16#include <asm/local.h>
17
17#include "trace.h" 18#include "trace.h"
19#include "trace_stat.h"
20#include "trace_output.h"
18 21
19#ifdef CONFIG_BRANCH_TRACER 22#ifdef CONFIG_BRANCH_TRACER
20 23
24static struct tracer branch_trace;
21static int branch_tracing_enabled __read_mostly; 25static int branch_tracing_enabled __read_mostly;
22static DEFINE_MUTEX(branch_tracing_mutex); 26static DEFINE_MUTEX(branch_tracing_mutex);
27
23static struct trace_array *branch_tracer; 28static struct trace_array *branch_tracer;
24 29
25static void 30static void
@@ -28,7 +33,7 @@ probe_likely_condition(struct ftrace_branch_data *f, int val, int expect)
28 struct trace_array *tr = branch_tracer; 33 struct trace_array *tr = branch_tracer;
29 struct ring_buffer_event *event; 34 struct ring_buffer_event *event;
30 struct trace_branch *entry; 35 struct trace_branch *entry;
31 unsigned long flags, irq_flags; 36 unsigned long flags;
32 int cpu, pc; 37 int cpu, pc;
33 const char *p; 38 const char *p;
34 39
@@ -47,15 +52,13 @@ probe_likely_condition(struct ftrace_branch_data *f, int val, int expect)
47 if (atomic_inc_return(&tr->data[cpu]->disabled) != 1) 52 if (atomic_inc_return(&tr->data[cpu]->disabled) != 1)
48 goto out; 53 goto out;
49 54
50 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry), 55 pc = preempt_count();
51 &irq_flags); 56 event = trace_buffer_lock_reserve(tr, TRACE_BRANCH,
57 sizeof(*entry), flags, pc);
52 if (!event) 58 if (!event)
53 goto out; 59 goto out;
54 60
55 pc = preempt_count();
56 entry = ring_buffer_event_data(event); 61 entry = ring_buffer_event_data(event);
57 tracing_generic_entry_update(&entry->ent, flags, pc);
58 entry->ent.type = TRACE_BRANCH;
59 62
60 /* Strip off the path, only save the file */ 63 /* Strip off the path, only save the file */
61 p = f->file + strlen(f->file); 64 p = f->file + strlen(f->file);
@@ -70,7 +73,7 @@ probe_likely_condition(struct ftrace_branch_data *f, int val, int expect)
70 entry->line = f->line; 73 entry->line = f->line;
71 entry->correct = val == expect; 74 entry->correct = val == expect;
72 75
73 ring_buffer_unlock_commit(tr->buffer, event, irq_flags); 76 ring_buffer_unlock_commit(tr->buffer, event);
74 77
75 out: 78 out:
76 atomic_dec(&tr->data[cpu]->disabled); 79 atomic_dec(&tr->data[cpu]->disabled);
@@ -88,8 +91,6 @@ void trace_likely_condition(struct ftrace_branch_data *f, int val, int expect)
88 91
89int enable_branch_tracing(struct trace_array *tr) 92int enable_branch_tracing(struct trace_array *tr)
90{ 93{
91 int ret = 0;
92
93 mutex_lock(&branch_tracing_mutex); 94 mutex_lock(&branch_tracing_mutex);
94 branch_tracer = tr; 95 branch_tracer = tr;
95 /* 96 /*
@@ -100,7 +101,7 @@ int enable_branch_tracing(struct trace_array *tr)
100 branch_tracing_enabled++; 101 branch_tracing_enabled++;
101 mutex_unlock(&branch_tracing_mutex); 102 mutex_unlock(&branch_tracing_mutex);
102 103
103 return ret; 104 return 0;
104} 105}
105 106
106void disable_branch_tracing(void) 107void disable_branch_tracing(void)
@@ -128,11 +129,6 @@ static void stop_branch_trace(struct trace_array *tr)
128 129
129static int branch_trace_init(struct trace_array *tr) 130static int branch_trace_init(struct trace_array *tr)
130{ 131{
131 int cpu;
132
133 for_each_online_cpu(cpu)
134 tracing_reset(tr, cpu);
135
136 start_branch_trace(tr); 132 start_branch_trace(tr);
137 return 0; 133 return 0;
138} 134}
@@ -142,22 +138,53 @@ static void branch_trace_reset(struct trace_array *tr)
142 stop_branch_trace(tr); 138 stop_branch_trace(tr);
143} 139}
144 140
145struct tracer branch_trace __read_mostly = 141static enum print_line_t trace_branch_print(struct trace_iterator *iter,
142 int flags)
143{
144 struct trace_branch *field;
145
146 trace_assign_type(field, iter->ent);
147
148 if (trace_seq_printf(&iter->seq, "[%s] %s:%s:%d\n",
149 field->correct ? " ok " : " MISS ",
150 field->func,
151 field->file,
152 field->line))
153 return TRACE_TYPE_PARTIAL_LINE;
154
155 return TRACE_TYPE_HANDLED;
156}
157
158
159static struct trace_event trace_branch_event = {
160 .type = TRACE_BRANCH,
161 .trace = trace_branch_print,
162};
163
164static struct tracer branch_trace __read_mostly =
146{ 165{
147 .name = "branch", 166 .name = "branch",
148 .init = branch_trace_init, 167 .init = branch_trace_init,
149 .reset = branch_trace_reset, 168 .reset = branch_trace_reset,
150#ifdef CONFIG_FTRACE_SELFTEST 169#ifdef CONFIG_FTRACE_SELFTEST
151 .selftest = trace_selftest_startup_branch, 170 .selftest = trace_selftest_startup_branch,
152#endif 171#endif /* CONFIG_FTRACE_SELFTEST */
153}; 172};
154 173
155__init static int init_branch_trace(void) 174__init static int init_branch_tracer(void)
156{ 175{
176 int ret;
177
178 ret = register_ftrace_event(&trace_branch_event);
179 if (!ret) {
180 printk(KERN_WARNING "Warning: could not register "
181 "branch events\n");
182 return 1;
183 }
157 return register_tracer(&branch_trace); 184 return register_tracer(&branch_trace);
158} 185}
186device_initcall(init_branch_tracer);
159 187
160device_initcall(init_branch_trace);
161#else 188#else
162static inline 189static inline
163void trace_likely_condition(struct ftrace_branch_data *f, int val, int expect) 190void trace_likely_condition(struct ftrace_branch_data *f, int val, int expect)
@@ -183,66 +210,39 @@ void ftrace_likely_update(struct ftrace_branch_data *f, int val, int expect)
183} 210}
184EXPORT_SYMBOL(ftrace_likely_update); 211EXPORT_SYMBOL(ftrace_likely_update);
185 212
186struct ftrace_pointer { 213extern unsigned long __start_annotated_branch_profile[];
187 void *start; 214extern unsigned long __stop_annotated_branch_profile[];
188 void *stop;
189 int hit;
190};
191 215
192static void * 216static int annotated_branch_stat_headers(struct seq_file *m)
193t_next(struct seq_file *m, void *v, loff_t *pos)
194{ 217{
195 const struct ftrace_pointer *f = m->private; 218 seq_printf(m, " correct incorrect %% ");
196 struct ftrace_branch_data *p = v; 219 seq_printf(m, " Function "
197 220 " File Line\n"
198 (*pos)++; 221 " ------- --------- - "
199 222 " -------- "
200 if (v == (void *)1) 223 " ---- ----\n");
201 return f->start; 224 return 0;
202
203 ++p;
204
205 if ((void *)p >= (void *)f->stop)
206 return NULL;
207
208 return p;
209} 225}
210 226
211static void *t_start(struct seq_file *m, loff_t *pos) 227static inline long get_incorrect_percent(struct ftrace_branch_data *p)
212{ 228{
213 void *t = (void *)1; 229 long percent;
214 loff_t l = 0;
215
216 for (; t && l < *pos; t = t_next(m, t, &l))
217 ;
218 230
219 return t; 231 if (p->correct) {
220} 232 percent = p->incorrect * 100;
233 percent /= p->correct + p->incorrect;
234 } else
235 percent = p->incorrect ? 100 : -1;
221 236
222static void t_stop(struct seq_file *m, void *p) 237 return percent;
223{
224} 238}
225 239
226static int t_show(struct seq_file *m, void *v) 240static int branch_stat_show(struct seq_file *m, void *v)
227{ 241{
228 const struct ftrace_pointer *fp = m->private;
229 struct ftrace_branch_data *p = v; 242 struct ftrace_branch_data *p = v;
230 const char *f; 243 const char *f;
231 long percent; 244 long percent;
232 245
233 if (v == (void *)1) {
234 if (fp->hit)
235 seq_printf(m, " miss hit %% ");
236 else
237 seq_printf(m, " correct incorrect %% ");
238 seq_printf(m, " Function "
239 " File Line\n"
240 " ------- --------- - "
241 " -------- "
242 " ---- ----\n");
243 return 0;
244 }
245
246 /* Only print the file, not the path */ 246 /* Only print the file, not the path */
247 f = p->file + strlen(p->file); 247 f = p->file + strlen(p->file);
248 while (f >= p->file && *f != '/') 248 while (f >= p->file && *f != '/')
@@ -252,11 +252,7 @@ static int t_show(struct seq_file *m, void *v)
252 /* 252 /*
253 * The miss is overlayed on correct, and hit on incorrect. 253 * The miss is overlayed on correct, and hit on incorrect.
254 */ 254 */
255 if (p->correct) { 255 percent = get_incorrect_percent(p);
256 percent = p->incorrect * 100;
257 percent /= p->correct + p->incorrect;
258 } else
259 percent = p->incorrect ? 100 : -1;
260 256
261 seq_printf(m, "%8lu %8lu ", p->correct, p->incorrect); 257 seq_printf(m, "%8lu %8lu ", p->correct, p->incorrect);
262 if (percent < 0) 258 if (percent < 0)
@@ -267,76 +263,118 @@ static int t_show(struct seq_file *m, void *v)
267 return 0; 263 return 0;
268} 264}
269 265
270static struct seq_operations tracing_likely_seq_ops = { 266static void *annotated_branch_stat_start(void)
271 .start = t_start, 267{
272 .next = t_next, 268 return __start_annotated_branch_profile;
273 .stop = t_stop, 269}
274 .show = t_show, 270
271static void *
272annotated_branch_stat_next(void *v, int idx)
273{
274 struct ftrace_branch_data *p = v;
275
276 ++p;
277
278 if ((void *)p >= (void *)__stop_annotated_branch_profile)
279 return NULL;
280
281 return p;
282}
283
284static int annotated_branch_stat_cmp(void *p1, void *p2)
285{
286 struct ftrace_branch_data *a = p1;
287 struct ftrace_branch_data *b = p2;
288
289 long percent_a, percent_b;
290
291 percent_a = get_incorrect_percent(a);
292 percent_b = get_incorrect_percent(b);
293
294 if (percent_a < percent_b)
295 return -1;
296 if (percent_a > percent_b)
297 return 1;
298 else
299 return 0;
300}
301
302static struct tracer_stat annotated_branch_stats = {
303 .name = "branch_annotated",
304 .stat_start = annotated_branch_stat_start,
305 .stat_next = annotated_branch_stat_next,
306 .stat_cmp = annotated_branch_stat_cmp,
307 .stat_headers = annotated_branch_stat_headers,
308 .stat_show = branch_stat_show
275}; 309};
276 310
277static int tracing_branch_open(struct inode *inode, struct file *file) 311__init static int init_annotated_branch_stats(void)
278{ 312{
279 int ret; 313 int ret;
280 314
281 ret = seq_open(file, &tracing_likely_seq_ops); 315 ret = register_stat_tracer(&annotated_branch_stats);
282 if (!ret) { 316 if (!ret) {
283 struct seq_file *m = file->private_data; 317 printk(KERN_WARNING "Warning: could not register "
284 m->private = (void *)inode->i_private; 318 "annotated branches stats\n");
319 return 1;
285 } 320 }
286 321 return 0;
287 return ret;
288} 322}
289 323fs_initcall(init_annotated_branch_stats);
290static const struct file_operations tracing_branch_fops = {
291 .open = tracing_branch_open,
292 .read = seq_read,
293 .llseek = seq_lseek,
294};
295 324
296#ifdef CONFIG_PROFILE_ALL_BRANCHES 325#ifdef CONFIG_PROFILE_ALL_BRANCHES
326
297extern unsigned long __start_branch_profile[]; 327extern unsigned long __start_branch_profile[];
298extern unsigned long __stop_branch_profile[]; 328extern unsigned long __stop_branch_profile[];
299 329
300static const struct ftrace_pointer ftrace_branch_pos = { 330static int all_branch_stat_headers(struct seq_file *m)
301 .start = __start_branch_profile, 331{
302 .stop = __stop_branch_profile, 332 seq_printf(m, " miss hit %% ");
303 .hit = 1, 333 seq_printf(m, " Function "
304}; 334 " File Line\n"
335 " ------- --------- - "
336 " -------- "
337 " ---- ----\n");
338 return 0;
339}
305 340
306#endif /* CONFIG_PROFILE_ALL_BRANCHES */ 341static void *all_branch_stat_start(void)
342{
343 return __start_branch_profile;
344}
307 345
308extern unsigned long __start_annotated_branch_profile[]; 346static void *
309extern unsigned long __stop_annotated_branch_profile[]; 347all_branch_stat_next(void *v, int idx)
348{
349 struct ftrace_branch_data *p = v;
310 350
311static const struct ftrace_pointer ftrace_annotated_branch_pos = { 351 ++p;
312 .start = __start_annotated_branch_profile,
313 .stop = __stop_annotated_branch_profile,
314};
315 352
316static __init int ftrace_branch_init(void) 353 if ((void *)p >= (void *)__stop_branch_profile)
317{ 354 return NULL;
318 struct dentry *d_tracer;
319 struct dentry *entry;
320 355
321 d_tracer = tracing_init_dentry(); 356 return p;
357}
322 358
323 entry = debugfs_create_file("profile_annotated_branch", 0444, d_tracer, 359static struct tracer_stat all_branch_stats = {
324 (void *)&ftrace_annotated_branch_pos, 360 .name = "branch_all",
325 &tracing_branch_fops); 361 .stat_start = all_branch_stat_start,
326 if (!entry) 362 .stat_next = all_branch_stat_next,
327 pr_warning("Could not create debugfs " 363 .stat_headers = all_branch_stat_headers,
328 "'profile_annotatet_branch' entry\n"); 364 .stat_show = branch_stat_show
365};
329 366
330#ifdef CONFIG_PROFILE_ALL_BRANCHES 367__init static int all_annotated_branch_stats(void)
331 entry = debugfs_create_file("profile_branch", 0444, d_tracer, 368{
332 (void *)&ftrace_branch_pos, 369 int ret;
333 &tracing_branch_fops);
334 if (!entry)
335 pr_warning("Could not create debugfs"
336 " 'profile_branch' entry\n");
337#endif
338 370
371 ret = register_stat_tracer(&all_branch_stats);
372 if (!ret) {
373 printk(KERN_WARNING "Warning: could not register "
374 "all branches stats\n");
375 return 1;
376 }
339 return 0; 377 return 0;
340} 378}
341 379fs_initcall(all_annotated_branch_stats);
342device_initcall(ftrace_branch_init); 380#endif /* CONFIG_PROFILE_ALL_BRANCHES */
diff --git a/kernel/trace/trace_clock.c b/kernel/trace/trace_clock.c
new file mode 100644
index 000000000000..05b176abfd30
--- /dev/null
+++ b/kernel/trace/trace_clock.c
@@ -0,0 +1,108 @@
1/*
2 * tracing clocks
3 *
4 * Copyright (C) 2009 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
5 *
6 * Implements 3 trace clock variants, with differing scalability/precision
7 * tradeoffs:
8 *
9 * - local: CPU-local trace clock
10 * - medium: scalable global clock with some jitter
11 * - global: globally monotonic, serialized clock
12 *
13 * Tracer plugins will chose a default from these clocks.
14 */
15#include <linux/spinlock.h>
16#include <linux/hardirq.h>
17#include <linux/module.h>
18#include <linux/percpu.h>
19#include <linux/sched.h>
20#include <linux/ktime.h>
21
22/*
23 * trace_clock_local(): the simplest and least coherent tracing clock.
24 *
25 * Useful for tracing that does not cross to other CPUs nor
26 * does it go through idle events.
27 */
28u64 notrace trace_clock_local(void)
29{
30 unsigned long flags;
31 u64 clock;
32
33 /*
34 * sched_clock() is an architecture implemented, fast, scalable,
35 * lockless clock. It is not guaranteed to be coherent across
36 * CPUs, nor across CPU idle events.
37 */
38 raw_local_irq_save(flags);
39 clock = sched_clock();
40 raw_local_irq_restore(flags);
41
42 return clock;
43}
44
45/*
46 * trace_clock(): 'inbetween' trace clock. Not completely serialized,
47 * but not completely incorrect when crossing CPUs either.
48 *
49 * This is based on cpu_clock(), which will allow at most ~1 jiffy of
50 * jitter between CPUs. So it's a pretty scalable clock, but there
51 * can be offsets in the trace data.
52 */
53u64 notrace trace_clock(void)
54{
55 return cpu_clock(raw_smp_processor_id());
56}
57
58
59/*
60 * trace_clock_global(): special globally coherent trace clock
61 *
62 * It has higher overhead than the other trace clocks but is still
63 * an order of magnitude faster than GTOD derived hardware clocks.
64 *
65 * Used by plugins that need globally coherent timestamps.
66 */
67
68static u64 prev_trace_clock_time;
69
70static raw_spinlock_t trace_clock_lock ____cacheline_aligned_in_smp =
71 (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
72
73u64 notrace trace_clock_global(void)
74{
75 unsigned long flags;
76 int this_cpu;
77 u64 now;
78
79 raw_local_irq_save(flags);
80
81 this_cpu = raw_smp_processor_id();
82 now = cpu_clock(this_cpu);
83 /*
84 * If in an NMI context then dont risk lockups and return the
85 * cpu_clock() time:
86 */
87 if (unlikely(in_nmi()))
88 goto out;
89
90 __raw_spin_lock(&trace_clock_lock);
91
92 /*
93 * TODO: if this happens often then maybe we should reset
94 * my_scd->clock to prev_trace_clock_time+1, to make sure
95 * we start ticking with the local clock from now on?
96 */
97 if ((s64)(now - prev_trace_clock_time) < 0)
98 now = prev_trace_clock_time + 1;
99
100 prev_trace_clock_time = now;
101
102 __raw_spin_unlock(&trace_clock_lock);
103
104 out:
105 raw_local_irq_restore(flags);
106
107 return now;
108}
diff --git a/kernel/trace/trace_event_types.h b/kernel/trace/trace_event_types.h
new file mode 100644
index 000000000000..5cca4c978bde
--- /dev/null
+++ b/kernel/trace/trace_event_types.h
@@ -0,0 +1,166 @@
1#undef TRACE_SYSTEM
2#define TRACE_SYSTEM ftrace
3
4/*
5 * We cheat and use the proto type field as the ID
6 * and args as the entry type (minus 'struct')
7 */
8TRACE_EVENT_FORMAT(function, TRACE_FN, ftrace_entry, ignore,
9 TRACE_STRUCT(
10 TRACE_FIELD(unsigned long, ip, ip)
11 TRACE_FIELD(unsigned long, parent_ip, parent_ip)
12 ),
13 TP_RAW_FMT(" %lx <-- %lx")
14);
15
16TRACE_EVENT_FORMAT(funcgraph_entry, TRACE_GRAPH_ENT,
17 ftrace_graph_ent_entry, ignore,
18 TRACE_STRUCT(
19 TRACE_FIELD(unsigned long, graph_ent.func, func)
20 TRACE_FIELD(int, graph_ent.depth, depth)
21 ),
22 TP_RAW_FMT("--> %lx (%d)")
23);
24
25TRACE_EVENT_FORMAT(funcgraph_exit, TRACE_GRAPH_RET,
26 ftrace_graph_ret_entry, ignore,
27 TRACE_STRUCT(
28 TRACE_FIELD(unsigned long, ret.func, func)
29 TRACE_FIELD(int, ret.depth, depth)
30 ),
31 TP_RAW_FMT("<-- %lx (%d)")
32);
33
34TRACE_EVENT_FORMAT(wakeup, TRACE_WAKE, ctx_switch_entry, ignore,
35 TRACE_STRUCT(
36 TRACE_FIELD(unsigned int, prev_pid, prev_pid)
37 TRACE_FIELD(unsigned char, prev_prio, prev_prio)
38 TRACE_FIELD(unsigned char, prev_state, prev_state)
39 TRACE_FIELD(unsigned int, next_pid, next_pid)
40 TRACE_FIELD(unsigned char, next_prio, next_prio)
41 TRACE_FIELD(unsigned char, next_state, next_state)
42 TRACE_FIELD(unsigned int, next_cpu, next_cpu)
43 ),
44 TP_RAW_FMT("%u:%u:%u ==+ %u:%u:%u [%03u]")
45);
46
47TRACE_EVENT_FORMAT(context_switch, TRACE_CTX, ctx_switch_entry, ignore,
48 TRACE_STRUCT(
49 TRACE_FIELD(unsigned int, prev_pid, prev_pid)
50 TRACE_FIELD(unsigned char, prev_prio, prev_prio)
51 TRACE_FIELD(unsigned char, prev_state, prev_state)
52 TRACE_FIELD(unsigned int, next_pid, next_pid)
53 TRACE_FIELD(unsigned char, next_prio, next_prio)
54 TRACE_FIELD(unsigned char, next_state, next_state)
55 TRACE_FIELD(unsigned int, next_cpu, next_cpu)
56 ),
57 TP_RAW_FMT("%u:%u:%u ==+ %u:%u:%u [%03u]")
58);
59
60TRACE_EVENT_FORMAT(special, TRACE_SPECIAL, special_entry, ignore,
61 TRACE_STRUCT(
62 TRACE_FIELD(unsigned long, arg1, arg1)
63 TRACE_FIELD(unsigned long, arg2, arg2)
64 TRACE_FIELD(unsigned long, arg3, arg3)
65 ),
66 TP_RAW_FMT("(%08lx) (%08lx) (%08lx)")
67);
68
69/*
70 * Stack-trace entry:
71 */
72
73/* #define FTRACE_STACK_ENTRIES 8 */
74
75TRACE_EVENT_FORMAT(kernel_stack, TRACE_STACK, stack_entry, ignore,
76 TRACE_STRUCT(
77 TRACE_FIELD(unsigned long, caller[0], stack0)
78 TRACE_FIELD(unsigned long, caller[1], stack1)
79 TRACE_FIELD(unsigned long, caller[2], stack2)
80 TRACE_FIELD(unsigned long, caller[3], stack3)
81 TRACE_FIELD(unsigned long, caller[4], stack4)
82 TRACE_FIELD(unsigned long, caller[5], stack5)
83 TRACE_FIELD(unsigned long, caller[6], stack6)
84 TRACE_FIELD(unsigned long, caller[7], stack7)
85 ),
86 TP_RAW_FMT("\t=> (%08lx)\n\t=> (%08lx)\n\t=> (%08lx)\n\t=> (%08lx)\n"
87 "\t=> (%08lx)\n\t=> (%08lx)\n\t=> (%08lx)\n\t=> (%08lx)\n")
88);
89
90TRACE_EVENT_FORMAT(user_stack, TRACE_USER_STACK, userstack_entry, ignore,
91 TRACE_STRUCT(
92 TRACE_FIELD(unsigned long, caller[0], stack0)
93 TRACE_FIELD(unsigned long, caller[1], stack1)
94 TRACE_FIELD(unsigned long, caller[2], stack2)
95 TRACE_FIELD(unsigned long, caller[3], stack3)
96 TRACE_FIELD(unsigned long, caller[4], stack4)
97 TRACE_FIELD(unsigned long, caller[5], stack5)
98 TRACE_FIELD(unsigned long, caller[6], stack6)
99 TRACE_FIELD(unsigned long, caller[7], stack7)
100 ),
101 TP_RAW_FMT("\t=> (%08lx)\n\t=> (%08lx)\n\t=> (%08lx)\n\t=> (%08lx)\n"
102 "\t=> (%08lx)\n\t=> (%08lx)\n\t=> (%08lx)\n\t=> (%08lx)\n")
103);
104
105TRACE_EVENT_FORMAT(print, TRACE_PRINT, print_entry, ignore,
106 TRACE_STRUCT(
107 TRACE_FIELD(unsigned long, ip, ip)
108 TRACE_FIELD(unsigned int, depth, depth)
109 TRACE_FIELD(char *, fmt, fmt)
110 TRACE_FIELD_ZERO_CHAR(buf)
111 ),
112 TP_RAW_FMT("%08lx (%d) fmt:%p %s")
113);
114
115TRACE_EVENT_FORMAT(branch, TRACE_BRANCH, trace_branch, ignore,
116 TRACE_STRUCT(
117 TRACE_FIELD(unsigned int, line, line)
118 TRACE_FIELD_SPECIAL(char func[TRACE_FUNC_SIZE+1], func, func)
119 TRACE_FIELD_SPECIAL(char file[TRACE_FUNC_SIZE+1], file, file)
120 TRACE_FIELD(char, correct, correct)
121 ),
122 TP_RAW_FMT("%u:%s:%s (%u)")
123);
124
125TRACE_EVENT_FORMAT(hw_branch, TRACE_HW_BRANCHES, hw_branch_entry, ignore,
126 TRACE_STRUCT(
127 TRACE_FIELD(u64, from, from)
128 TRACE_FIELD(u64, to, to)
129 ),
130 TP_RAW_FMT("from: %llx to: %llx")
131);
132
133TRACE_EVENT_FORMAT(power, TRACE_POWER, trace_power, ignore,
134 TRACE_STRUCT(
135 TRACE_FIELD(ktime_t, state_data.stamp, stamp)
136 TRACE_FIELD(ktime_t, state_data.end, end)
137 TRACE_FIELD(int, state_data.type, type)
138 TRACE_FIELD(int, state_data.state, state)
139 ),
140 TP_RAW_FMT("%llx->%llx type:%u state:%u")
141);
142
143TRACE_EVENT_FORMAT(kmem_alloc, TRACE_KMEM_ALLOC, kmemtrace_alloc_entry, ignore,
144 TRACE_STRUCT(
145 TRACE_FIELD(enum kmemtrace_type_id, type_id, type_id)
146 TRACE_FIELD(unsigned long, call_site, call_site)
147 TRACE_FIELD(const void *, ptr, ptr)
148 TRACE_FIELD(size_t, bytes_req, bytes_req)
149 TRACE_FIELD(size_t, bytes_alloc, bytes_alloc)
150 TRACE_FIELD(gfp_t, gfp_flags, gfp_flags)
151 TRACE_FIELD(int, node, node)
152 ),
153 TP_RAW_FMT("type:%u call_site:%lx ptr:%p req:%lu alloc:%lu"
154 " flags:%x node:%d")
155);
156
157TRACE_EVENT_FORMAT(kmem_free, TRACE_KMEM_FREE, kmemtrace_free_entry, ignore,
158 TRACE_STRUCT(
159 TRACE_FIELD(enum kmemtrace_type_id, type_id, type_id)
160 TRACE_FIELD(unsigned long, call_site, call_site)
161 TRACE_FIELD(const void *, ptr, ptr)
162 ),
163 TP_RAW_FMT("type:%u call_site:%lx ptr:%p")
164);
165
166#undef TRACE_SYSTEM
diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c
new file mode 100644
index 000000000000..1880a6438097
--- /dev/null
+++ b/kernel/trace/trace_events.c
@@ -0,0 +1,581 @@
1/*
2 * event tracer
3 *
4 * Copyright (C) 2008 Red Hat Inc, Steven Rostedt <srostedt@redhat.com>
5 *
6 * - Added format output of fields of the trace point.
7 * This was based off of work by Tom Zanussi <tzanussi@gmail.com>.
8 *
9 */
10
11#include <linux/debugfs.h>
12#include <linux/uaccess.h>
13#include <linux/module.h>
14#include <linux/ctype.h>
15
16#include "trace_output.h"
17
18#define TRACE_SYSTEM "TRACE_SYSTEM"
19
20static DEFINE_MUTEX(event_mutex);
21
22#define events_for_each(event) \
23 for (event = __start_ftrace_events; \
24 (unsigned long)event < (unsigned long)__stop_ftrace_events; \
25 event++)
26
27void event_trace_printk(unsigned long ip, const char *fmt, ...)
28{
29 va_list ap;
30
31 va_start(ap, fmt);
32 tracing_record_cmdline(current);
33 trace_vprintk(ip, task_curr_ret_stack(current), fmt, ap);
34 va_end(ap);
35}
36
37static void ftrace_clear_events(void)
38{
39 struct ftrace_event_call *call = (void *)__start_ftrace_events;
40
41
42 while ((unsigned long)call < (unsigned long)__stop_ftrace_events) {
43
44 if (call->enabled) {
45 call->enabled = 0;
46 call->unregfunc();
47 }
48 call++;
49 }
50}
51
52static void ftrace_event_enable_disable(struct ftrace_event_call *call,
53 int enable)
54{
55
56 switch (enable) {
57 case 0:
58 if (call->enabled) {
59 call->enabled = 0;
60 call->unregfunc();
61 }
62 break;
63 case 1:
64 if (!call->enabled) {
65 call->enabled = 1;
66 call->regfunc();
67 }
68 break;
69 }
70}
71
72static int ftrace_set_clr_event(char *buf, int set)
73{
74 struct ftrace_event_call *call = __start_ftrace_events;
75 char *event = NULL, *sub = NULL, *match;
76 int ret = -EINVAL;
77
78 /*
79 * The buf format can be <subsystem>:<event-name>
80 * *:<event-name> means any event by that name.
81 * :<event-name> is the same.
82 *
83 * <subsystem>:* means all events in that subsystem
84 * <subsystem>: means the same.
85 *
86 * <name> (no ':') means all events in a subsystem with
87 * the name <name> or any event that matches <name>
88 */
89
90 match = strsep(&buf, ":");
91 if (buf) {
92 sub = match;
93 event = buf;
94 match = NULL;
95
96 if (!strlen(sub) || strcmp(sub, "*") == 0)
97 sub = NULL;
98 if (!strlen(event) || strcmp(event, "*") == 0)
99 event = NULL;
100 }
101
102 mutex_lock(&event_mutex);
103 events_for_each(call) {
104
105 if (!call->name)
106 continue;
107
108 if (match &&
109 strcmp(match, call->name) != 0 &&
110 strcmp(match, call->system) != 0)
111 continue;
112
113 if (sub && strcmp(sub, call->system) != 0)
114 continue;
115
116 if (event && strcmp(event, call->name) != 0)
117 continue;
118
119 ftrace_event_enable_disable(call, set);
120
121 ret = 0;
122 }
123 mutex_unlock(&event_mutex);
124
125 return ret;
126}
127
128/* 128 should be much more than enough */
129#define EVENT_BUF_SIZE 127
130
131static ssize_t
132ftrace_event_write(struct file *file, const char __user *ubuf,
133 size_t cnt, loff_t *ppos)
134{
135 size_t read = 0;
136 int i, set = 1;
137 ssize_t ret;
138 char *buf;
139 char ch;
140
141 if (!cnt || cnt < 0)
142 return 0;
143
144 ret = get_user(ch, ubuf++);
145 if (ret)
146 return ret;
147 read++;
148 cnt--;
149
150 /* skip white space */
151 while (cnt && isspace(ch)) {
152 ret = get_user(ch, ubuf++);
153 if (ret)
154 return ret;
155 read++;
156 cnt--;
157 }
158
159 /* Only white space found? */
160 if (isspace(ch)) {
161 file->f_pos += read;
162 ret = read;
163 return ret;
164 }
165
166 buf = kmalloc(EVENT_BUF_SIZE+1, GFP_KERNEL);
167 if (!buf)
168 return -ENOMEM;
169
170 if (cnt > EVENT_BUF_SIZE)
171 cnt = EVENT_BUF_SIZE;
172
173 i = 0;
174 while (cnt && !isspace(ch)) {
175 if (!i && ch == '!')
176 set = 0;
177 else
178 buf[i++] = ch;
179
180 ret = get_user(ch, ubuf++);
181 if (ret)
182 goto out_free;
183 read++;
184 cnt--;
185 }
186 buf[i] = 0;
187
188 file->f_pos += read;
189
190 ret = ftrace_set_clr_event(buf, set);
191 if (ret)
192 goto out_free;
193
194 ret = read;
195
196 out_free:
197 kfree(buf);
198
199 return ret;
200}
201
202static void *
203t_next(struct seq_file *m, void *v, loff_t *pos)
204{
205 struct ftrace_event_call *call = m->private;
206 struct ftrace_event_call *next = call;
207
208 (*pos)++;
209
210 if ((unsigned long)call >= (unsigned long)__stop_ftrace_events)
211 return NULL;
212
213 m->private = ++next;
214
215 return call;
216}
217
218static void *t_start(struct seq_file *m, loff_t *pos)
219{
220 return t_next(m, NULL, pos);
221}
222
223static void *
224s_next(struct seq_file *m, void *v, loff_t *pos)
225{
226 struct ftrace_event_call *call = m->private;
227 struct ftrace_event_call *next;
228
229 (*pos)++;
230
231 retry:
232 if ((unsigned long)call >= (unsigned long)__stop_ftrace_events)
233 return NULL;
234
235 if (!call->enabled) {
236 call++;
237 goto retry;
238 }
239
240 next = call;
241 m->private = ++next;
242
243 return call;
244}
245
246static void *s_start(struct seq_file *m, loff_t *pos)
247{
248 return s_next(m, NULL, pos);
249}
250
251static int t_show(struct seq_file *m, void *v)
252{
253 struct ftrace_event_call *call = v;
254
255 if (strcmp(call->system, TRACE_SYSTEM) != 0)
256 seq_printf(m, "%s:", call->system);
257 seq_printf(m, "%s\n", call->name);
258
259 return 0;
260}
261
262static void t_stop(struct seq_file *m, void *p)
263{
264}
265
266static int
267ftrace_event_seq_open(struct inode *inode, struct file *file)
268{
269 int ret;
270 const struct seq_operations *seq_ops;
271
272 if ((file->f_mode & FMODE_WRITE) &&
273 !(file->f_flags & O_APPEND))
274 ftrace_clear_events();
275
276 seq_ops = inode->i_private;
277 ret = seq_open(file, seq_ops);
278 if (!ret) {
279 struct seq_file *m = file->private_data;
280
281 m->private = __start_ftrace_events;
282 }
283 return ret;
284}
285
286static ssize_t
287event_enable_read(struct file *filp, char __user *ubuf, size_t cnt,
288 loff_t *ppos)
289{
290 struct ftrace_event_call *call = filp->private_data;
291 char *buf;
292
293 if (call->enabled)
294 buf = "1\n";
295 else
296 buf = "0\n";
297
298 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
299}
300
301static ssize_t
302event_enable_write(struct file *filp, const char __user *ubuf, size_t cnt,
303 loff_t *ppos)
304{
305 struct ftrace_event_call *call = filp->private_data;
306 char buf[64];
307 unsigned long val;
308 int ret;
309
310 if (cnt >= sizeof(buf))
311 return -EINVAL;
312
313 if (copy_from_user(&buf, ubuf, cnt))
314 return -EFAULT;
315
316 buf[cnt] = 0;
317
318 ret = strict_strtoul(buf, 10, &val);
319 if (ret < 0)
320 return ret;
321
322 switch (val) {
323 case 0:
324 case 1:
325 mutex_lock(&event_mutex);
326 ftrace_event_enable_disable(call, val);
327 mutex_unlock(&event_mutex);
328 break;
329
330 default:
331 return -EINVAL;
332 }
333
334 *ppos += cnt;
335
336 return cnt;
337}
338
339#undef FIELD
340#define FIELD(type, name) \
341 #type, #name, (unsigned int)offsetof(typeof(field), name), \
342 (unsigned int)sizeof(field.name)
343
344static int trace_write_header(struct trace_seq *s)
345{
346 struct trace_entry field;
347
348 /* struct trace_entry */
349 return trace_seq_printf(s,
350 "\tfield:%s %s;\toffset:%u;\tsize:%u;\n"
351 "\tfield:%s %s;\toffset:%u;\tsize:%u;\n"
352 "\tfield:%s %s;\toffset:%u;\tsize:%u;\n"
353 "\tfield:%s %s;\toffset:%u;\tsize:%u;\n"
354 "\tfield:%s %s;\toffset:%u;\tsize:%u;\n"
355 "\n",
356 FIELD(unsigned char, type),
357 FIELD(unsigned char, flags),
358 FIELD(unsigned char, preempt_count),
359 FIELD(int, pid),
360 FIELD(int, tgid));
361}
362
363static ssize_t
364event_format_read(struct file *filp, char __user *ubuf, size_t cnt,
365 loff_t *ppos)
366{
367 struct ftrace_event_call *call = filp->private_data;
368 struct trace_seq *s;
369 char *buf;
370 int r;
371
372 s = kmalloc(sizeof(*s), GFP_KERNEL);
373 if (!s)
374 return -ENOMEM;
375
376 trace_seq_init(s);
377
378 if (*ppos)
379 return 0;
380
381 /* If any of the first writes fail, so will the show_format. */
382
383 trace_seq_printf(s, "name: %s\n", call->name);
384 trace_seq_printf(s, "ID: %d\n", call->id);
385 trace_seq_printf(s, "format:\n");
386 trace_write_header(s);
387
388 r = call->show_format(s);
389 if (!r) {
390 /*
391 * ug! The format output is bigger than a PAGE!!
392 */
393 buf = "FORMAT TOO BIG\n";
394 r = simple_read_from_buffer(ubuf, cnt, ppos,
395 buf, strlen(buf));
396 goto out;
397 }
398
399 r = simple_read_from_buffer(ubuf, cnt, ppos,
400 s->buffer, s->len);
401 out:
402 kfree(s);
403 return r;
404}
405
406static const struct seq_operations show_event_seq_ops = {
407 .start = t_start,
408 .next = t_next,
409 .show = t_show,
410 .stop = t_stop,
411};
412
413static const struct seq_operations show_set_event_seq_ops = {
414 .start = s_start,
415 .next = s_next,
416 .show = t_show,
417 .stop = t_stop,
418};
419
420static const struct file_operations ftrace_set_event_fops = {
421 .open = ftrace_event_seq_open,
422 .read = seq_read,
423 .write = ftrace_event_write,
424 .llseek = seq_lseek,
425 .release = seq_release,
426};
427
428static const struct file_operations ftrace_enable_fops = {
429 .open = tracing_open_generic,
430 .read = event_enable_read,
431 .write = event_enable_write,
432};
433
434static const struct file_operations ftrace_event_format_fops = {
435 .open = tracing_open_generic,
436 .read = event_format_read,
437};
438
439static struct dentry *event_trace_events_dir(void)
440{
441 static struct dentry *d_tracer;
442 static struct dentry *d_events;
443
444 if (d_events)
445 return d_events;
446
447 d_tracer = tracing_init_dentry();
448 if (!d_tracer)
449 return NULL;
450
451 d_events = debugfs_create_dir("events", d_tracer);
452 if (!d_events)
453 pr_warning("Could not create debugfs "
454 "'events' directory\n");
455
456 return d_events;
457}
458
459struct event_subsystem {
460 struct list_head list;
461 const char *name;
462 struct dentry *entry;
463};
464
465static LIST_HEAD(event_subsystems);
466
467static struct dentry *
468event_subsystem_dir(const char *name, struct dentry *d_events)
469{
470 struct event_subsystem *system;
471
472 /* First see if we did not already create this dir */
473 list_for_each_entry(system, &event_subsystems, list) {
474 if (strcmp(system->name, name) == 0)
475 return system->entry;
476 }
477
478 /* need to create new entry */
479 system = kmalloc(sizeof(*system), GFP_KERNEL);
480 if (!system) {
481 pr_warning("No memory to create event subsystem %s\n",
482 name);
483 return d_events;
484 }
485
486 system->entry = debugfs_create_dir(name, d_events);
487 if (!system->entry) {
488 pr_warning("Could not create event subsystem %s\n",
489 name);
490 kfree(system);
491 return d_events;
492 }
493
494 system->name = name;
495 list_add(&system->list, &event_subsystems);
496
497 return system->entry;
498}
499
500static int
501event_create_dir(struct ftrace_event_call *call, struct dentry *d_events)
502{
503 struct dentry *entry;
504 int ret;
505
506 /*
507 * If the trace point header did not define TRACE_SYSTEM
508 * then the system would be called "TRACE_SYSTEM".
509 */
510 if (strcmp(call->system, "TRACE_SYSTEM") != 0)
511 d_events = event_subsystem_dir(call->system, d_events);
512
513 if (call->raw_init) {
514 ret = call->raw_init();
515 if (ret < 0) {
516 pr_warning("Could not initialize trace point"
517 " events/%s\n", call->name);
518 return ret;
519 }
520 }
521
522 call->dir = debugfs_create_dir(call->name, d_events);
523 if (!call->dir) {
524 pr_warning("Could not create debugfs "
525 "'%s' directory\n", call->name);
526 return -1;
527 }
528
529 if (call->regfunc) {
530 entry = debugfs_create_file("enable", 0644, call->dir, call,
531 &ftrace_enable_fops);
532 if (!entry)
533 pr_warning("Could not create debugfs "
534 "'%s/enable' entry\n", call->name);
535 }
536
537 /* A trace may not want to export its format */
538 if (!call->show_format)
539 return 0;
540
541 entry = debugfs_create_file("format", 0444, call->dir, call,
542 &ftrace_event_format_fops);
543 if (!entry)
544 pr_warning("Could not create debugfs "
545 "'%s/format' entry\n", call->name);
546
547 return 0;
548}
549
550static __init int event_trace_init(void)
551{
552 struct ftrace_event_call *call = __start_ftrace_events;
553 struct dentry *d_tracer;
554 struct dentry *entry;
555 struct dentry *d_events;
556
557 d_tracer = tracing_init_dentry();
558 if (!d_tracer)
559 return 0;
560
561 entry = debugfs_create_file("set_event", 0644, d_tracer,
562 (void *)&show_set_event_seq_ops,
563 &ftrace_set_event_fops);
564 if (!entry)
565 pr_warning("Could not create debugfs "
566 "'set_event' entry\n");
567
568 d_events = event_trace_events_dir();
569 if (!d_events)
570 return 0;
571
572 events_for_each(call) {
573 /* The linker may leave blanks */
574 if (!call->name)
575 continue;
576 event_create_dir(call, d_events);
577 }
578
579 return 0;
580}
581fs_initcall(event_trace_init);
diff --git a/kernel/trace/trace_events_stage_1.h b/kernel/trace/trace_events_stage_1.h
new file mode 100644
index 000000000000..15e9bf965a18
--- /dev/null
+++ b/kernel/trace/trace_events_stage_1.h
@@ -0,0 +1,37 @@
1/*
2 * Stage 1 of the trace events.
3 *
4 * Override the macros in <trace/trace_event_types.h> to include the following:
5 *
6 * struct ftrace_raw_<call> {
7 * struct trace_entry ent;
8 * <type> <item>;
9 * [...]
10 * };
11 *
12 * The <type> <item> is created by the TRACE_FIELD(type, item, assign)
13 * macro. We simply do "type item;", and that will create the fields
14 * in the structure.
15 */
16
17#undef TRACE_FORMAT
18#define TRACE_FORMAT(call, proto, args, fmt)
19
20#undef __array
21#define __array(type, item, len) type item[len];
22
23#undef __field
24#define __field(type, item) type item;
25
26#undef TP_STRUCT__entry
27#define TP_STRUCT__entry(args...) args
28
29#undef TRACE_EVENT
30#define TRACE_EVENT(name, proto, args, tstruct, print, assign) \
31 struct ftrace_raw_##name { \
32 struct trace_entry ent; \
33 tstruct \
34 }; \
35 static struct ftrace_event_call event_##name
36
37#include <trace/trace_event_types.h>
diff --git a/kernel/trace/trace_events_stage_2.h b/kernel/trace/trace_events_stage_2.h
new file mode 100644
index 000000000000..d91bf4c56661
--- /dev/null
+++ b/kernel/trace/trace_events_stage_2.h
@@ -0,0 +1,132 @@
1/*
2 * Stage 2 of the trace events.
3 *
4 * Override the macros in <trace/trace_event_types.h> to include the following:
5 *
6 * enum print_line_t
7 * ftrace_raw_output_<call>(struct trace_iterator *iter, int flags)
8 * {
9 * struct trace_seq *s = &iter->seq;
10 * struct ftrace_raw_<call> *field; <-- defined in stage 1
11 * struct trace_entry *entry;
12 * int ret;
13 *
14 * entry = iter->ent;
15 *
16 * if (entry->type != event_<call>.id) {
17 * WARN_ON_ONCE(1);
18 * return TRACE_TYPE_UNHANDLED;
19 * }
20 *
21 * field = (typeof(field))entry;
22 *
23 * ret = trace_seq_printf(s, <TP_RAW_FMT> "%s", <ARGS> "\n");
24 * if (!ret)
25 * return TRACE_TYPE_PARTIAL_LINE;
26 *
27 * return TRACE_TYPE_HANDLED;
28 * }
29 *
30 * This is the method used to print the raw event to the trace
31 * output format. Note, this is not needed if the data is read
32 * in binary.
33 */
34
35#undef __entry
36#define __entry field
37
38#undef TP_printk
39#define TP_printk(fmt, args...) fmt "\n", args
40
41#undef TRACE_EVENT
42#define TRACE_EVENT(call, proto, args, tstruct, print, assign) \
43enum print_line_t \
44ftrace_raw_output_##call(struct trace_iterator *iter, int flags) \
45{ \
46 struct trace_seq *s = &iter->seq; \
47 struct ftrace_raw_##call *field; \
48 struct trace_entry *entry; \
49 int ret; \
50 \
51 entry = iter->ent; \
52 \
53 if (entry->type != event_##call.id) { \
54 WARN_ON_ONCE(1); \
55 return TRACE_TYPE_UNHANDLED; \
56 } \
57 \
58 field = (typeof(field))entry; \
59 \
60 ret = trace_seq_printf(s, print); \
61 if (!ret) \
62 return TRACE_TYPE_PARTIAL_LINE; \
63 \
64 return TRACE_TYPE_HANDLED; \
65}
66
67#include <trace/trace_event_types.h>
68
69/*
70 * Setup the showing format of trace point.
71 *
72 * int
73 * ftrace_format_##call(struct trace_seq *s)
74 * {
75 * struct ftrace_raw_##call field;
76 * int ret;
77 *
78 * ret = trace_seq_printf(s, #type " " #item ";"
79 * " size:%d; offset:%d;\n",
80 * sizeof(field.type),
81 * offsetof(struct ftrace_raw_##call,
82 * item));
83 *
84 * }
85 */
86
87#undef TP_STRUCT__entry
88#define TP_STRUCT__entry(args...) args
89
90#undef __field
91#define __field(type, item) \
92 ret = trace_seq_printf(s, "\tfield:" #type " " #item ";\t" \
93 "offset:%u;\tsize:%u;\n", \
94 (unsigned int)offsetof(typeof(field), item), \
95 (unsigned int)sizeof(field.item)); \
96 if (!ret) \
97 return 0;
98
99#undef __array
100#define __array(type, item, len) \
101 ret = trace_seq_printf(s, "\tfield:" #type " " #item "[" #len "];\t" \
102 "offset:%u;\tsize:%u;\n", \
103 (unsigned int)offsetof(typeof(field), item), \
104 (unsigned int)sizeof(field.item)); \
105 if (!ret) \
106 return 0;
107
108#undef __entry
109#define __entry "REC"
110
111#undef TP_printk
112#define TP_printk(fmt, args...) "%s, %s\n", #fmt, #args
113
114#undef TP_fast_assign
115#define TP_fast_assign(args...) args
116
117#undef TRACE_EVENT
118#define TRACE_EVENT(call, proto, args, tstruct, print, func) \
119static int \
120ftrace_format_##call(struct trace_seq *s) \
121{ \
122 struct ftrace_raw_##call field; \
123 int ret; \
124 \
125 tstruct; \
126 \
127 trace_seq_printf(s, "\nprint fmt: " print); \
128 \
129 return ret; \
130}
131
132#include <trace/trace_event_types.h>
diff --git a/kernel/trace/trace_events_stage_3.h b/kernel/trace/trace_events_stage_3.h
new file mode 100644
index 000000000000..3ba55d4ab073
--- /dev/null
+++ b/kernel/trace/trace_events_stage_3.h
@@ -0,0 +1,217 @@
1/*
2 * Stage 3 of the trace events.
3 *
4 * Override the macros in <trace/trace_event_types.h> to include the following:
5 *
6 * static void ftrace_event_<call>(proto)
7 * {
8 * event_trace_printk(_RET_IP_, "<call>: " <fmt>);
9 * }
10 *
11 * static int ftrace_reg_event_<call>(void)
12 * {
13 * int ret;
14 *
15 * ret = register_trace_<call>(ftrace_event_<call>);
16 * if (!ret)
17 * pr_info("event trace: Could not activate trace point "
18 * "probe to <call>");
19 * return ret;
20 * }
21 *
22 * static void ftrace_unreg_event_<call>(void)
23 * {
24 * unregister_trace_<call>(ftrace_event_<call>);
25 * }
26 *
27 * For those macros defined with TRACE_FORMAT:
28 *
29 * static struct ftrace_event_call __used
30 * __attribute__((__aligned__(4)))
31 * __attribute__((section("_ftrace_events"))) event_<call> = {
32 * .name = "<call>",
33 * .regfunc = ftrace_reg_event_<call>,
34 * .unregfunc = ftrace_unreg_event_<call>,
35 * }
36 *
37 *
38 * For those macros defined with TRACE_EVENT:
39 *
40 * static struct ftrace_event_call event_<call>;
41 *
42 * static void ftrace_raw_event_<call>(proto)
43 * {
44 * struct ring_buffer_event *event;
45 * struct ftrace_raw_<call> *entry; <-- defined in stage 1
46 * unsigned long irq_flags;
47 * int pc;
48 *
49 * local_save_flags(irq_flags);
50 * pc = preempt_count();
51 *
52 * event = trace_current_buffer_lock_reserve(event_<call>.id,
53 * sizeof(struct ftrace_raw_<call>),
54 * irq_flags, pc);
55 * if (!event)
56 * return;
57 * entry = ring_buffer_event_data(event);
58 *
59 * <tstruct>; <-- Here we assign the entries by the TRACE_FIELD.
60 *
61 * trace_current_buffer_unlock_commit(event, irq_flags, pc);
62 * }
63 *
64 * static int ftrace_raw_reg_event_<call>(void)
65 * {
66 * int ret;
67 *
68 * ret = register_trace_<call>(ftrace_raw_event_<call>);
69 * if (!ret)
70 * pr_info("event trace: Could not activate trace point "
71 * "probe to <call>");
72 * return ret;
73 * }
74 *
75 * static void ftrace_unreg_event_<call>(void)
76 * {
77 * unregister_trace_<call>(ftrace_raw_event_<call>);
78 * }
79 *
80 * static struct trace_event ftrace_event_type_<call> = {
81 * .trace = ftrace_raw_output_<call>, <-- stage 2
82 * };
83 *
84 * static int ftrace_raw_init_event_<call>(void)
85 * {
86 * int id;
87 *
88 * id = register_ftrace_event(&ftrace_event_type_<call>);
89 * if (!id)
90 * return -ENODEV;
91 * event_<call>.id = id;
92 * return 0;
93 * }
94 *
95 * static struct ftrace_event_call __used
96 * __attribute__((__aligned__(4)))
97 * __attribute__((section("_ftrace_events"))) event_<call> = {
98 * .name = "<call>",
99 * .regfunc = ftrace_reg_event_<call>,
100 * .unregfunc = ftrace_unreg_event_<call>,
101 * .raw_init = ftrace_raw_init_event_<call>,
102 * .raw_reg = ftrace_raw_reg_event_<call>,
103 * .raw_unreg = ftrace_raw_unreg_event_<call>,
104 * .show_format = ftrace_format_<call>,
105 * }
106 *
107 */
108
109#undef TP_FMT
110#define TP_FMT(fmt, args...) fmt "\n", ##args
111
112#define _TRACE_FORMAT(call, proto, args, fmt) \
113static void ftrace_event_##call(proto) \
114{ \
115 event_trace_printk(_RET_IP_, #call ": " fmt); \
116} \
117 \
118static int ftrace_reg_event_##call(void) \
119{ \
120 int ret; \
121 \
122 ret = register_trace_##call(ftrace_event_##call); \
123 if (ret) \
124 pr_info("event trace: Could not activate trace point " \
125 "probe to " #call "\n"); \
126 return ret; \
127} \
128 \
129static void ftrace_unreg_event_##call(void) \
130{ \
131 unregister_trace_##call(ftrace_event_##call); \
132} \
133
134
135#undef TRACE_FORMAT
136#define TRACE_FORMAT(call, proto, args, fmt) \
137_TRACE_FORMAT(call, PARAMS(proto), PARAMS(args), PARAMS(fmt)) \
138static struct ftrace_event_call __used \
139__attribute__((__aligned__(4))) \
140__attribute__((section("_ftrace_events"))) event_##call = { \
141 .name = #call, \
142 .system = __stringify(TRACE_SYSTEM), \
143 .regfunc = ftrace_reg_event_##call, \
144 .unregfunc = ftrace_unreg_event_##call, \
145}
146
147#undef __entry
148#define __entry entry
149
150#undef TRACE_EVENT
151#define TRACE_EVENT(call, proto, args, tstruct, print, assign) \
152 \
153static struct ftrace_event_call event_##call; \
154 \
155static void ftrace_raw_event_##call(proto) \
156{ \
157 struct ring_buffer_event *event; \
158 struct ftrace_raw_##call *entry; \
159 unsigned long irq_flags; \
160 int pc; \
161 \
162 local_save_flags(irq_flags); \
163 pc = preempt_count(); \
164 \
165 event = trace_current_buffer_lock_reserve(event_##call.id, \
166 sizeof(struct ftrace_raw_##call), \
167 irq_flags, pc); \
168 if (!event) \
169 return; \
170 entry = ring_buffer_event_data(event); \
171 \
172 assign; \
173 \
174 trace_current_buffer_unlock_commit(event, irq_flags, pc); \
175} \
176 \
177static int ftrace_raw_reg_event_##call(void) \
178{ \
179 int ret; \
180 \
181 ret = register_trace_##call(ftrace_raw_event_##call); \
182 if (ret) \
183 pr_info("event trace: Could not activate trace point " \
184 "probe to " #call "\n"); \
185 return ret; \
186} \
187 \
188static void ftrace_raw_unreg_event_##call(void) \
189{ \
190 unregister_trace_##call(ftrace_raw_event_##call); \
191} \
192 \
193static struct trace_event ftrace_event_type_##call = { \
194 .trace = ftrace_raw_output_##call, \
195}; \
196 \
197static int ftrace_raw_init_event_##call(void) \
198{ \
199 int id; \
200 \
201 id = register_ftrace_event(&ftrace_event_type_##call); \
202 if (!id) \
203 return -ENODEV; \
204 event_##call.id = id; \
205 return 0; \
206} \
207 \
208static struct ftrace_event_call __used \
209__attribute__((__aligned__(4))) \
210__attribute__((section("_ftrace_events"))) event_##call = { \
211 .name = #call, \
212 .system = __stringify(TRACE_SYSTEM), \
213 .raw_init = ftrace_raw_init_event_##call, \
214 .regfunc = ftrace_raw_reg_event_##call, \
215 .unregfunc = ftrace_raw_unreg_event_##call, \
216 .show_format = ftrace_format_##call, \
217}
diff --git a/kernel/trace/trace_export.c b/kernel/trace/trace_export.c
new file mode 100644
index 000000000000..23ae78430d58
--- /dev/null
+++ b/kernel/trace/trace_export.c
@@ -0,0 +1,102 @@
1/*
2 * trace_export.c - export basic ftrace utilities to user space
3 *
4 * Copyright (C) 2009 Steven Rostedt <srostedt@redhat.com>
5 */
6#include <linux/stringify.h>
7#include <linux/kallsyms.h>
8#include <linux/seq_file.h>
9#include <linux/debugfs.h>
10#include <linux/uaccess.h>
11#include <linux/ftrace.h>
12#include <linux/module.h>
13#include <linux/init.h>
14#include <linux/fs.h>
15
16#include "trace_output.h"
17
18
19#undef TRACE_STRUCT
20#define TRACE_STRUCT(args...) args
21
22#undef TRACE_FIELD
23#define TRACE_FIELD(type, item, assign) \
24 ret = trace_seq_printf(s, "\tfield:" #type " " #item ";\t" \
25 "offset:%u;\tsize:%u;\n", \
26 (unsigned int)offsetof(typeof(field), item), \
27 (unsigned int)sizeof(field.item)); \
28 if (!ret) \
29 return 0;
30
31
32#undef TRACE_FIELD_SPECIAL
33#define TRACE_FIELD_SPECIAL(type_item, item, cmd) \
34 ret = trace_seq_printf(s, "\tfield special:" #type_item ";\t" \
35 "offset:%u;\tsize:%u;\n", \
36 (unsigned int)offsetof(typeof(field), item), \
37 (unsigned int)sizeof(field.item)); \
38 if (!ret) \
39 return 0;
40
41#undef TRACE_FIELD_ZERO_CHAR
42#define TRACE_FIELD_ZERO_CHAR(item) \
43 ret = trace_seq_printf(s, "\tfield: char " #item ";\t" \
44 "offset:%u;\tsize:0;\n", \
45 (unsigned int)offsetof(typeof(field), item)); \
46 if (!ret) \
47 return 0;
48
49
50#undef TP_RAW_FMT
51#define TP_RAW_FMT(args...) args
52
53#undef TRACE_EVENT_FORMAT
54#define TRACE_EVENT_FORMAT(call, proto, args, fmt, tstruct, tpfmt) \
55static int \
56ftrace_format_##call(struct trace_seq *s) \
57{ \
58 struct args field; \
59 int ret; \
60 \
61 tstruct; \
62 \
63 trace_seq_printf(s, "\nprint fmt: \"%s\"\n", tpfmt); \
64 \
65 return ret; \
66}
67
68#include "trace_event_types.h"
69
70#undef TRACE_ZERO_CHAR
71#define TRACE_ZERO_CHAR(arg)
72
73#undef TRACE_FIELD
74#define TRACE_FIELD(type, item, assign)\
75 entry->item = assign;
76
77#undef TRACE_FIELD
78#define TRACE_FIELD(type, item, assign)\
79 entry->item = assign;
80
81#undef TP_CMD
82#define TP_CMD(cmd...) cmd
83
84#undef TRACE_ENTRY
85#define TRACE_ENTRY entry
86
87#undef TRACE_FIELD_SPECIAL
88#define TRACE_FIELD_SPECIAL(type_item, item, cmd) \
89 cmd;
90
91#undef TRACE_EVENT_FORMAT
92#define TRACE_EVENT_FORMAT(call, proto, args, fmt, tstruct, tpfmt) \
93 \
94static struct ftrace_event_call __used \
95__attribute__((__aligned__(4))) \
96__attribute__((section("_ftrace_events"))) event_##call = { \
97 .name = #call, \
98 .id = proto, \
99 .system = __stringify(TRACE_SYSTEM), \
100 .show_format = ftrace_format_##call, \
101}
102#include "trace_event_types.h"
diff --git a/kernel/trace/trace_functions.c b/kernel/trace/trace_functions.c
index 9236d7e25a16..c9a0b7df44ff 100644
--- a/kernel/trace/trace_functions.c
+++ b/kernel/trace/trace_functions.c
@@ -9,6 +9,7 @@
9 * Copyright (C) 2004-2006 Ingo Molnar 9 * Copyright (C) 2004-2006 Ingo Molnar
10 * Copyright (C) 2004 William Lee Irwin III 10 * Copyright (C) 2004 William Lee Irwin III
11 */ 11 */
12#include <linux/ring_buffer.h>
12#include <linux/debugfs.h> 13#include <linux/debugfs.h>
13#include <linux/uaccess.h> 14#include <linux/uaccess.h>
14#include <linux/ftrace.h> 15#include <linux/ftrace.h>
@@ -16,52 +17,388 @@
16 17
17#include "trace.h" 18#include "trace.h"
18 19
19static void start_function_trace(struct trace_array *tr) 20/* function tracing enabled */
21static int ftrace_function_enabled;
22
23static struct trace_array *func_trace;
24
25static void tracing_start_function_trace(void);
26static void tracing_stop_function_trace(void);
27
28static int function_trace_init(struct trace_array *tr)
20{ 29{
30 func_trace = tr;
21 tr->cpu = get_cpu(); 31 tr->cpu = get_cpu();
22 tracing_reset_online_cpus(tr);
23 put_cpu(); 32 put_cpu();
24 33
25 tracing_start_cmdline_record(); 34 tracing_start_cmdline_record();
26 tracing_start_function_trace(); 35 tracing_start_function_trace();
36 return 0;
27} 37}
28 38
29static void stop_function_trace(struct trace_array *tr) 39static void function_trace_reset(struct trace_array *tr)
30{ 40{
31 tracing_stop_function_trace(); 41 tracing_stop_function_trace();
32 tracing_stop_cmdline_record(); 42 tracing_stop_cmdline_record();
33} 43}
34 44
35static int function_trace_init(struct trace_array *tr) 45static void function_trace_start(struct trace_array *tr)
36{ 46{
37 start_function_trace(tr); 47 tracing_reset_online_cpus(tr);
38 return 0;
39} 48}
40 49
41static void function_trace_reset(struct trace_array *tr) 50static void
51function_trace_call_preempt_only(unsigned long ip, unsigned long parent_ip)
52{
53 struct trace_array *tr = func_trace;
54 struct trace_array_cpu *data;
55 unsigned long flags;
56 long disabled;
57 int cpu, resched;
58 int pc;
59
60 if (unlikely(!ftrace_function_enabled))
61 return;
62
63 pc = preempt_count();
64 resched = ftrace_preempt_disable();
65 local_save_flags(flags);
66 cpu = raw_smp_processor_id();
67 data = tr->data[cpu];
68 disabled = atomic_inc_return(&data->disabled);
69
70 if (likely(disabled == 1))
71 trace_function(tr, ip, parent_ip, flags, pc);
72
73 atomic_dec(&data->disabled);
74 ftrace_preempt_enable(resched);
75}
76
77static void
78function_trace_call(unsigned long ip, unsigned long parent_ip)
42{ 79{
43 stop_function_trace(tr); 80 struct trace_array *tr = func_trace;
81 struct trace_array_cpu *data;
82 unsigned long flags;
83 long disabled;
84 int cpu;
85 int pc;
86
87 if (unlikely(!ftrace_function_enabled))
88 return;
89
90 /*
91 * Need to use raw, since this must be called before the
92 * recursive protection is performed.
93 */
94 local_irq_save(flags);
95 cpu = raw_smp_processor_id();
96 data = tr->data[cpu];
97 disabled = atomic_inc_return(&data->disabled);
98
99 if (likely(disabled == 1)) {
100 pc = preempt_count();
101 trace_function(tr, ip, parent_ip, flags, pc);
102 }
103
104 atomic_dec(&data->disabled);
105 local_irq_restore(flags);
44} 106}
45 107
46static void function_trace_start(struct trace_array *tr) 108static void
109function_stack_trace_call(unsigned long ip, unsigned long parent_ip)
47{ 110{
48 tracing_reset_online_cpus(tr); 111 struct trace_array *tr = func_trace;
112 struct trace_array_cpu *data;
113 unsigned long flags;
114 long disabled;
115 int cpu;
116 int pc;
117
118 if (unlikely(!ftrace_function_enabled))
119 return;
120
121 /*
122 * Need to use raw, since this must be called before the
123 * recursive protection is performed.
124 */
125 local_irq_save(flags);
126 cpu = raw_smp_processor_id();
127 data = tr->data[cpu];
128 disabled = atomic_inc_return(&data->disabled);
129
130 if (likely(disabled == 1)) {
131 pc = preempt_count();
132 trace_function(tr, ip, parent_ip, flags, pc);
133 /*
134 * skip over 5 funcs:
135 * __ftrace_trace_stack,
136 * __trace_stack,
137 * function_stack_trace_call
138 * ftrace_list_func
139 * ftrace_call
140 */
141 __trace_stack(tr, flags, 5, pc);
142 }
143
144 atomic_dec(&data->disabled);
145 local_irq_restore(flags);
146}
147
148
149static struct ftrace_ops trace_ops __read_mostly =
150{
151 .func = function_trace_call,
152};
153
154static struct ftrace_ops trace_stack_ops __read_mostly =
155{
156 .func = function_stack_trace_call,
157};
158
159/* Our two options */
160enum {
161 TRACE_FUNC_OPT_STACK = 0x1,
162};
163
164static struct tracer_opt func_opts[] = {
165#ifdef CONFIG_STACKTRACE
166 { TRACER_OPT(func_stack_trace, TRACE_FUNC_OPT_STACK) },
167#endif
168 { } /* Always set a last empty entry */
169};
170
171static struct tracer_flags func_flags = {
172 .val = 0, /* By default: all flags disabled */
173 .opts = func_opts
174};
175
176static void tracing_start_function_trace(void)
177{
178 ftrace_function_enabled = 0;
179
180 if (trace_flags & TRACE_ITER_PREEMPTONLY)
181 trace_ops.func = function_trace_call_preempt_only;
182 else
183 trace_ops.func = function_trace_call;
184
185 if (func_flags.val & TRACE_FUNC_OPT_STACK)
186 register_ftrace_function(&trace_stack_ops);
187 else
188 register_ftrace_function(&trace_ops);
189
190 ftrace_function_enabled = 1;
191}
192
193static void tracing_stop_function_trace(void)
194{
195 ftrace_function_enabled = 0;
196 /* OK if they are not registered */
197 unregister_ftrace_function(&trace_stack_ops);
198 unregister_ftrace_function(&trace_ops);
199}
200
201static int func_set_flag(u32 old_flags, u32 bit, int set)
202{
203 if (bit == TRACE_FUNC_OPT_STACK) {
204 /* do nothing if already set */
205 if (!!set == !!(func_flags.val & TRACE_FUNC_OPT_STACK))
206 return 0;
207
208 if (set) {
209 unregister_ftrace_function(&trace_ops);
210 register_ftrace_function(&trace_stack_ops);
211 } else {
212 unregister_ftrace_function(&trace_stack_ops);
213 register_ftrace_function(&trace_ops);
214 }
215
216 return 0;
217 }
218
219 return -EINVAL;
49} 220}
50 221
51static struct tracer function_trace __read_mostly = 222static struct tracer function_trace __read_mostly =
52{ 223{
53 .name = "function", 224 .name = "function",
54 .init = function_trace_init, 225 .init = function_trace_init,
55 .reset = function_trace_reset, 226 .reset = function_trace_reset,
56 .start = function_trace_start, 227 .start = function_trace_start,
228 .wait_pipe = poll_wait_pipe,
229 .flags = &func_flags,
230 .set_flag = func_set_flag,
57#ifdef CONFIG_FTRACE_SELFTEST 231#ifdef CONFIG_FTRACE_SELFTEST
58 .selftest = trace_selftest_startup_function, 232 .selftest = trace_selftest_startup_function,
59#endif 233#endif
60}; 234};
61 235
236#ifdef CONFIG_DYNAMIC_FTRACE
237static void
238ftrace_traceon(unsigned long ip, unsigned long parent_ip, void **data)
239{
240 long *count = (long *)data;
241
242 if (tracing_is_on())
243 return;
244
245 if (!*count)
246 return;
247
248 if (*count != -1)
249 (*count)--;
250
251 tracing_on();
252}
253
254static void
255ftrace_traceoff(unsigned long ip, unsigned long parent_ip, void **data)
256{
257 long *count = (long *)data;
258
259 if (!tracing_is_on())
260 return;
261
262 if (!*count)
263 return;
264
265 if (*count != -1)
266 (*count)--;
267
268 tracing_off();
269}
270
271static int
272ftrace_trace_onoff_print(struct seq_file *m, unsigned long ip,
273 struct ftrace_probe_ops *ops, void *data);
274
275static struct ftrace_probe_ops traceon_probe_ops = {
276 .func = ftrace_traceon,
277 .print = ftrace_trace_onoff_print,
278};
279
280static struct ftrace_probe_ops traceoff_probe_ops = {
281 .func = ftrace_traceoff,
282 .print = ftrace_trace_onoff_print,
283};
284
285static int
286ftrace_trace_onoff_print(struct seq_file *m, unsigned long ip,
287 struct ftrace_probe_ops *ops, void *data)
288{
289 char str[KSYM_SYMBOL_LEN];
290 long count = (long)data;
291
292 kallsyms_lookup(ip, NULL, NULL, NULL, str);
293 seq_printf(m, "%s:", str);
294
295 if (ops == &traceon_probe_ops)
296 seq_printf(m, "traceon");
297 else
298 seq_printf(m, "traceoff");
299
300 if (count == -1)
301 seq_printf(m, ":unlimited\n");
302 else
303 seq_printf(m, ":count=%ld", count);
304 seq_putc(m, '\n');
305
306 return 0;
307}
308
309static int
310ftrace_trace_onoff_unreg(char *glob, char *cmd, char *param)
311{
312 struct ftrace_probe_ops *ops;
313
314 /* we register both traceon and traceoff to this callback */
315 if (strcmp(cmd, "traceon") == 0)
316 ops = &traceon_probe_ops;
317 else
318 ops = &traceoff_probe_ops;
319
320 unregister_ftrace_function_probe_func(glob, ops);
321
322 return 0;
323}
324
325static int
326ftrace_trace_onoff_callback(char *glob, char *cmd, char *param, int enable)
327{
328 struct ftrace_probe_ops *ops;
329 void *count = (void *)-1;
330 char *number;
331 int ret;
332
333 /* hash funcs only work with set_ftrace_filter */
334 if (!enable)
335 return -EINVAL;
336
337 if (glob[0] == '!')
338 return ftrace_trace_onoff_unreg(glob+1, cmd, param);
339
340 /* we register both traceon and traceoff to this callback */
341 if (strcmp(cmd, "traceon") == 0)
342 ops = &traceon_probe_ops;
343 else
344 ops = &traceoff_probe_ops;
345
346 if (!param)
347 goto out_reg;
348
349 number = strsep(&param, ":");
350
351 if (!strlen(number))
352 goto out_reg;
353
354 /*
355 * We use the callback data field (which is a pointer)
356 * as our counter.
357 */
358 ret = strict_strtoul(number, 0, (unsigned long *)&count);
359 if (ret)
360 return ret;
361
362 out_reg:
363 ret = register_ftrace_function_probe(glob, ops, count);
364
365 return ret;
366}
367
368static struct ftrace_func_command ftrace_traceon_cmd = {
369 .name = "traceon",
370 .func = ftrace_trace_onoff_callback,
371};
372
373static struct ftrace_func_command ftrace_traceoff_cmd = {
374 .name = "traceoff",
375 .func = ftrace_trace_onoff_callback,
376};
377
378static int __init init_func_cmd_traceon(void)
379{
380 int ret;
381
382 ret = register_ftrace_command(&ftrace_traceoff_cmd);
383 if (ret)
384 return ret;
385
386 ret = register_ftrace_command(&ftrace_traceon_cmd);
387 if (ret)
388 unregister_ftrace_command(&ftrace_traceoff_cmd);
389 return ret;
390}
391#else
392static inline int init_func_cmd_traceon(void)
393{
394 return 0;
395}
396#endif /* CONFIG_DYNAMIC_FTRACE */
397
62static __init int init_function_trace(void) 398static __init int init_function_trace(void)
63{ 399{
400 init_func_cmd_traceon();
64 return register_tracer(&function_trace); 401 return register_tracer(&function_trace);
65} 402}
66
67device_initcall(init_function_trace); 403device_initcall(init_function_trace);
404
diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c
index 930c08e5b38e..35257be6a9d6 100644
--- a/kernel/trace/trace_functions_graph.c
+++ b/kernel/trace/trace_functions_graph.c
@@ -1,7 +1,7 @@
1/* 1/*
2 * 2 *
3 * Function graph tracer. 3 * Function graph tracer.
4 * Copyright (c) 2008 Frederic Weisbecker <fweisbec@gmail.com> 4 * Copyright (c) 2008-2009 Frederic Weisbecker <fweisbec@gmail.com>
5 * Mostly borrowed from function tracer which 5 * Mostly borrowed from function tracer which
6 * is Copyright (c) Steven Rostedt <srostedt@redhat.com> 6 * is Copyright (c) Steven Rostedt <srostedt@redhat.com>
7 * 7 *
@@ -12,6 +12,7 @@
12#include <linux/fs.h> 12#include <linux/fs.h>
13 13
14#include "trace.h" 14#include "trace.h"
15#include "trace_output.h"
15 16
16#define TRACE_GRAPH_INDENT 2 17#define TRACE_GRAPH_INDENT 2
17 18
@@ -20,9 +21,11 @@
20#define TRACE_GRAPH_PRINT_CPU 0x2 21#define TRACE_GRAPH_PRINT_CPU 0x2
21#define TRACE_GRAPH_PRINT_OVERHEAD 0x4 22#define TRACE_GRAPH_PRINT_OVERHEAD 0x4
22#define TRACE_GRAPH_PRINT_PROC 0x8 23#define TRACE_GRAPH_PRINT_PROC 0x8
24#define TRACE_GRAPH_PRINT_DURATION 0x10
25#define TRACE_GRAPH_PRINT_ABS_TIME 0X20
23 26
24static struct tracer_opt trace_opts[] = { 27static struct tracer_opt trace_opts[] = {
25 /* Display overruns ? */ 28 /* Display overruns? (for self-debug purpose) */
26 { TRACER_OPT(funcgraph-overrun, TRACE_GRAPH_PRINT_OVERRUN) }, 29 { TRACER_OPT(funcgraph-overrun, TRACE_GRAPH_PRINT_OVERRUN) },
27 /* Display CPU ? */ 30 /* Display CPU ? */
28 { TRACER_OPT(funcgraph-cpu, TRACE_GRAPH_PRINT_CPU) }, 31 { TRACER_OPT(funcgraph-cpu, TRACE_GRAPH_PRINT_CPU) },
@@ -30,26 +33,101 @@ static struct tracer_opt trace_opts[] = {
30 { TRACER_OPT(funcgraph-overhead, TRACE_GRAPH_PRINT_OVERHEAD) }, 33 { TRACER_OPT(funcgraph-overhead, TRACE_GRAPH_PRINT_OVERHEAD) },
31 /* Display proc name/pid */ 34 /* Display proc name/pid */
32 { TRACER_OPT(funcgraph-proc, TRACE_GRAPH_PRINT_PROC) }, 35 { TRACER_OPT(funcgraph-proc, TRACE_GRAPH_PRINT_PROC) },
36 /* Display duration of execution */
37 { TRACER_OPT(funcgraph-duration, TRACE_GRAPH_PRINT_DURATION) },
38 /* Display absolute time of an entry */
39 { TRACER_OPT(funcgraph-abstime, TRACE_GRAPH_PRINT_ABS_TIME) },
33 { } /* Empty entry */ 40 { } /* Empty entry */
34}; 41};
35 42
36static struct tracer_flags tracer_flags = { 43static struct tracer_flags tracer_flags = {
37 /* Don't display overruns and proc by default */ 44 /* Don't display overruns and proc by default */
38 .val = TRACE_GRAPH_PRINT_CPU | TRACE_GRAPH_PRINT_OVERHEAD, 45 .val = TRACE_GRAPH_PRINT_CPU | TRACE_GRAPH_PRINT_OVERHEAD |
46 TRACE_GRAPH_PRINT_DURATION,
39 .opts = trace_opts 47 .opts = trace_opts
40}; 48};
41 49
42/* pid on the last trace processed */ 50/* pid on the last trace processed */
43static pid_t last_pid[NR_CPUS] = { [0 ... NR_CPUS-1] = -1 };
44 51
45static int graph_trace_init(struct trace_array *tr) 52
53/* Add a function return address to the trace stack on thread info.*/
54int
55ftrace_push_return_trace(unsigned long ret, unsigned long long time,
56 unsigned long func, int *depth)
57{
58 int index;
59
60 if (!current->ret_stack)
61 return -EBUSY;
62
63 /* The return trace stack is full */
64 if (current->curr_ret_stack == FTRACE_RETFUNC_DEPTH - 1) {
65 atomic_inc(&current->trace_overrun);
66 return -EBUSY;
67 }
68
69 index = ++current->curr_ret_stack;
70 barrier();
71 current->ret_stack[index].ret = ret;
72 current->ret_stack[index].func = func;
73 current->ret_stack[index].calltime = time;
74 *depth = index;
75
76 return 0;
77}
78
79/* Retrieve a function return address to the trace stack on thread info.*/
80void
81ftrace_pop_return_trace(struct ftrace_graph_ret *trace, unsigned long *ret)
82{
83 int index;
84
85 index = current->curr_ret_stack;
86
87 if (unlikely(index < 0)) {
88 ftrace_graph_stop();
89 WARN_ON(1);
90 /* Might as well panic, otherwise we have no where to go */
91 *ret = (unsigned long)panic;
92 return;
93 }
94
95 *ret = current->ret_stack[index].ret;
96 trace->func = current->ret_stack[index].func;
97 trace->calltime = current->ret_stack[index].calltime;
98 trace->overrun = atomic_read(&current->trace_overrun);
99 trace->depth = index;
100 barrier();
101 current->curr_ret_stack--;
102
103}
104
105/*
106 * Send the trace to the ring-buffer.
107 * @return the original return address.
108 */
109unsigned long ftrace_return_to_handler(void)
46{ 110{
47 int cpu, ret; 111 struct ftrace_graph_ret trace;
112 unsigned long ret;
113
114 ftrace_pop_return_trace(&trace, &ret);
115 trace.rettime = trace_clock_local();
116 ftrace_graph_return(&trace);
117
118 if (unlikely(!ret)) {
119 ftrace_graph_stop();
120 WARN_ON(1);
121 /* Might as well panic. What else to do? */
122 ret = (unsigned long)panic;
123 }
48 124
49 for_each_online_cpu(cpu) 125 return ret;
50 tracing_reset(tr, cpu); 126}
51 127
52 ret = register_ftrace_graph(&trace_graph_return, 128static int graph_trace_init(struct trace_array *tr)
129{
130 int ret = register_ftrace_graph(&trace_graph_return,
53 &trace_graph_entry); 131 &trace_graph_entry);
54 if (ret) 132 if (ret)
55 return ret; 133 return ret;
@@ -153,17 +231,25 @@ print_graph_proc(struct trace_seq *s, pid_t pid)
153 231
154/* If the pid changed since the last trace, output this event */ 232/* If the pid changed since the last trace, output this event */
155static enum print_line_t 233static enum print_line_t
156verif_pid(struct trace_seq *s, pid_t pid, int cpu) 234verif_pid(struct trace_seq *s, pid_t pid, int cpu, pid_t *last_pids_cpu)
157{ 235{
158 pid_t prev_pid; 236 pid_t prev_pid;
237 pid_t *last_pid;
159 int ret; 238 int ret;
160 239
161 if (last_pid[cpu] != -1 && last_pid[cpu] == pid) 240 if (!last_pids_cpu)
241 return TRACE_TYPE_HANDLED;
242
243 last_pid = per_cpu_ptr(last_pids_cpu, cpu);
244
245 if (*last_pid == pid)
162 return TRACE_TYPE_HANDLED; 246 return TRACE_TYPE_HANDLED;
163 247
164 prev_pid = last_pid[cpu]; 248 prev_pid = *last_pid;
165 last_pid[cpu] = pid; 249 *last_pid = pid;
166 250
251 if (prev_pid == -1)
252 return TRACE_TYPE_HANDLED;
167/* 253/*
168 * Context-switch trace line: 254 * Context-switch trace line:
169 255
@@ -175,34 +261,34 @@ verif_pid(struct trace_seq *s, pid_t pid, int cpu)
175 ret = trace_seq_printf(s, 261 ret = trace_seq_printf(s,
176 " ------------------------------------------\n"); 262 " ------------------------------------------\n");
177 if (!ret) 263 if (!ret)
178 TRACE_TYPE_PARTIAL_LINE; 264 return TRACE_TYPE_PARTIAL_LINE;
179 265
180 ret = print_graph_cpu(s, cpu); 266 ret = print_graph_cpu(s, cpu);
181 if (ret == TRACE_TYPE_PARTIAL_LINE) 267 if (ret == TRACE_TYPE_PARTIAL_LINE)
182 TRACE_TYPE_PARTIAL_LINE; 268 return TRACE_TYPE_PARTIAL_LINE;
183 269
184 ret = print_graph_proc(s, prev_pid); 270 ret = print_graph_proc(s, prev_pid);
185 if (ret == TRACE_TYPE_PARTIAL_LINE) 271 if (ret == TRACE_TYPE_PARTIAL_LINE)
186 TRACE_TYPE_PARTIAL_LINE; 272 return TRACE_TYPE_PARTIAL_LINE;
187 273
188 ret = trace_seq_printf(s, " => "); 274 ret = trace_seq_printf(s, " => ");
189 if (!ret) 275 if (!ret)
190 TRACE_TYPE_PARTIAL_LINE; 276 return TRACE_TYPE_PARTIAL_LINE;
191 277
192 ret = print_graph_proc(s, pid); 278 ret = print_graph_proc(s, pid);
193 if (ret == TRACE_TYPE_PARTIAL_LINE) 279 if (ret == TRACE_TYPE_PARTIAL_LINE)
194 TRACE_TYPE_PARTIAL_LINE; 280 return TRACE_TYPE_PARTIAL_LINE;
195 281
196 ret = trace_seq_printf(s, 282 ret = trace_seq_printf(s,
197 "\n ------------------------------------------\n\n"); 283 "\n ------------------------------------------\n\n");
198 if (!ret) 284 if (!ret)
199 TRACE_TYPE_PARTIAL_LINE; 285 return TRACE_TYPE_PARTIAL_LINE;
200 286
201 return ret; 287 return TRACE_TYPE_HANDLED;
202} 288}
203 289
204static bool 290static struct ftrace_graph_ret_entry *
205trace_branch_is_leaf(struct trace_iterator *iter, 291get_return_for_leaf(struct trace_iterator *iter,
206 struct ftrace_graph_ent_entry *curr) 292 struct ftrace_graph_ent_entry *curr)
207{ 293{
208 struct ring_buffer_iter *ring_iter; 294 struct ring_buffer_iter *ring_iter;
@@ -211,65 +297,123 @@ trace_branch_is_leaf(struct trace_iterator *iter,
211 297
212 ring_iter = iter->buffer_iter[iter->cpu]; 298 ring_iter = iter->buffer_iter[iter->cpu];
213 299
214 if (!ring_iter) 300 /* First peek to compare current entry and the next one */
215 return false; 301 if (ring_iter)
216 302 event = ring_buffer_iter_peek(ring_iter, NULL);
217 event = ring_buffer_iter_peek(ring_iter, NULL); 303 else {
304 /* We need to consume the current entry to see the next one */
305 ring_buffer_consume(iter->tr->buffer, iter->cpu, NULL);
306 event = ring_buffer_peek(iter->tr->buffer, iter->cpu,
307 NULL);
308 }
218 309
219 if (!event) 310 if (!event)
220 return false; 311 return NULL;
221 312
222 next = ring_buffer_event_data(event); 313 next = ring_buffer_event_data(event);
223 314
224 if (next->ent.type != TRACE_GRAPH_RET) 315 if (next->ent.type != TRACE_GRAPH_RET)
225 return false; 316 return NULL;
226 317
227 if (curr->ent.pid != next->ent.pid || 318 if (curr->ent.pid != next->ent.pid ||
228 curr->graph_ent.func != next->ret.func) 319 curr->graph_ent.func != next->ret.func)
229 return false; 320 return NULL;
321
322 /* this is a leaf, now advance the iterator */
323 if (ring_iter)
324 ring_buffer_read(ring_iter, NULL);
230 325
231 return true; 326 return next;
327}
328
329/* Signal a overhead of time execution to the output */
330static int
331print_graph_overhead(unsigned long long duration, struct trace_seq *s)
332{
333 /* If duration disappear, we don't need anything */
334 if (!(tracer_flags.val & TRACE_GRAPH_PRINT_DURATION))
335 return 1;
336
337 /* Non nested entry or return */
338 if (duration == -1)
339 return trace_seq_printf(s, " ");
340
341 if (tracer_flags.val & TRACE_GRAPH_PRINT_OVERHEAD) {
342 /* Duration exceeded 100 msecs */
343 if (duration > 100000ULL)
344 return trace_seq_printf(s, "! ");
345
346 /* Duration exceeded 10 msecs */
347 if (duration > 10000ULL)
348 return trace_seq_printf(s, "+ ");
349 }
350
351 return trace_seq_printf(s, " ");
352}
353
354static int print_graph_abs_time(u64 t, struct trace_seq *s)
355{
356 unsigned long usecs_rem;
357
358 usecs_rem = do_div(t, NSEC_PER_SEC);
359 usecs_rem /= 1000;
360
361 return trace_seq_printf(s, "%5lu.%06lu | ",
362 (unsigned long)t, usecs_rem);
232} 363}
233 364
234static enum print_line_t 365static enum print_line_t
235print_graph_irq(struct trace_seq *s, unsigned long addr, 366print_graph_irq(struct trace_iterator *iter, unsigned long addr,
236 enum trace_type type, int cpu, pid_t pid) 367 enum trace_type type, int cpu, pid_t pid)
237{ 368{
238 int ret; 369 int ret;
370 struct trace_seq *s = &iter->seq;
239 371
240 if (addr < (unsigned long)__irqentry_text_start || 372 if (addr < (unsigned long)__irqentry_text_start ||
241 addr >= (unsigned long)__irqentry_text_end) 373 addr >= (unsigned long)__irqentry_text_end)
242 return TRACE_TYPE_UNHANDLED; 374 return TRACE_TYPE_UNHANDLED;
243 375
244 if (type == TRACE_GRAPH_ENT) { 376 /* Absolute time */
245 ret = trace_seq_printf(s, "==========> | "); 377 if (tracer_flags.val & TRACE_GRAPH_PRINT_ABS_TIME) {
246 } else { 378 ret = print_graph_abs_time(iter->ts, s);
247 /* Cpu */ 379 if (!ret)
248 if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU) { 380 return TRACE_TYPE_PARTIAL_LINE;
249 ret = print_graph_cpu(s, cpu); 381 }
250 if (ret == TRACE_TYPE_PARTIAL_LINE)
251 return TRACE_TYPE_PARTIAL_LINE;
252 }
253 /* Proc */
254 if (tracer_flags.val & TRACE_GRAPH_PRINT_PROC) {
255 ret = print_graph_proc(s, pid);
256 if (ret == TRACE_TYPE_PARTIAL_LINE)
257 return TRACE_TYPE_PARTIAL_LINE;
258 382
259 ret = trace_seq_printf(s, " | "); 383 /* Cpu */
260 if (!ret) 384 if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU) {
261 return TRACE_TYPE_PARTIAL_LINE; 385 ret = print_graph_cpu(s, cpu);
262 } 386 if (ret == TRACE_TYPE_PARTIAL_LINE)
387 return TRACE_TYPE_PARTIAL_LINE;
388 }
389 /* Proc */
390 if (tracer_flags.val & TRACE_GRAPH_PRINT_PROC) {
391 ret = print_graph_proc(s, pid);
392 if (ret == TRACE_TYPE_PARTIAL_LINE)
393 return TRACE_TYPE_PARTIAL_LINE;
394 ret = trace_seq_printf(s, " | ");
395 if (!ret)
396 return TRACE_TYPE_PARTIAL_LINE;
397 }
263 398
264 /* No overhead */ 399 /* No overhead */
265 if (tracer_flags.val & TRACE_GRAPH_PRINT_OVERHEAD) { 400 ret = print_graph_overhead(-1, s);
266 ret = trace_seq_printf(s, " "); 401 if (!ret)
267 if (!ret) 402 return TRACE_TYPE_PARTIAL_LINE;
268 return TRACE_TYPE_PARTIAL_LINE; 403
269 } 404 if (type == TRACE_GRAPH_ENT)
405 ret = trace_seq_printf(s, "==========>");
406 else
407 ret = trace_seq_printf(s, "<==========");
408
409 if (!ret)
410 return TRACE_TYPE_PARTIAL_LINE;
411
412 /* Don't close the duration column if haven't one */
413 if (tracer_flags.val & TRACE_GRAPH_PRINT_DURATION)
414 trace_seq_printf(s, " |");
415 ret = trace_seq_printf(s, "\n");
270 416
271 ret = trace_seq_printf(s, "<========== |\n");
272 }
273 if (!ret) 417 if (!ret)
274 return TRACE_TYPE_PARTIAL_LINE; 418 return TRACE_TYPE_PARTIAL_LINE;
275 return TRACE_TYPE_HANDLED; 419 return TRACE_TYPE_HANDLED;
@@ -288,7 +432,7 @@ print_graph_duration(unsigned long long duration, struct trace_seq *s)
288 sprintf(msecs_str, "%lu", (unsigned long) duration); 432 sprintf(msecs_str, "%lu", (unsigned long) duration);
289 433
290 /* Print msecs */ 434 /* Print msecs */
291 ret = trace_seq_printf(s, msecs_str); 435 ret = trace_seq_printf(s, "%s", msecs_str);
292 if (!ret) 436 if (!ret)
293 return TRACE_TYPE_PARTIAL_LINE; 437 return TRACE_TYPE_PARTIAL_LINE;
294 438
@@ -321,51 +465,33 @@ print_graph_duration(unsigned long long duration, struct trace_seq *s)
321 465
322} 466}
323 467
324/* Signal a overhead of time execution to the output */
325static int
326print_graph_overhead(unsigned long long duration, struct trace_seq *s)
327{
328 /* Duration exceeded 100 msecs */
329 if (duration > 100000ULL)
330 return trace_seq_printf(s, "! ");
331
332 /* Duration exceeded 10 msecs */
333 if (duration > 10000ULL)
334 return trace_seq_printf(s, "+ ");
335
336 return trace_seq_printf(s, " ");
337}
338
339/* Case of a leaf function on its call entry */ 468/* Case of a leaf function on its call entry */
340static enum print_line_t 469static enum print_line_t
341print_graph_entry_leaf(struct trace_iterator *iter, 470print_graph_entry_leaf(struct trace_iterator *iter,
342 struct ftrace_graph_ent_entry *entry, struct trace_seq *s) 471 struct ftrace_graph_ent_entry *entry,
472 struct ftrace_graph_ret_entry *ret_entry, struct trace_seq *s)
343{ 473{
344 struct ftrace_graph_ret_entry *ret_entry;
345 struct ftrace_graph_ret *graph_ret; 474 struct ftrace_graph_ret *graph_ret;
346 struct ring_buffer_event *event;
347 struct ftrace_graph_ent *call; 475 struct ftrace_graph_ent *call;
348 unsigned long long duration; 476 unsigned long long duration;
349 int ret; 477 int ret;
350 int i; 478 int i;
351 479
352 event = ring_buffer_read(iter->buffer_iter[iter->cpu], NULL);
353 ret_entry = ring_buffer_event_data(event);
354 graph_ret = &ret_entry->ret; 480 graph_ret = &ret_entry->ret;
355 call = &entry->graph_ent; 481 call = &entry->graph_ent;
356 duration = graph_ret->rettime - graph_ret->calltime; 482 duration = graph_ret->rettime - graph_ret->calltime;
357 483
358 /* Overhead */ 484 /* Overhead */
359 if (tracer_flags.val & TRACE_GRAPH_PRINT_OVERHEAD) { 485 ret = print_graph_overhead(duration, s);
360 ret = print_graph_overhead(duration, s); 486 if (!ret)
361 if (!ret) 487 return TRACE_TYPE_PARTIAL_LINE;
362 return TRACE_TYPE_PARTIAL_LINE;
363 }
364 488
365 /* Duration */ 489 /* Duration */
366 ret = print_graph_duration(duration, s); 490 if (tracer_flags.val & TRACE_GRAPH_PRINT_DURATION) {
367 if (ret == TRACE_TYPE_PARTIAL_LINE) 491 ret = print_graph_duration(duration, s);
368 return TRACE_TYPE_PARTIAL_LINE; 492 if (ret == TRACE_TYPE_PARTIAL_LINE)
493 return TRACE_TYPE_PARTIAL_LINE;
494 }
369 495
370 /* Function */ 496 /* Function */
371 for (i = 0; i < call->depth * TRACE_GRAPH_INDENT; i++) { 497 for (i = 0; i < call->depth * TRACE_GRAPH_INDENT; i++) {
@@ -394,25 +520,17 @@ print_graph_entry_nested(struct ftrace_graph_ent_entry *entry,
394 struct ftrace_graph_ent *call = &entry->graph_ent; 520 struct ftrace_graph_ent *call = &entry->graph_ent;
395 521
396 /* No overhead */ 522 /* No overhead */
397 if (tracer_flags.val & TRACE_GRAPH_PRINT_OVERHEAD) { 523 ret = print_graph_overhead(-1, s);
398 ret = trace_seq_printf(s, " "); 524 if (!ret)
399 if (!ret) 525 return TRACE_TYPE_PARTIAL_LINE;
400 return TRACE_TYPE_PARTIAL_LINE;
401 }
402 526
403 /* Interrupt */ 527 /* No time */
404 ret = print_graph_irq(s, call->func, TRACE_GRAPH_ENT, cpu, pid); 528 if (tracer_flags.val & TRACE_GRAPH_PRINT_DURATION) {
405 if (ret == TRACE_TYPE_UNHANDLED) {
406 /* No time */
407 ret = trace_seq_printf(s, " | "); 529 ret = trace_seq_printf(s, " | ");
408 if (!ret) 530 if (!ret)
409 return TRACE_TYPE_PARTIAL_LINE; 531 return TRACE_TYPE_PARTIAL_LINE;
410 } else {
411 if (ret == TRACE_TYPE_PARTIAL_LINE)
412 return TRACE_TYPE_PARTIAL_LINE;
413 } 532 }
414 533
415
416 /* Function */ 534 /* Function */
417 for (i = 0; i < call->depth * TRACE_GRAPH_INDENT; i++) { 535 for (i = 0; i < call->depth * TRACE_GRAPH_INDENT; i++) {
418 ret = trace_seq_printf(s, " "); 536 ret = trace_seq_printf(s, " ");
@@ -428,20 +546,40 @@ print_graph_entry_nested(struct ftrace_graph_ent_entry *entry,
428 if (!ret) 546 if (!ret)
429 return TRACE_TYPE_PARTIAL_LINE; 547 return TRACE_TYPE_PARTIAL_LINE;
430 548
431 return TRACE_TYPE_HANDLED; 549 /*
550 * we already consumed the current entry to check the next one
551 * and see if this is a leaf.
552 */
553 return TRACE_TYPE_NO_CONSUME;
432} 554}
433 555
434static enum print_line_t 556static enum print_line_t
435print_graph_entry(struct ftrace_graph_ent_entry *field, struct trace_seq *s, 557print_graph_entry(struct ftrace_graph_ent_entry *field, struct trace_seq *s,
436 struct trace_iterator *iter, int cpu) 558 struct trace_iterator *iter)
437{ 559{
438 int ret; 560 int ret;
561 int cpu = iter->cpu;
562 pid_t *last_entry = iter->private;
439 struct trace_entry *ent = iter->ent; 563 struct trace_entry *ent = iter->ent;
564 struct ftrace_graph_ent *call = &field->graph_ent;
565 struct ftrace_graph_ret_entry *leaf_ret;
440 566
441 /* Pid */ 567 /* Pid */
442 if (verif_pid(s, ent->pid, cpu) == TRACE_TYPE_PARTIAL_LINE) 568 if (verif_pid(s, ent->pid, cpu, last_entry) == TRACE_TYPE_PARTIAL_LINE)
569 return TRACE_TYPE_PARTIAL_LINE;
570
571 /* Interrupt */
572 ret = print_graph_irq(iter, call->func, TRACE_GRAPH_ENT, cpu, ent->pid);
573 if (ret == TRACE_TYPE_PARTIAL_LINE)
443 return TRACE_TYPE_PARTIAL_LINE; 574 return TRACE_TYPE_PARTIAL_LINE;
444 575
576 /* Absolute time */
577 if (tracer_flags.val & TRACE_GRAPH_PRINT_ABS_TIME) {
578 ret = print_graph_abs_time(iter->ts, s);
579 if (!ret)
580 return TRACE_TYPE_PARTIAL_LINE;
581 }
582
445 /* Cpu */ 583 /* Cpu */
446 if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU) { 584 if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU) {
447 ret = print_graph_cpu(s, cpu); 585 ret = print_graph_cpu(s, cpu);
@@ -460,8 +598,9 @@ print_graph_entry(struct ftrace_graph_ent_entry *field, struct trace_seq *s,
460 return TRACE_TYPE_PARTIAL_LINE; 598 return TRACE_TYPE_PARTIAL_LINE;
461 } 599 }
462 600
463 if (trace_branch_is_leaf(iter, field)) 601 leaf_ret = get_return_for_leaf(iter, field);
464 return print_graph_entry_leaf(iter, field, s); 602 if (leaf_ret)
603 return print_graph_entry_leaf(iter, field, leaf_ret, s);
465 else 604 else
466 return print_graph_entry_nested(field, s, iter->ent->pid, cpu); 605 return print_graph_entry_nested(field, s, iter->ent->pid, cpu);
467 606
@@ -469,16 +608,25 @@ print_graph_entry(struct ftrace_graph_ent_entry *field, struct trace_seq *s,
469 608
470static enum print_line_t 609static enum print_line_t
471print_graph_return(struct ftrace_graph_ret *trace, struct trace_seq *s, 610print_graph_return(struct ftrace_graph_ret *trace, struct trace_seq *s,
472 struct trace_entry *ent, int cpu) 611 struct trace_entry *ent, struct trace_iterator *iter)
473{ 612{
474 int i; 613 int i;
475 int ret; 614 int ret;
615 int cpu = iter->cpu;
616 pid_t *last_pid = iter->private, pid = ent->pid;
476 unsigned long long duration = trace->rettime - trace->calltime; 617 unsigned long long duration = trace->rettime - trace->calltime;
477 618
478 /* Pid */ 619 /* Pid */
479 if (verif_pid(s, ent->pid, cpu) == TRACE_TYPE_PARTIAL_LINE) 620 if (verif_pid(s, pid, cpu, last_pid) == TRACE_TYPE_PARTIAL_LINE)
480 return TRACE_TYPE_PARTIAL_LINE; 621 return TRACE_TYPE_PARTIAL_LINE;
481 622
623 /* Absolute time */
624 if (tracer_flags.val & TRACE_GRAPH_PRINT_ABS_TIME) {
625 ret = print_graph_abs_time(iter->ts, s);
626 if (!ret)
627 return TRACE_TYPE_PARTIAL_LINE;
628 }
629
482 /* Cpu */ 630 /* Cpu */
483 if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU) { 631 if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU) {
484 ret = print_graph_cpu(s, cpu); 632 ret = print_graph_cpu(s, cpu);
@@ -498,16 +646,16 @@ print_graph_return(struct ftrace_graph_ret *trace, struct trace_seq *s,
498 } 646 }
499 647
500 /* Overhead */ 648 /* Overhead */
501 if (tracer_flags.val & TRACE_GRAPH_PRINT_OVERHEAD) { 649 ret = print_graph_overhead(duration, s);
502 ret = print_graph_overhead(duration, s); 650 if (!ret)
503 if (!ret) 651 return TRACE_TYPE_PARTIAL_LINE;
504 return TRACE_TYPE_PARTIAL_LINE;
505 }
506 652
507 /* Duration */ 653 /* Duration */
508 ret = print_graph_duration(duration, s); 654 if (tracer_flags.val & TRACE_GRAPH_PRINT_DURATION) {
509 if (ret == TRACE_TYPE_PARTIAL_LINE) 655 ret = print_graph_duration(duration, s);
510 return TRACE_TYPE_PARTIAL_LINE; 656 if (ret == TRACE_TYPE_PARTIAL_LINE)
657 return TRACE_TYPE_PARTIAL_LINE;
658 }
511 659
512 /* Closing brace */ 660 /* Closing brace */
513 for (i = 0; i < trace->depth * TRACE_GRAPH_INDENT; i++) { 661 for (i = 0; i < trace->depth * TRACE_GRAPH_INDENT; i++) {
@@ -528,7 +676,7 @@ print_graph_return(struct ftrace_graph_ret *trace, struct trace_seq *s,
528 return TRACE_TYPE_PARTIAL_LINE; 676 return TRACE_TYPE_PARTIAL_LINE;
529 } 677 }
530 678
531 ret = print_graph_irq(s, trace->func, TRACE_GRAPH_RET, cpu, ent->pid); 679 ret = print_graph_irq(iter, trace->func, TRACE_GRAPH_RET, cpu, pid);
532 if (ret == TRACE_TYPE_PARTIAL_LINE) 680 if (ret == TRACE_TYPE_PARTIAL_LINE)
533 return TRACE_TYPE_PARTIAL_LINE; 681 return TRACE_TYPE_PARTIAL_LINE;
534 682
@@ -541,14 +689,23 @@ print_graph_comment(struct print_entry *trace, struct trace_seq *s,
541{ 689{
542 int i; 690 int i;
543 int ret; 691 int ret;
692 int cpu = iter->cpu;
693 pid_t *last_pid = iter->private;
544 694
545 /* Pid */ 695 /* Pid */
546 if (verif_pid(s, ent->pid, iter->cpu) == TRACE_TYPE_PARTIAL_LINE) 696 if (verif_pid(s, ent->pid, cpu, last_pid) == TRACE_TYPE_PARTIAL_LINE)
547 return TRACE_TYPE_PARTIAL_LINE; 697 return TRACE_TYPE_PARTIAL_LINE;
548 698
699 /* Absolute time */
700 if (tracer_flags.val & TRACE_GRAPH_PRINT_ABS_TIME) {
701 ret = print_graph_abs_time(iter->ts, s);
702 if (!ret)
703 return TRACE_TYPE_PARTIAL_LINE;
704 }
705
549 /* Cpu */ 706 /* Cpu */
550 if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU) { 707 if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU) {
551 ret = print_graph_cpu(s, iter->cpu); 708 ret = print_graph_cpu(s, cpu);
552 if (ret == TRACE_TYPE_PARTIAL_LINE) 709 if (ret == TRACE_TYPE_PARTIAL_LINE)
553 return TRACE_TYPE_PARTIAL_LINE; 710 return TRACE_TYPE_PARTIAL_LINE;
554 } 711 }
@@ -565,17 +722,17 @@ print_graph_comment(struct print_entry *trace, struct trace_seq *s,
565 } 722 }
566 723
567 /* No overhead */ 724 /* No overhead */
568 if (tracer_flags.val & TRACE_GRAPH_PRINT_OVERHEAD) { 725 ret = print_graph_overhead(-1, s);
569 ret = trace_seq_printf(s, " "); 726 if (!ret)
727 return TRACE_TYPE_PARTIAL_LINE;
728
729 /* No time */
730 if (tracer_flags.val & TRACE_GRAPH_PRINT_DURATION) {
731 ret = trace_seq_printf(s, " | ");
570 if (!ret) 732 if (!ret)
571 return TRACE_TYPE_PARTIAL_LINE; 733 return TRACE_TYPE_PARTIAL_LINE;
572 } 734 }
573 735
574 /* No time */
575 ret = trace_seq_printf(s, " | ");
576 if (!ret)
577 return TRACE_TYPE_PARTIAL_LINE;
578
579 /* Indentation */ 736 /* Indentation */
580 if (trace->depth > 0) 737 if (trace->depth > 0)
581 for (i = 0; i < (trace->depth + 1) * TRACE_GRAPH_INDENT; i++) { 738 for (i = 0; i < (trace->depth + 1) * TRACE_GRAPH_INDENT; i++) {
@@ -585,12 +742,19 @@ print_graph_comment(struct print_entry *trace, struct trace_seq *s,
585 } 742 }
586 743
587 /* The comment */ 744 /* The comment */
588 ret = trace_seq_printf(s, "/* %s", trace->buf); 745 ret = trace_seq_printf(s, "/* ");
746 if (!ret)
747 return TRACE_TYPE_PARTIAL_LINE;
748
749 ret = trace_seq_bprintf(s, trace->fmt, trace->buf);
589 if (!ret) 750 if (!ret)
590 return TRACE_TYPE_PARTIAL_LINE; 751 return TRACE_TYPE_PARTIAL_LINE;
591 752
592 if (ent->flags & TRACE_FLAG_CONT) 753 /* Strip ending newline */
593 trace_seq_print_cont(s, iter); 754 if (s->buffer[s->len - 1] == '\n') {
755 s->buffer[s->len - 1] = '\0';
756 s->len--;
757 }
594 758
595 ret = trace_seq_printf(s, " */\n"); 759 ret = trace_seq_printf(s, " */\n");
596 if (!ret) 760 if (!ret)
@@ -610,13 +774,12 @@ print_graph_function(struct trace_iterator *iter)
610 case TRACE_GRAPH_ENT: { 774 case TRACE_GRAPH_ENT: {
611 struct ftrace_graph_ent_entry *field; 775 struct ftrace_graph_ent_entry *field;
612 trace_assign_type(field, entry); 776 trace_assign_type(field, entry);
613 return print_graph_entry(field, s, iter, 777 return print_graph_entry(field, s, iter);
614 iter->cpu);
615 } 778 }
616 case TRACE_GRAPH_RET: { 779 case TRACE_GRAPH_RET: {
617 struct ftrace_graph_ret_entry *field; 780 struct ftrace_graph_ret_entry *field;
618 trace_assign_type(field, entry); 781 trace_assign_type(field, entry);
619 return print_graph_return(&field->ret, s, entry, iter->cpu); 782 return print_graph_return(&field->ret, s, entry, iter);
620 } 783 }
621 case TRACE_PRINT: { 784 case TRACE_PRINT: {
622 struct print_entry *field; 785 struct print_entry *field;
@@ -632,33 +795,64 @@ static void print_graph_headers(struct seq_file *s)
632{ 795{
633 /* 1st line */ 796 /* 1st line */
634 seq_printf(s, "# "); 797 seq_printf(s, "# ");
798 if (tracer_flags.val & TRACE_GRAPH_PRINT_ABS_TIME)
799 seq_printf(s, " TIME ");
635 if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU) 800 if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU)
636 seq_printf(s, "CPU "); 801 seq_printf(s, "CPU");
637 if (tracer_flags.val & TRACE_GRAPH_PRINT_PROC) 802 if (tracer_flags.val & TRACE_GRAPH_PRINT_PROC)
638 seq_printf(s, "TASK/PID "); 803 seq_printf(s, " TASK/PID ");
639 if (tracer_flags.val & TRACE_GRAPH_PRINT_OVERHEAD) 804 if (tracer_flags.val & TRACE_GRAPH_PRINT_DURATION)
640 seq_printf(s, "OVERHEAD/"); 805 seq_printf(s, " DURATION ");
641 seq_printf(s, "DURATION FUNCTION CALLS\n"); 806 seq_printf(s, " FUNCTION CALLS\n");
642 807
643 /* 2nd line */ 808 /* 2nd line */
644 seq_printf(s, "# "); 809 seq_printf(s, "# ");
810 if (tracer_flags.val & TRACE_GRAPH_PRINT_ABS_TIME)
811 seq_printf(s, " | ");
645 if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU) 812 if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU)
646 seq_printf(s, "| "); 813 seq_printf(s, "| ");
647 if (tracer_flags.val & TRACE_GRAPH_PRINT_PROC) 814 if (tracer_flags.val & TRACE_GRAPH_PRINT_PROC)
648 seq_printf(s, "| | "); 815 seq_printf(s, " | | ");
649 if (tracer_flags.val & TRACE_GRAPH_PRINT_OVERHEAD) { 816 if (tracer_flags.val & TRACE_GRAPH_PRINT_DURATION)
650 seq_printf(s, "| "); 817 seq_printf(s, " | | ");
651 seq_printf(s, "| | | | |\n"); 818 seq_printf(s, " | | | |\n");
652 } else
653 seq_printf(s, " | | | | |\n");
654} 819}
820
821static void graph_trace_open(struct trace_iterator *iter)
822{
823 /* pid on the last trace processed */
824 pid_t *last_pid = alloc_percpu(pid_t);
825 int cpu;
826
827 if (!last_pid)
828 pr_warning("function graph tracer: not enough memory\n");
829 else
830 for_each_possible_cpu(cpu) {
831 pid_t *pid = per_cpu_ptr(last_pid, cpu);
832 *pid = -1;
833 }
834
835 iter->private = last_pid;
836}
837
838static void graph_trace_close(struct trace_iterator *iter)
839{
840 free_percpu(iter->private);
841}
842
655static struct tracer graph_trace __read_mostly = { 843static struct tracer graph_trace __read_mostly = {
656 .name = "function_graph", 844 .name = "function_graph",
845 .open = graph_trace_open,
846 .close = graph_trace_close,
847 .wait_pipe = poll_wait_pipe,
657 .init = graph_trace_init, 848 .init = graph_trace_init,
658 .reset = graph_trace_reset, 849 .reset = graph_trace_reset,
659 .print_line = print_graph_function, 850 .print_line = print_graph_function,
660 .print_header = print_graph_headers, 851 .print_header = print_graph_headers,
661 .flags = &tracer_flags, 852 .flags = &tracer_flags,
853#ifdef CONFIG_FTRACE_SELFTEST
854 .selftest = trace_selftest_startup_function_graph,
855#endif
662}; 856};
663 857
664static __init int init_graph_trace(void) 858static __init int init_graph_trace(void)
diff --git a/kernel/trace/trace_hw_branches.c b/kernel/trace/trace_hw_branches.c
index 649df22d435f..7bfdf4c2347f 100644
--- a/kernel/trace/trace_hw_branches.c
+++ b/kernel/trace/trace_hw_branches.c
@@ -1,30 +1,53 @@
1/* 1/*
2 * h/w branch tracer for x86 based on bts 2 * h/w branch tracer for x86 based on bts
3 * 3 *
4 * Copyright (C) 2008 Markus Metzger <markus.t.metzger@gmail.com> 4 * Copyright (C) 2008-2009 Intel Corporation.
5 * 5 * Markus Metzger <markus.t.metzger@gmail.com>, 2008-2009
6 */ 6 */
7 7#include <linux/spinlock.h>
8#include <linux/module.h> 8#include <linux/kallsyms.h>
9#include <linux/fs.h>
10#include <linux/debugfs.h> 9#include <linux/debugfs.h>
11#include <linux/ftrace.h> 10#include <linux/ftrace.h>
12#include <linux/kallsyms.h> 11#include <linux/module.h>
12#include <linux/cpu.h>
13#include <linux/smp.h>
14#include <linux/fs.h>
13 15
14#include <asm/ds.h> 16#include <asm/ds.h>
15 17
16#include "trace.h" 18#include "trace.h"
19#include "trace_output.h"
17 20
18 21
19#define SIZEOF_BTS (1 << 13) 22#define SIZEOF_BTS (1 << 13)
20 23
24/*
25 * The tracer lock protects the below per-cpu tracer array.
26 * It needs to be held to:
27 * - start tracing on all cpus
28 * - stop tracing on all cpus
29 * - start tracing on a single hotplug cpu
30 * - stop tracing on a single hotplug cpu
31 * - read the trace from all cpus
32 * - read the trace from a single cpu
33 */
34static DEFINE_SPINLOCK(bts_tracer_lock);
21static DEFINE_PER_CPU(struct bts_tracer *, tracer); 35static DEFINE_PER_CPU(struct bts_tracer *, tracer);
22static DEFINE_PER_CPU(unsigned char[SIZEOF_BTS], buffer); 36static DEFINE_PER_CPU(unsigned char[SIZEOF_BTS], buffer);
23 37
24#define this_tracer per_cpu(tracer, smp_processor_id()) 38#define this_tracer per_cpu(tracer, smp_processor_id())
25#define this_buffer per_cpu(buffer, smp_processor_id()) 39#define this_buffer per_cpu(buffer, smp_processor_id())
26 40
41static int __read_mostly trace_hw_branches_enabled;
42static struct trace_array *hw_branch_trace __read_mostly;
43
27 44
45/*
46 * Start tracing on the current cpu.
47 * The argument is ignored.
48 *
49 * pre: bts_tracer_lock must be locked.
50 */
28static void bts_trace_start_cpu(void *arg) 51static void bts_trace_start_cpu(void *arg)
29{ 52{
30 if (this_tracer) 53 if (this_tracer)
@@ -42,14 +65,20 @@ static void bts_trace_start_cpu(void *arg)
42 65
43static void bts_trace_start(struct trace_array *tr) 66static void bts_trace_start(struct trace_array *tr)
44{ 67{
45 int cpu; 68 spin_lock(&bts_tracer_lock);
46 69
47 tracing_reset_online_cpus(tr); 70 on_each_cpu(bts_trace_start_cpu, NULL, 1);
71 trace_hw_branches_enabled = 1;
48 72
49 for_each_cpu(cpu, cpu_possible_mask) 73 spin_unlock(&bts_tracer_lock);
50 smp_call_function_single(cpu, bts_trace_start_cpu, NULL, 1);
51} 74}
52 75
76/*
77 * Stop tracing on the current cpu.
78 * The argument is ignored.
79 *
80 * pre: bts_tracer_lock must be locked.
81 */
53static void bts_trace_stop_cpu(void *arg) 82static void bts_trace_stop_cpu(void *arg)
54{ 83{
55 if (this_tracer) { 84 if (this_tracer) {
@@ -60,26 +89,60 @@ static void bts_trace_stop_cpu(void *arg)
60 89
61static void bts_trace_stop(struct trace_array *tr) 90static void bts_trace_stop(struct trace_array *tr)
62{ 91{
63 int cpu; 92 spin_lock(&bts_tracer_lock);
93
94 trace_hw_branches_enabled = 0;
95 on_each_cpu(bts_trace_stop_cpu, NULL, 1);
96
97 spin_unlock(&bts_tracer_lock);
98}
99
100static int __cpuinit bts_hotcpu_handler(struct notifier_block *nfb,
101 unsigned long action, void *hcpu)
102{
103 unsigned int cpu = (unsigned long)hcpu;
64 104
65 for_each_cpu(cpu, cpu_possible_mask) 105 spin_lock(&bts_tracer_lock);
106
107 if (!trace_hw_branches_enabled)
108 goto out;
109
110 switch (action) {
111 case CPU_ONLINE:
112 case CPU_DOWN_FAILED:
113 smp_call_function_single(cpu, bts_trace_start_cpu, NULL, 1);
114 break;
115 case CPU_DOWN_PREPARE:
66 smp_call_function_single(cpu, bts_trace_stop_cpu, NULL, 1); 116 smp_call_function_single(cpu, bts_trace_stop_cpu, NULL, 1);
117 break;
118 }
119
120 out:
121 spin_unlock(&bts_tracer_lock);
122 return NOTIFY_DONE;
67} 123}
68 124
125static struct notifier_block bts_hotcpu_notifier __cpuinitdata = {
126 .notifier_call = bts_hotcpu_handler
127};
128
69static int bts_trace_init(struct trace_array *tr) 129static int bts_trace_init(struct trace_array *tr)
70{ 130{
71 tracing_reset_online_cpus(tr); 131 hw_branch_trace = tr;
132
72 bts_trace_start(tr); 133 bts_trace_start(tr);
73 134
74 return 0; 135 return 0;
75} 136}
76 137
138static void bts_trace_reset(struct trace_array *tr)
139{
140 bts_trace_stop(tr);
141}
142
77static void bts_trace_print_header(struct seq_file *m) 143static void bts_trace_print_header(struct seq_file *m)
78{ 144{
79 seq_puts(m, 145 seq_puts(m, "# CPU# TO <- FROM\n");
80 "# CPU# FROM TO FUNCTION\n");
81 seq_puts(m,
82 "# | | | |\n");
83} 146}
84 147
85static enum print_line_t bts_trace_print_line(struct trace_iterator *iter) 148static enum print_line_t bts_trace_print_line(struct trace_iterator *iter)
@@ -87,15 +150,15 @@ static enum print_line_t bts_trace_print_line(struct trace_iterator *iter)
87 struct trace_entry *entry = iter->ent; 150 struct trace_entry *entry = iter->ent;
88 struct trace_seq *seq = &iter->seq; 151 struct trace_seq *seq = &iter->seq;
89 struct hw_branch_entry *it; 152 struct hw_branch_entry *it;
153 unsigned long symflags = TRACE_ITER_SYM_OFFSET;
90 154
91 trace_assign_type(it, entry); 155 trace_assign_type(it, entry);
92 156
93 if (entry->type == TRACE_HW_BRANCHES) { 157 if (entry->type == TRACE_HW_BRANCHES) {
94 if (trace_seq_printf(seq, "%4d ", entry->cpu) && 158 if (trace_seq_printf(seq, "%4d ", iter->cpu) &&
95 trace_seq_printf(seq, "0x%016llx -> 0x%016llx ", 159 seq_print_ip_sym(seq, it->to, symflags) &&
96 it->from, it->to) && 160 trace_seq_printf(seq, "\t <- ") &&
97 (!it->from || 161 seq_print_ip_sym(seq, it->from, symflags) &&
98 seq_print_ip_sym(seq, it->from, /* sym_flags = */ 0)) &&
99 trace_seq_printf(seq, "\n")) 162 trace_seq_printf(seq, "\n"))
100 return TRACE_TYPE_HANDLED; 163 return TRACE_TYPE_HANDLED;
101 return TRACE_TYPE_PARTIAL_LINE;; 164 return TRACE_TYPE_PARTIAL_LINE;;
@@ -103,26 +166,42 @@ static enum print_line_t bts_trace_print_line(struct trace_iterator *iter)
103 return TRACE_TYPE_UNHANDLED; 166 return TRACE_TYPE_UNHANDLED;
104} 167}
105 168
106void trace_hw_branch(struct trace_array *tr, u64 from, u64 to) 169void trace_hw_branch(u64 from, u64 to)
107{ 170{
171 struct trace_array *tr = hw_branch_trace;
108 struct ring_buffer_event *event; 172 struct ring_buffer_event *event;
109 struct hw_branch_entry *entry; 173 struct hw_branch_entry *entry;
110 unsigned long irq; 174 unsigned long irq1;
175 int cpu;
111 176
112 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry), &irq); 177 if (unlikely(!tr))
113 if (!event)
114 return; 178 return;
179
180 if (unlikely(!trace_hw_branches_enabled))
181 return;
182
183 local_irq_save(irq1);
184 cpu = raw_smp_processor_id();
185 if (atomic_inc_return(&tr->data[cpu]->disabled) != 1)
186 goto out;
187
188 event = trace_buffer_lock_reserve(tr, TRACE_HW_BRANCHES,
189 sizeof(*entry), 0, 0);
190 if (!event)
191 goto out;
115 entry = ring_buffer_event_data(event); 192 entry = ring_buffer_event_data(event);
116 tracing_generic_entry_update(&entry->ent, 0, from); 193 tracing_generic_entry_update(&entry->ent, 0, from);
117 entry->ent.type = TRACE_HW_BRANCHES; 194 entry->ent.type = TRACE_HW_BRANCHES;
118 entry->ent.cpu = smp_processor_id();
119 entry->from = from; 195 entry->from = from;
120 entry->to = to; 196 entry->to = to;
121 ring_buffer_unlock_commit(tr->buffer, event, irq); 197 trace_buffer_unlock_commit(tr, event, 0, 0);
198
199 out:
200 atomic_dec(&tr->data[cpu]->disabled);
201 local_irq_restore(irq1);
122} 202}
123 203
124static void trace_bts_at(struct trace_array *tr, 204static void trace_bts_at(const struct bts_trace *trace, void *at)
125 const struct bts_trace *trace, void *at)
126{ 205{
127 struct bts_struct bts; 206 struct bts_struct bts;
128 int err = 0; 207 int err = 0;
@@ -137,18 +216,29 @@ static void trace_bts_at(struct trace_array *tr,
137 216
138 switch (bts.qualifier) { 217 switch (bts.qualifier) {
139 case BTS_BRANCH: 218 case BTS_BRANCH:
140 trace_hw_branch(tr, bts.variant.lbr.from, bts.variant.lbr.to); 219 trace_hw_branch(bts.variant.lbr.from, bts.variant.lbr.to);
141 break; 220 break;
142 } 221 }
143} 222}
144 223
224/*
225 * Collect the trace on the current cpu and write it into the ftrace buffer.
226 *
227 * pre: bts_tracer_lock must be locked
228 */
145static void trace_bts_cpu(void *arg) 229static void trace_bts_cpu(void *arg)
146{ 230{
147 struct trace_array *tr = (struct trace_array *) arg; 231 struct trace_array *tr = (struct trace_array *) arg;
148 const struct bts_trace *trace; 232 const struct bts_trace *trace;
149 unsigned char *at; 233 unsigned char *at;
150 234
151 if (!this_tracer) 235 if (unlikely(!tr))
236 return;
237
238 if (unlikely(atomic_read(&tr->data[raw_smp_processor_id()]->disabled)))
239 return;
240
241 if (unlikely(!this_tracer))
152 return; 242 return;
153 243
154 ds_suspend_bts(this_tracer); 244 ds_suspend_bts(this_tracer);
@@ -158,11 +248,11 @@ static void trace_bts_cpu(void *arg)
158 248
159 for (at = trace->ds.top; (void *)at < trace->ds.end; 249 for (at = trace->ds.top; (void *)at < trace->ds.end;
160 at += trace->ds.size) 250 at += trace->ds.size)
161 trace_bts_at(tr, trace, at); 251 trace_bts_at(trace, at);
162 252
163 for (at = trace->ds.begin; (void *)at < trace->ds.top; 253 for (at = trace->ds.begin; (void *)at < trace->ds.top;
164 at += trace->ds.size) 254 at += trace->ds.size)
165 trace_bts_at(tr, trace, at); 255 trace_bts_at(trace, at);
166 256
167out: 257out:
168 ds_resume_bts(this_tracer); 258 ds_resume_bts(this_tracer);
@@ -170,26 +260,43 @@ out:
170 260
171static void trace_bts_prepare(struct trace_iterator *iter) 261static void trace_bts_prepare(struct trace_iterator *iter)
172{ 262{
173 int cpu; 263 spin_lock(&bts_tracer_lock);
264
265 on_each_cpu(trace_bts_cpu, iter->tr, 1);
266
267 spin_unlock(&bts_tracer_lock);
268}
269
270static void trace_bts_close(struct trace_iterator *iter)
271{
272 tracing_reset_online_cpus(iter->tr);
273}
274
275void trace_hw_branch_oops(void)
276{
277 spin_lock(&bts_tracer_lock);
278
279 trace_bts_cpu(hw_branch_trace);
174 280
175 for_each_cpu(cpu, cpu_possible_mask) 281 spin_unlock(&bts_tracer_lock);
176 smp_call_function_single(cpu, trace_bts_cpu, iter->tr, 1);
177} 282}
178 283
179struct tracer bts_tracer __read_mostly = 284struct tracer bts_tracer __read_mostly =
180{ 285{
181 .name = "hw-branch-tracer", 286 .name = "hw-branch-tracer",
182 .init = bts_trace_init, 287 .init = bts_trace_init,
183 .reset = bts_trace_stop, 288 .reset = bts_trace_reset,
184 .print_header = bts_trace_print_header, 289 .print_header = bts_trace_print_header,
185 .print_line = bts_trace_print_line, 290 .print_line = bts_trace_print_line,
186 .start = bts_trace_start, 291 .start = bts_trace_start,
187 .stop = bts_trace_stop, 292 .stop = bts_trace_stop,
188 .open = trace_bts_prepare 293 .open = trace_bts_prepare,
294 .close = trace_bts_close
189}; 295};
190 296
191__init static int init_bts_trace(void) 297__init static int init_bts_trace(void)
192{ 298{
299 register_hotcpu_notifier(&bts_hotcpu_notifier);
193 return register_tracer(&bts_tracer); 300 return register_tracer(&bts_tracer);
194} 301}
195device_initcall(init_bts_trace); 302device_initcall(init_bts_trace);
diff --git a/kernel/trace/trace_irqsoff.c b/kernel/trace/trace_irqsoff.c
index 62a78d943534..b923d13e2fad 100644
--- a/kernel/trace/trace_irqsoff.c
+++ b/kernel/trace/trace_irqsoff.c
@@ -1,5 +1,5 @@
1/* 1/*
2 * trace irqs off criticall timings 2 * trace irqs off critical timings
3 * 3 *
4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> 4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com> 5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
@@ -32,6 +32,8 @@ enum {
32 32
33static int trace_type __read_mostly; 33static int trace_type __read_mostly;
34 34
35static int save_lat_flag;
36
35#ifdef CONFIG_PREEMPT_TRACER 37#ifdef CONFIG_PREEMPT_TRACER
36static inline int 38static inline int
37preempt_trace(void) 39preempt_trace(void)
@@ -95,7 +97,7 @@ irqsoff_tracer_call(unsigned long ip, unsigned long parent_ip)
95 disabled = atomic_inc_return(&data->disabled); 97 disabled = atomic_inc_return(&data->disabled);
96 98
97 if (likely(disabled == 1)) 99 if (likely(disabled == 1))
98 trace_function(tr, data, ip, parent_ip, flags, preempt_count()); 100 trace_function(tr, ip, parent_ip, flags, preempt_count());
99 101
100 atomic_dec(&data->disabled); 102 atomic_dec(&data->disabled);
101} 103}
@@ -153,7 +155,7 @@ check_critical_timing(struct trace_array *tr,
153 if (!report_latency(delta)) 155 if (!report_latency(delta))
154 goto out_unlock; 156 goto out_unlock;
155 157
156 trace_function(tr, data, CALLER_ADDR0, parent_ip, flags, pc); 158 trace_function(tr, CALLER_ADDR0, parent_ip, flags, pc);
157 159
158 latency = nsecs_to_usecs(delta); 160 latency = nsecs_to_usecs(delta);
159 161
@@ -177,7 +179,7 @@ out:
177 data->critical_sequence = max_sequence; 179 data->critical_sequence = max_sequence;
178 data->preempt_timestamp = ftrace_now(cpu); 180 data->preempt_timestamp = ftrace_now(cpu);
179 tracing_reset(tr, cpu); 181 tracing_reset(tr, cpu);
180 trace_function(tr, data, CALLER_ADDR0, parent_ip, flags, pc); 182 trace_function(tr, CALLER_ADDR0, parent_ip, flags, pc);
181} 183}
182 184
183static inline void 185static inline void
@@ -210,7 +212,7 @@ start_critical_timing(unsigned long ip, unsigned long parent_ip)
210 212
211 local_save_flags(flags); 213 local_save_flags(flags);
212 214
213 trace_function(tr, data, ip, parent_ip, flags, preempt_count()); 215 trace_function(tr, ip, parent_ip, flags, preempt_count());
214 216
215 per_cpu(tracing_cpu, cpu) = 1; 217 per_cpu(tracing_cpu, cpu) = 1;
216 218
@@ -244,7 +246,7 @@ stop_critical_timing(unsigned long ip, unsigned long parent_ip)
244 atomic_inc(&data->disabled); 246 atomic_inc(&data->disabled);
245 247
246 local_save_flags(flags); 248 local_save_flags(flags);
247 trace_function(tr, data, ip, parent_ip, flags, preempt_count()); 249 trace_function(tr, ip, parent_ip, flags, preempt_count());
248 check_critical_timing(tr, data, parent_ip ? : ip, cpu); 250 check_critical_timing(tr, data, parent_ip ? : ip, cpu);
249 data->critical_start = 0; 251 data->critical_start = 0;
250 atomic_dec(&data->disabled); 252 atomic_dec(&data->disabled);
@@ -353,33 +355,26 @@ void trace_preempt_off(unsigned long a0, unsigned long a1)
353} 355}
354#endif /* CONFIG_PREEMPT_TRACER */ 356#endif /* CONFIG_PREEMPT_TRACER */
355 357
356/*
357 * save_tracer_enabled is used to save the state of the tracer_enabled
358 * variable when we disable it when we open a trace output file.
359 */
360static int save_tracer_enabled;
361
362static void start_irqsoff_tracer(struct trace_array *tr) 358static void start_irqsoff_tracer(struct trace_array *tr)
363{ 359{
364 register_ftrace_function(&trace_ops); 360 register_ftrace_function(&trace_ops);
365 if (tracing_is_enabled()) { 361 if (tracing_is_enabled())
366 tracer_enabled = 1; 362 tracer_enabled = 1;
367 save_tracer_enabled = 1; 363 else
368 } else {
369 tracer_enabled = 0; 364 tracer_enabled = 0;
370 save_tracer_enabled = 0;
371 }
372} 365}
373 366
374static void stop_irqsoff_tracer(struct trace_array *tr) 367static void stop_irqsoff_tracer(struct trace_array *tr)
375{ 368{
376 tracer_enabled = 0; 369 tracer_enabled = 0;
377 save_tracer_enabled = 0;
378 unregister_ftrace_function(&trace_ops); 370 unregister_ftrace_function(&trace_ops);
379} 371}
380 372
381static void __irqsoff_tracer_init(struct trace_array *tr) 373static void __irqsoff_tracer_init(struct trace_array *tr)
382{ 374{
375 save_lat_flag = trace_flags & TRACE_ITER_LATENCY_FMT;
376 trace_flags |= TRACE_ITER_LATENCY_FMT;
377
383 tracing_max_latency = 0; 378 tracing_max_latency = 0;
384 irqsoff_trace = tr; 379 irqsoff_trace = tr;
385 /* make sure that the tracer is visible */ 380 /* make sure that the tracer is visible */
@@ -390,30 +385,19 @@ static void __irqsoff_tracer_init(struct trace_array *tr)
390static void irqsoff_tracer_reset(struct trace_array *tr) 385static void irqsoff_tracer_reset(struct trace_array *tr)
391{ 386{
392 stop_irqsoff_tracer(tr); 387 stop_irqsoff_tracer(tr);
388
389 if (!save_lat_flag)
390 trace_flags &= ~TRACE_ITER_LATENCY_FMT;
393} 391}
394 392
395static void irqsoff_tracer_start(struct trace_array *tr) 393static void irqsoff_tracer_start(struct trace_array *tr)
396{ 394{
397 tracer_enabled = 1; 395 tracer_enabled = 1;
398 save_tracer_enabled = 1;
399} 396}
400 397
401static void irqsoff_tracer_stop(struct trace_array *tr) 398static void irqsoff_tracer_stop(struct trace_array *tr)
402{ 399{
403 tracer_enabled = 0; 400 tracer_enabled = 0;
404 save_tracer_enabled = 0;
405}
406
407static void irqsoff_tracer_open(struct trace_iterator *iter)
408{
409 /* stop the trace while dumping */
410 tracer_enabled = 0;
411}
412
413static void irqsoff_tracer_close(struct trace_iterator *iter)
414{
415 /* restart tracing */
416 tracer_enabled = save_tracer_enabled;
417} 401}
418 402
419#ifdef CONFIG_IRQSOFF_TRACER 403#ifdef CONFIG_IRQSOFF_TRACER
@@ -431,8 +415,6 @@ static struct tracer irqsoff_tracer __read_mostly =
431 .reset = irqsoff_tracer_reset, 415 .reset = irqsoff_tracer_reset,
432 .start = irqsoff_tracer_start, 416 .start = irqsoff_tracer_start,
433 .stop = irqsoff_tracer_stop, 417 .stop = irqsoff_tracer_stop,
434 .open = irqsoff_tracer_open,
435 .close = irqsoff_tracer_close,
436 .print_max = 1, 418 .print_max = 1,
437#ifdef CONFIG_FTRACE_SELFTEST 419#ifdef CONFIG_FTRACE_SELFTEST
438 .selftest = trace_selftest_startup_irqsoff, 420 .selftest = trace_selftest_startup_irqsoff,
@@ -459,8 +441,6 @@ static struct tracer preemptoff_tracer __read_mostly =
459 .reset = irqsoff_tracer_reset, 441 .reset = irqsoff_tracer_reset,
460 .start = irqsoff_tracer_start, 442 .start = irqsoff_tracer_start,
461 .stop = irqsoff_tracer_stop, 443 .stop = irqsoff_tracer_stop,
462 .open = irqsoff_tracer_open,
463 .close = irqsoff_tracer_close,
464 .print_max = 1, 444 .print_max = 1,
465#ifdef CONFIG_FTRACE_SELFTEST 445#ifdef CONFIG_FTRACE_SELFTEST
466 .selftest = trace_selftest_startup_preemptoff, 446 .selftest = trace_selftest_startup_preemptoff,
@@ -489,8 +469,6 @@ static struct tracer preemptirqsoff_tracer __read_mostly =
489 .reset = irqsoff_tracer_reset, 469 .reset = irqsoff_tracer_reset,
490 .start = irqsoff_tracer_start, 470 .start = irqsoff_tracer_start,
491 .stop = irqsoff_tracer_stop, 471 .stop = irqsoff_tracer_stop,
492 .open = irqsoff_tracer_open,
493 .close = irqsoff_tracer_close,
494 .print_max = 1, 472 .print_max = 1,
495#ifdef CONFIG_FTRACE_SELFTEST 473#ifdef CONFIG_FTRACE_SELFTEST
496 .selftest = trace_selftest_startup_preemptirqsoff, 474 .selftest = trace_selftest_startup_preemptirqsoff,
diff --git a/kernel/trace/trace_mmiotrace.c b/kernel/trace/trace_mmiotrace.c
index 80e503ef6136..23e346a734ca 100644
--- a/kernel/trace/trace_mmiotrace.c
+++ b/kernel/trace/trace_mmiotrace.c
@@ -12,6 +12,7 @@
12#include <asm/atomic.h> 12#include <asm/atomic.h>
13 13
14#include "trace.h" 14#include "trace.h"
15#include "trace_output.h"
15 16
16struct header_iter { 17struct header_iter {
17 struct pci_dev *dev; 18 struct pci_dev *dev;
@@ -183,21 +184,22 @@ static enum print_line_t mmio_print_rw(struct trace_iterator *iter)
183 switch (rw->opcode) { 184 switch (rw->opcode) {
184 case MMIO_READ: 185 case MMIO_READ:
185 ret = trace_seq_printf(s, 186 ret = trace_seq_printf(s,
186 "R %d %lu.%06lu %d 0x%llx 0x%lx 0x%lx %d\n", 187 "R %d %u.%06lu %d 0x%llx 0x%lx 0x%lx %d\n",
187 rw->width, secs, usec_rem, rw->map_id, 188 rw->width, secs, usec_rem, rw->map_id,
188 (unsigned long long)rw->phys, 189 (unsigned long long)rw->phys,
189 rw->value, rw->pc, 0); 190 rw->value, rw->pc, 0);
190 break; 191 break;
191 case MMIO_WRITE: 192 case MMIO_WRITE:
192 ret = trace_seq_printf(s, 193 ret = trace_seq_printf(s,
193 "W %d %lu.%06lu %d 0x%llx 0x%lx 0x%lx %d\n", 194 "W %d %u.%06lu %d 0x%llx 0x%lx 0x%lx %d\n",
194 rw->width, secs, usec_rem, rw->map_id, 195 rw->width, secs, usec_rem, rw->map_id,
195 (unsigned long long)rw->phys, 196 (unsigned long long)rw->phys,
196 rw->value, rw->pc, 0); 197 rw->value, rw->pc, 0);
197 break; 198 break;
198 case MMIO_UNKNOWN_OP: 199 case MMIO_UNKNOWN_OP:
199 ret = trace_seq_printf(s, 200 ret = trace_seq_printf(s,
200 "UNKNOWN %lu.%06lu %d 0x%llx %02x,%02x,%02x 0x%lx %d\n", 201 "UNKNOWN %u.%06lu %d 0x%llx %02lx,%02lx,"
202 "%02lx 0x%lx %d\n",
201 secs, usec_rem, rw->map_id, 203 secs, usec_rem, rw->map_id,
202 (unsigned long long)rw->phys, 204 (unsigned long long)rw->phys,
203 (rw->value >> 16) & 0xff, (rw->value >> 8) & 0xff, 205 (rw->value >> 16) & 0xff, (rw->value >> 8) & 0xff,
@@ -229,14 +231,14 @@ static enum print_line_t mmio_print_map(struct trace_iterator *iter)
229 switch (m->opcode) { 231 switch (m->opcode) {
230 case MMIO_PROBE: 232 case MMIO_PROBE:
231 ret = trace_seq_printf(s, 233 ret = trace_seq_printf(s,
232 "MAP %lu.%06lu %d 0x%llx 0x%lx 0x%lx 0x%lx %d\n", 234 "MAP %u.%06lu %d 0x%llx 0x%lx 0x%lx 0x%lx %d\n",
233 secs, usec_rem, m->map_id, 235 secs, usec_rem, m->map_id,
234 (unsigned long long)m->phys, m->virt, m->len, 236 (unsigned long long)m->phys, m->virt, m->len,
235 0UL, 0); 237 0UL, 0);
236 break; 238 break;
237 case MMIO_UNPROBE: 239 case MMIO_UNPROBE:
238 ret = trace_seq_printf(s, 240 ret = trace_seq_printf(s,
239 "UNMAP %lu.%06lu %d 0x%lx %d\n", 241 "UNMAP %u.%06lu %d 0x%lx %d\n",
240 secs, usec_rem, m->map_id, 0UL, 0); 242 secs, usec_rem, m->map_id, 0UL, 0);
241 break; 243 break;
242 default: 244 default:
@@ -252,20 +254,20 @@ static enum print_line_t mmio_print_mark(struct trace_iterator *iter)
252{ 254{
253 struct trace_entry *entry = iter->ent; 255 struct trace_entry *entry = iter->ent;
254 struct print_entry *print = (struct print_entry *)entry; 256 struct print_entry *print = (struct print_entry *)entry;
255 const char *msg = print->buf;
256 struct trace_seq *s = &iter->seq; 257 struct trace_seq *s = &iter->seq;
257 unsigned long long t = ns2usecs(iter->ts); 258 unsigned long long t = ns2usecs(iter->ts);
258 unsigned long usec_rem = do_div(t, 1000000ULL); 259 unsigned long usec_rem = do_div(t, USEC_PER_SEC);
259 unsigned secs = (unsigned long)t; 260 unsigned secs = (unsigned long)t;
260 int ret; 261 int ret;
261 262
262 /* The trailing newline must be in the message. */ 263 /* The trailing newline must be in the message. */
263 ret = trace_seq_printf(s, "MARK %lu.%06lu %s", secs, usec_rem, msg); 264 ret = trace_seq_printf(s, "MARK %u.%06lu ", secs, usec_rem);
264 if (!ret) 265 if (!ret)
265 return TRACE_TYPE_PARTIAL_LINE; 266 return TRACE_TYPE_PARTIAL_LINE;
266 267
267 if (entry->flags & TRACE_FLAG_CONT) 268 ret = trace_seq_bprintf(s, print->fmt, print->buf);
268 trace_seq_print_cont(s, iter); 269 if (!ret)
270 return TRACE_TYPE_PARTIAL_LINE;
269 271
270 return TRACE_TYPE_HANDLED; 272 return TRACE_TYPE_HANDLED;
271} 273}
@@ -308,21 +310,17 @@ static void __trace_mmiotrace_rw(struct trace_array *tr,
308{ 310{
309 struct ring_buffer_event *event; 311 struct ring_buffer_event *event;
310 struct trace_mmiotrace_rw *entry; 312 struct trace_mmiotrace_rw *entry;
311 unsigned long irq_flags; 313 int pc = preempt_count();
312 314
313 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry), 315 event = trace_buffer_lock_reserve(tr, TRACE_MMIO_RW,
314 &irq_flags); 316 sizeof(*entry), 0, pc);
315 if (!event) { 317 if (!event) {
316 atomic_inc(&dropped_count); 318 atomic_inc(&dropped_count);
317 return; 319 return;
318 } 320 }
319 entry = ring_buffer_event_data(event); 321 entry = ring_buffer_event_data(event);
320 tracing_generic_entry_update(&entry->ent, 0, preempt_count());
321 entry->ent.type = TRACE_MMIO_RW;
322 entry->rw = *rw; 322 entry->rw = *rw;
323 ring_buffer_unlock_commit(tr->buffer, event, irq_flags); 323 trace_buffer_unlock_commit(tr, event, 0, pc);
324
325 trace_wake_up();
326} 324}
327 325
328void mmio_trace_rw(struct mmiotrace_rw *rw) 326void mmio_trace_rw(struct mmiotrace_rw *rw)
@@ -338,21 +336,17 @@ static void __trace_mmiotrace_map(struct trace_array *tr,
338{ 336{
339 struct ring_buffer_event *event; 337 struct ring_buffer_event *event;
340 struct trace_mmiotrace_map *entry; 338 struct trace_mmiotrace_map *entry;
341 unsigned long irq_flags; 339 int pc = preempt_count();
342 340
343 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry), 341 event = trace_buffer_lock_reserve(tr, TRACE_MMIO_MAP,
344 &irq_flags); 342 sizeof(*entry), 0, pc);
345 if (!event) { 343 if (!event) {
346 atomic_inc(&dropped_count); 344 atomic_inc(&dropped_count);
347 return; 345 return;
348 } 346 }
349 entry = ring_buffer_event_data(event); 347 entry = ring_buffer_event_data(event);
350 tracing_generic_entry_update(&entry->ent, 0, preempt_count());
351 entry->ent.type = TRACE_MMIO_MAP;
352 entry->map = *map; 348 entry->map = *map;
353 ring_buffer_unlock_commit(tr->buffer, event, irq_flags); 349 trace_buffer_unlock_commit(tr, event, 0, pc);
354
355 trace_wake_up();
356} 350}
357 351
358void mmio_trace_mapping(struct mmiotrace_map *map) 352void mmio_trace_mapping(struct mmiotrace_map *map)
diff --git a/kernel/trace/trace_nop.c b/kernel/trace/trace_nop.c
index b9767acd30ac..9aa84bde23cd 100644
--- a/kernel/trace/trace_nop.c
+++ b/kernel/trace/trace_nop.c
@@ -47,12 +47,7 @@ static void stop_nop_trace(struct trace_array *tr)
47 47
48static int nop_trace_init(struct trace_array *tr) 48static int nop_trace_init(struct trace_array *tr)
49{ 49{
50 int cpu;
51 ctx_trace = tr; 50 ctx_trace = tr;
52
53 for_each_online_cpu(cpu)
54 tracing_reset(tr, cpu);
55
56 start_nop_trace(tr); 51 start_nop_trace(tr);
57 return 0; 52 return 0;
58} 53}
diff --git a/kernel/trace/trace_output.c b/kernel/trace/trace_output.c
new file mode 100644
index 000000000000..ef8fd661b217
--- /dev/null
+++ b/kernel/trace/trace_output.c
@@ -0,0 +1,916 @@
1/*
2 * trace_output.c
3 *
4 * Copyright (C) 2008 Red Hat Inc, Steven Rostedt <srostedt@redhat.com>
5 *
6 */
7
8#include <linux/module.h>
9#include <linux/mutex.h>
10#include <linux/ftrace.h>
11
12#include "trace_output.h"
13
14/* must be a power of 2 */
15#define EVENT_HASHSIZE 128
16
17static DEFINE_MUTEX(trace_event_mutex);
18static struct hlist_head event_hash[EVENT_HASHSIZE] __read_mostly;
19
20static int next_event_type = __TRACE_LAST_TYPE + 1;
21
22/**
23 * trace_seq_printf - sequence printing of trace information
24 * @s: trace sequence descriptor
25 * @fmt: printf format string
26 *
27 * The tracer may use either sequence operations or its own
28 * copy to user routines. To simplify formating of a trace
29 * trace_seq_printf is used to store strings into a special
30 * buffer (@s). Then the output may be either used by
31 * the sequencer or pulled into another buffer.
32 */
33int
34trace_seq_printf(struct trace_seq *s, const char *fmt, ...)
35{
36 int len = (PAGE_SIZE - 1) - s->len;
37 va_list ap;
38 int ret;
39
40 if (!len)
41 return 0;
42
43 va_start(ap, fmt);
44 ret = vsnprintf(s->buffer + s->len, len, fmt, ap);
45 va_end(ap);
46
47 /* If we can't write it all, don't bother writing anything */
48 if (ret >= len)
49 return 0;
50
51 s->len += ret;
52
53 return len;
54}
55
56int trace_seq_bprintf(struct trace_seq *s, const char *fmt, const u32 *binary)
57{
58 int len = (PAGE_SIZE - 1) - s->len;
59 int ret;
60
61 if (!len)
62 return 0;
63
64 ret = bstr_printf(s->buffer + s->len, len, fmt, binary);
65
66 /* If we can't write it all, don't bother writing anything */
67 if (ret >= len)
68 return 0;
69
70 s->len += ret;
71
72 return len;
73}
74
75/**
76 * trace_seq_puts - trace sequence printing of simple string
77 * @s: trace sequence descriptor
78 * @str: simple string to record
79 *
80 * The tracer may use either the sequence operations or its own
81 * copy to user routines. This function records a simple string
82 * into a special buffer (@s) for later retrieval by a sequencer
83 * or other mechanism.
84 */
85int trace_seq_puts(struct trace_seq *s, const char *str)
86{
87 int len = strlen(str);
88
89 if (len > ((PAGE_SIZE - 1) - s->len))
90 return 0;
91
92 memcpy(s->buffer + s->len, str, len);
93 s->len += len;
94
95 return len;
96}
97
98int trace_seq_putc(struct trace_seq *s, unsigned char c)
99{
100 if (s->len >= (PAGE_SIZE - 1))
101 return 0;
102
103 s->buffer[s->len++] = c;
104
105 return 1;
106}
107
108int trace_seq_putmem(struct trace_seq *s, void *mem, size_t len)
109{
110 if (len > ((PAGE_SIZE - 1) - s->len))
111 return 0;
112
113 memcpy(s->buffer + s->len, mem, len);
114 s->len += len;
115
116 return len;
117}
118
119int trace_seq_putmem_hex(struct trace_seq *s, void *mem, size_t len)
120{
121 unsigned char hex[HEX_CHARS];
122 unsigned char *data = mem;
123 int i, j;
124
125#ifdef __BIG_ENDIAN
126 for (i = 0, j = 0; i < len; i++) {
127#else
128 for (i = len-1, j = 0; i >= 0; i--) {
129#endif
130 hex[j++] = hex_asc_hi(data[i]);
131 hex[j++] = hex_asc_lo(data[i]);
132 }
133 hex[j++] = ' ';
134
135 return trace_seq_putmem(s, hex, j);
136}
137
138int trace_seq_path(struct trace_seq *s, struct path *path)
139{
140 unsigned char *p;
141
142 if (s->len >= (PAGE_SIZE - 1))
143 return 0;
144 p = d_path(path, s->buffer + s->len, PAGE_SIZE - s->len);
145 if (!IS_ERR(p)) {
146 p = mangle_path(s->buffer + s->len, p, "\n");
147 if (p) {
148 s->len = p - s->buffer;
149 return 1;
150 }
151 } else {
152 s->buffer[s->len++] = '?';
153 return 1;
154 }
155
156 return 0;
157}
158
159#ifdef CONFIG_KRETPROBES
160static inline const char *kretprobed(const char *name)
161{
162 static const char tramp_name[] = "kretprobe_trampoline";
163 int size = sizeof(tramp_name);
164
165 if (strncmp(tramp_name, name, size) == 0)
166 return "[unknown/kretprobe'd]";
167 return name;
168}
169#else
170static inline const char *kretprobed(const char *name)
171{
172 return name;
173}
174#endif /* CONFIG_KRETPROBES */
175
176static int
177seq_print_sym_short(struct trace_seq *s, const char *fmt, unsigned long address)
178{
179#ifdef CONFIG_KALLSYMS
180 char str[KSYM_SYMBOL_LEN];
181 const char *name;
182
183 kallsyms_lookup(address, NULL, NULL, NULL, str);
184
185 name = kretprobed(str);
186
187 return trace_seq_printf(s, fmt, name);
188#endif
189 return 1;
190}
191
192static int
193seq_print_sym_offset(struct trace_seq *s, const char *fmt,
194 unsigned long address)
195{
196#ifdef CONFIG_KALLSYMS
197 char str[KSYM_SYMBOL_LEN];
198 const char *name;
199
200 sprint_symbol(str, address);
201 name = kretprobed(str);
202
203 return trace_seq_printf(s, fmt, name);
204#endif
205 return 1;
206}
207
208#ifndef CONFIG_64BIT
209# define IP_FMT "%08lx"
210#else
211# define IP_FMT "%016lx"
212#endif
213
214int seq_print_user_ip(struct trace_seq *s, struct mm_struct *mm,
215 unsigned long ip, unsigned long sym_flags)
216{
217 struct file *file = NULL;
218 unsigned long vmstart = 0;
219 int ret = 1;
220
221 if (mm) {
222 const struct vm_area_struct *vma;
223
224 down_read(&mm->mmap_sem);
225 vma = find_vma(mm, ip);
226 if (vma) {
227 file = vma->vm_file;
228 vmstart = vma->vm_start;
229 }
230 if (file) {
231 ret = trace_seq_path(s, &file->f_path);
232 if (ret)
233 ret = trace_seq_printf(s, "[+0x%lx]",
234 ip - vmstart);
235 }
236 up_read(&mm->mmap_sem);
237 }
238 if (ret && ((sym_flags & TRACE_ITER_SYM_ADDR) || !file))
239 ret = trace_seq_printf(s, " <" IP_FMT ">", ip);
240 return ret;
241}
242
243int
244seq_print_userip_objs(const struct userstack_entry *entry, struct trace_seq *s,
245 unsigned long sym_flags)
246{
247 struct mm_struct *mm = NULL;
248 int ret = 1;
249 unsigned int i;
250
251 if (trace_flags & TRACE_ITER_SYM_USEROBJ) {
252 struct task_struct *task;
253 /*
254 * we do the lookup on the thread group leader,
255 * since individual threads might have already quit!
256 */
257 rcu_read_lock();
258 task = find_task_by_vpid(entry->ent.tgid);
259 if (task)
260 mm = get_task_mm(task);
261 rcu_read_unlock();
262 }
263
264 for (i = 0; i < FTRACE_STACK_ENTRIES; i++) {
265 unsigned long ip = entry->caller[i];
266
267 if (ip == ULONG_MAX || !ret)
268 break;
269 if (i && ret)
270 ret = trace_seq_puts(s, " <- ");
271 if (!ip) {
272 if (ret)
273 ret = trace_seq_puts(s, "??");
274 continue;
275 }
276 if (!ret)
277 break;
278 if (ret)
279 ret = seq_print_user_ip(s, mm, ip, sym_flags);
280 }
281
282 if (mm)
283 mmput(mm);
284 return ret;
285}
286
287int
288seq_print_ip_sym(struct trace_seq *s, unsigned long ip, unsigned long sym_flags)
289{
290 int ret;
291
292 if (!ip)
293 return trace_seq_printf(s, "0");
294
295 if (sym_flags & TRACE_ITER_SYM_OFFSET)
296 ret = seq_print_sym_offset(s, "%s", ip);
297 else
298 ret = seq_print_sym_short(s, "%s", ip);
299
300 if (!ret)
301 return 0;
302
303 if (sym_flags & TRACE_ITER_SYM_ADDR)
304 ret = trace_seq_printf(s, " <" IP_FMT ">", ip);
305 return ret;
306}
307
308static int
309lat_print_generic(struct trace_seq *s, struct trace_entry *entry, int cpu)
310{
311 int hardirq, softirq;
312 char *comm;
313
314 comm = trace_find_cmdline(entry->pid);
315 hardirq = entry->flags & TRACE_FLAG_HARDIRQ;
316 softirq = entry->flags & TRACE_FLAG_SOFTIRQ;
317
318 if (!trace_seq_printf(s, "%8.8s-%-5d %3d%c%c%c",
319 comm, entry->pid, cpu,
320 (entry->flags & TRACE_FLAG_IRQS_OFF) ? 'd' :
321 (entry->flags & TRACE_FLAG_IRQS_NOSUPPORT) ?
322 'X' : '.',
323 (entry->flags & TRACE_FLAG_NEED_RESCHED) ?
324 'N' : '.',
325 (hardirq && softirq) ? 'H' :
326 hardirq ? 'h' : softirq ? 's' : '.'))
327 return 0;
328
329 if (entry->preempt_count)
330 return trace_seq_printf(s, "%x", entry->preempt_count);
331 return trace_seq_puts(s, ".");
332}
333
334static unsigned long preempt_mark_thresh = 100;
335
336static int
337lat_print_timestamp(struct trace_seq *s, u64 abs_usecs,
338 unsigned long rel_usecs)
339{
340 return trace_seq_printf(s, " %4lldus%c: ", abs_usecs,
341 rel_usecs > preempt_mark_thresh ? '!' :
342 rel_usecs > 1 ? '+' : ' ');
343}
344
345int trace_print_context(struct trace_iterator *iter)
346{
347 struct trace_seq *s = &iter->seq;
348 struct trace_entry *entry = iter->ent;
349 char *comm = trace_find_cmdline(entry->pid);
350 unsigned long long t = ns2usecs(iter->ts);
351 unsigned long usec_rem = do_div(t, USEC_PER_SEC);
352 unsigned long secs = (unsigned long)t;
353
354 return trace_seq_printf(s, "%16s-%-5d [%03d] %5lu.%06lu: ",
355 comm, entry->pid, iter->cpu, secs, usec_rem);
356}
357
358int trace_print_lat_context(struct trace_iterator *iter)
359{
360 u64 next_ts;
361 int ret;
362 struct trace_seq *s = &iter->seq;
363 struct trace_entry *entry = iter->ent,
364 *next_entry = trace_find_next_entry(iter, NULL,
365 &next_ts);
366 unsigned long verbose = (trace_flags & TRACE_ITER_VERBOSE);
367 unsigned long abs_usecs = ns2usecs(iter->ts - iter->tr->time_start);
368 unsigned long rel_usecs;
369
370 if (!next_entry)
371 next_ts = iter->ts;
372 rel_usecs = ns2usecs(next_ts - iter->ts);
373
374 if (verbose) {
375 char *comm = trace_find_cmdline(entry->pid);
376 ret = trace_seq_printf(s, "%16s %5d %3d %d %08x %08lx [%08lx]"
377 " %ld.%03ldms (+%ld.%03ldms): ", comm,
378 entry->pid, iter->cpu, entry->flags,
379 entry->preempt_count, iter->idx,
380 ns2usecs(iter->ts),
381 abs_usecs / USEC_PER_MSEC,
382 abs_usecs % USEC_PER_MSEC,
383 rel_usecs / USEC_PER_MSEC,
384 rel_usecs % USEC_PER_MSEC);
385 } else {
386 ret = lat_print_generic(s, entry, iter->cpu);
387 if (ret)
388 ret = lat_print_timestamp(s, abs_usecs, rel_usecs);
389 }
390
391 return ret;
392}
393
394static const char state_to_char[] = TASK_STATE_TO_CHAR_STR;
395
396static int task_state_char(unsigned long state)
397{
398 int bit = state ? __ffs(state) + 1 : 0;
399
400 return bit < sizeof(state_to_char) - 1 ? state_to_char[bit] : '?';
401}
402
403/**
404 * ftrace_find_event - find a registered event
405 * @type: the type of event to look for
406 *
407 * Returns an event of type @type otherwise NULL
408 */
409struct trace_event *ftrace_find_event(int type)
410{
411 struct trace_event *event;
412 struct hlist_node *n;
413 unsigned key;
414
415 key = type & (EVENT_HASHSIZE - 1);
416
417 hlist_for_each_entry_rcu(event, n, &event_hash[key], node) {
418 if (event->type == type)
419 return event;
420 }
421
422 return NULL;
423}
424
425/**
426 * register_ftrace_event - register output for an event type
427 * @event: the event type to register
428 *
429 * Event types are stored in a hash and this hash is used to
430 * find a way to print an event. If the @event->type is set
431 * then it will use that type, otherwise it will assign a
432 * type to use.
433 *
434 * If you assign your own type, please make sure it is added
435 * to the trace_type enum in trace.h, to avoid collisions
436 * with the dynamic types.
437 *
438 * Returns the event type number or zero on error.
439 */
440int register_ftrace_event(struct trace_event *event)
441{
442 unsigned key;
443 int ret = 0;
444
445 mutex_lock(&trace_event_mutex);
446
447 if (!event->type)
448 event->type = next_event_type++;
449 else if (event->type > __TRACE_LAST_TYPE) {
450 printk(KERN_WARNING "Need to add type to trace.h\n");
451 WARN_ON(1);
452 }
453
454 if (ftrace_find_event(event->type))
455 goto out;
456
457 if (event->trace == NULL)
458 event->trace = trace_nop_print;
459 if (event->raw == NULL)
460 event->raw = trace_nop_print;
461 if (event->hex == NULL)
462 event->hex = trace_nop_print;
463 if (event->binary == NULL)
464 event->binary = trace_nop_print;
465
466 key = event->type & (EVENT_HASHSIZE - 1);
467
468 hlist_add_head_rcu(&event->node, &event_hash[key]);
469
470 ret = event->type;
471 out:
472 mutex_unlock(&trace_event_mutex);
473
474 return ret;
475}
476
477/**
478 * unregister_ftrace_event - remove a no longer used event
479 * @event: the event to remove
480 */
481int unregister_ftrace_event(struct trace_event *event)
482{
483 mutex_lock(&trace_event_mutex);
484 hlist_del(&event->node);
485 mutex_unlock(&trace_event_mutex);
486
487 return 0;
488}
489
490/*
491 * Standard events
492 */
493
494enum print_line_t trace_nop_print(struct trace_iterator *iter, int flags)
495{
496 return TRACE_TYPE_HANDLED;
497}
498
499/* TRACE_FN */
500static enum print_line_t trace_fn_trace(struct trace_iterator *iter, int flags)
501{
502 struct ftrace_entry *field;
503 struct trace_seq *s = &iter->seq;
504
505 trace_assign_type(field, iter->ent);
506
507 if (!seq_print_ip_sym(s, field->ip, flags))
508 goto partial;
509
510 if ((flags & TRACE_ITER_PRINT_PARENT) && field->parent_ip) {
511 if (!trace_seq_printf(s, " <-"))
512 goto partial;
513 if (!seq_print_ip_sym(s,
514 field->parent_ip,
515 flags))
516 goto partial;
517 }
518 if (!trace_seq_printf(s, "\n"))
519 goto partial;
520
521 return TRACE_TYPE_HANDLED;
522
523 partial:
524 return TRACE_TYPE_PARTIAL_LINE;
525}
526
527static enum print_line_t trace_fn_raw(struct trace_iterator *iter, int flags)
528{
529 struct ftrace_entry *field;
530
531 trace_assign_type(field, iter->ent);
532
533 if (!trace_seq_printf(&iter->seq, "%lx %lx\n",
534 field->ip,
535 field->parent_ip))
536 return TRACE_TYPE_PARTIAL_LINE;
537
538 return TRACE_TYPE_HANDLED;
539}
540
541static enum print_line_t trace_fn_hex(struct trace_iterator *iter, int flags)
542{
543 struct ftrace_entry *field;
544 struct trace_seq *s = &iter->seq;
545
546 trace_assign_type(field, iter->ent);
547
548 SEQ_PUT_HEX_FIELD_RET(s, field->ip);
549 SEQ_PUT_HEX_FIELD_RET(s, field->parent_ip);
550
551 return TRACE_TYPE_HANDLED;
552}
553
554static enum print_line_t trace_fn_bin(struct trace_iterator *iter, int flags)
555{
556 struct ftrace_entry *field;
557 struct trace_seq *s = &iter->seq;
558
559 trace_assign_type(field, iter->ent);
560
561 SEQ_PUT_FIELD_RET(s, field->ip);
562 SEQ_PUT_FIELD_RET(s, field->parent_ip);
563
564 return TRACE_TYPE_HANDLED;
565}
566
567static struct trace_event trace_fn_event = {
568 .type = TRACE_FN,
569 .trace = trace_fn_trace,
570 .raw = trace_fn_raw,
571 .hex = trace_fn_hex,
572 .binary = trace_fn_bin,
573};
574
575/* TRACE_CTX an TRACE_WAKE */
576static enum print_line_t trace_ctxwake_print(struct trace_iterator *iter,
577 char *delim)
578{
579 struct ctx_switch_entry *field;
580 char *comm;
581 int S, T;
582
583 trace_assign_type(field, iter->ent);
584
585 T = task_state_char(field->next_state);
586 S = task_state_char(field->prev_state);
587 comm = trace_find_cmdline(field->next_pid);
588 if (!trace_seq_printf(&iter->seq,
589 " %5d:%3d:%c %s [%03d] %5d:%3d:%c %s\n",
590 field->prev_pid,
591 field->prev_prio,
592 S, delim,
593 field->next_cpu,
594 field->next_pid,
595 field->next_prio,
596 T, comm))
597 return TRACE_TYPE_PARTIAL_LINE;
598
599 return TRACE_TYPE_HANDLED;
600}
601
602static enum print_line_t trace_ctx_print(struct trace_iterator *iter, int flags)
603{
604 return trace_ctxwake_print(iter, "==>");
605}
606
607static enum print_line_t trace_wake_print(struct trace_iterator *iter,
608 int flags)
609{
610 return trace_ctxwake_print(iter, " +");
611}
612
613static int trace_ctxwake_raw(struct trace_iterator *iter, char S)
614{
615 struct ctx_switch_entry *field;
616 int T;
617
618 trace_assign_type(field, iter->ent);
619
620 if (!S)
621 task_state_char(field->prev_state);
622 T = task_state_char(field->next_state);
623 if (!trace_seq_printf(&iter->seq, "%d %d %c %d %d %d %c\n",
624 field->prev_pid,
625 field->prev_prio,
626 S,
627 field->next_cpu,
628 field->next_pid,
629 field->next_prio,
630 T))
631 return TRACE_TYPE_PARTIAL_LINE;
632
633 return TRACE_TYPE_HANDLED;
634}
635
636static enum print_line_t trace_ctx_raw(struct trace_iterator *iter, int flags)
637{
638 return trace_ctxwake_raw(iter, 0);
639}
640
641static enum print_line_t trace_wake_raw(struct trace_iterator *iter, int flags)
642{
643 return trace_ctxwake_raw(iter, '+');
644}
645
646
647static int trace_ctxwake_hex(struct trace_iterator *iter, char S)
648{
649 struct ctx_switch_entry *field;
650 struct trace_seq *s = &iter->seq;
651 int T;
652
653 trace_assign_type(field, iter->ent);
654
655 if (!S)
656 task_state_char(field->prev_state);
657 T = task_state_char(field->next_state);
658
659 SEQ_PUT_HEX_FIELD_RET(s, field->prev_pid);
660 SEQ_PUT_HEX_FIELD_RET(s, field->prev_prio);
661 SEQ_PUT_HEX_FIELD_RET(s, S);
662 SEQ_PUT_HEX_FIELD_RET(s, field->next_cpu);
663 SEQ_PUT_HEX_FIELD_RET(s, field->next_pid);
664 SEQ_PUT_HEX_FIELD_RET(s, field->next_prio);
665 SEQ_PUT_HEX_FIELD_RET(s, T);
666
667 return TRACE_TYPE_HANDLED;
668}
669
670static enum print_line_t trace_ctx_hex(struct trace_iterator *iter, int flags)
671{
672 return trace_ctxwake_hex(iter, 0);
673}
674
675static enum print_line_t trace_wake_hex(struct trace_iterator *iter, int flags)
676{
677 return trace_ctxwake_hex(iter, '+');
678}
679
680static enum print_line_t trace_ctxwake_bin(struct trace_iterator *iter,
681 int flags)
682{
683 struct ctx_switch_entry *field;
684 struct trace_seq *s = &iter->seq;
685
686 trace_assign_type(field, iter->ent);
687
688 SEQ_PUT_FIELD_RET(s, field->prev_pid);
689 SEQ_PUT_FIELD_RET(s, field->prev_prio);
690 SEQ_PUT_FIELD_RET(s, field->prev_state);
691 SEQ_PUT_FIELD_RET(s, field->next_pid);
692 SEQ_PUT_FIELD_RET(s, field->next_prio);
693 SEQ_PUT_FIELD_RET(s, field->next_state);
694
695 return TRACE_TYPE_HANDLED;
696}
697
698static struct trace_event trace_ctx_event = {
699 .type = TRACE_CTX,
700 .trace = trace_ctx_print,
701 .raw = trace_ctx_raw,
702 .hex = trace_ctx_hex,
703 .binary = trace_ctxwake_bin,
704};
705
706static struct trace_event trace_wake_event = {
707 .type = TRACE_WAKE,
708 .trace = trace_wake_print,
709 .raw = trace_wake_raw,
710 .hex = trace_wake_hex,
711 .binary = trace_ctxwake_bin,
712};
713
714/* TRACE_SPECIAL */
715static enum print_line_t trace_special_print(struct trace_iterator *iter,
716 int flags)
717{
718 struct special_entry *field;
719
720 trace_assign_type(field, iter->ent);
721
722 if (!trace_seq_printf(&iter->seq, "# %ld %ld %ld\n",
723 field->arg1,
724 field->arg2,
725 field->arg3))
726 return TRACE_TYPE_PARTIAL_LINE;
727
728 return TRACE_TYPE_HANDLED;
729}
730
731static enum print_line_t trace_special_hex(struct trace_iterator *iter,
732 int flags)
733{
734 struct special_entry *field;
735 struct trace_seq *s = &iter->seq;
736
737 trace_assign_type(field, iter->ent);
738
739 SEQ_PUT_HEX_FIELD_RET(s, field->arg1);
740 SEQ_PUT_HEX_FIELD_RET(s, field->arg2);
741 SEQ_PUT_HEX_FIELD_RET(s, field->arg3);
742
743 return TRACE_TYPE_HANDLED;
744}
745
746static enum print_line_t trace_special_bin(struct trace_iterator *iter,
747 int flags)
748{
749 struct special_entry *field;
750 struct trace_seq *s = &iter->seq;
751
752 trace_assign_type(field, iter->ent);
753
754 SEQ_PUT_FIELD_RET(s, field->arg1);
755 SEQ_PUT_FIELD_RET(s, field->arg2);
756 SEQ_PUT_FIELD_RET(s, field->arg3);
757
758 return TRACE_TYPE_HANDLED;
759}
760
761static struct trace_event trace_special_event = {
762 .type = TRACE_SPECIAL,
763 .trace = trace_special_print,
764 .raw = trace_special_print,
765 .hex = trace_special_hex,
766 .binary = trace_special_bin,
767};
768
769/* TRACE_STACK */
770
771static enum print_line_t trace_stack_print(struct trace_iterator *iter,
772 int flags)
773{
774 struct stack_entry *field;
775 struct trace_seq *s = &iter->seq;
776 int i;
777
778 trace_assign_type(field, iter->ent);
779
780 for (i = 0; i < FTRACE_STACK_ENTRIES; i++) {
781 if (i) {
782 if (!trace_seq_puts(s, " <= "))
783 goto partial;
784
785 if (!seq_print_ip_sym(s, field->caller[i], flags))
786 goto partial;
787 }
788 if (!trace_seq_puts(s, "\n"))
789 goto partial;
790 }
791
792 return TRACE_TYPE_HANDLED;
793
794 partial:
795 return TRACE_TYPE_PARTIAL_LINE;
796}
797
798static struct trace_event trace_stack_event = {
799 .type = TRACE_STACK,
800 .trace = trace_stack_print,
801 .raw = trace_special_print,
802 .hex = trace_special_hex,
803 .binary = trace_special_bin,
804};
805
806/* TRACE_USER_STACK */
807static enum print_line_t trace_user_stack_print(struct trace_iterator *iter,
808 int flags)
809{
810 struct userstack_entry *field;
811 struct trace_seq *s = &iter->seq;
812
813 trace_assign_type(field, iter->ent);
814
815 if (!seq_print_userip_objs(field, s, flags))
816 goto partial;
817
818 if (!trace_seq_putc(s, '\n'))
819 goto partial;
820
821 return TRACE_TYPE_HANDLED;
822
823 partial:
824 return TRACE_TYPE_PARTIAL_LINE;
825}
826
827static struct trace_event trace_user_stack_event = {
828 .type = TRACE_USER_STACK,
829 .trace = trace_user_stack_print,
830 .raw = trace_special_print,
831 .hex = trace_special_hex,
832 .binary = trace_special_bin,
833};
834
835/* TRACE_PRINT */
836static enum print_line_t
837trace_print_print(struct trace_iterator *iter, int flags)
838{
839 struct trace_entry *entry = iter->ent;
840 struct trace_seq *s = &iter->seq;
841 struct print_entry *field;
842
843 trace_assign_type(field, entry);
844
845 if (!seq_print_ip_sym(s, field->ip, flags))
846 goto partial;
847
848 if (!trace_seq_puts(s, ": "))
849 goto partial;
850
851 if (!trace_seq_bprintf(s, field->fmt, field->buf))
852 goto partial;
853
854 return TRACE_TYPE_HANDLED;
855
856 partial:
857 return TRACE_TYPE_PARTIAL_LINE;
858}
859
860
861static enum print_line_t trace_print_raw(struct trace_iterator *iter, int flags)
862{
863 struct print_entry *field;
864 struct trace_seq *s = &iter->seq;
865
866 trace_assign_type(field, iter->ent);
867
868 if (!trace_seq_printf(s, ": %lx : ", field->ip))
869 goto partial;
870
871 if (!trace_seq_bprintf(s, field->fmt, field->buf))
872 goto partial;
873
874 return TRACE_TYPE_HANDLED;
875
876 partial:
877 return TRACE_TYPE_PARTIAL_LINE;
878}
879
880
881static struct trace_event trace_print_event = {
882 .type = TRACE_PRINT,
883 .trace = trace_print_print,
884 .raw = trace_print_raw,
885};
886
887static struct trace_event *events[] __initdata = {
888 &trace_fn_event,
889 &trace_ctx_event,
890 &trace_wake_event,
891 &trace_special_event,
892 &trace_stack_event,
893 &trace_user_stack_event,
894 &trace_print_event,
895 NULL
896};
897
898__init static int init_events(void)
899{
900 struct trace_event *event;
901 int i, ret;
902
903 for (i = 0; events[i]; i++) {
904 event = events[i];
905
906 ret = register_ftrace_event(event);
907 if (!ret) {
908 printk(KERN_WARNING "event %d failed to register\n",
909 event->type);
910 WARN_ON_ONCE(1);
911 }
912 }
913
914 return 0;
915}
916device_initcall(init_events);
diff --git a/kernel/trace/trace_output.h b/kernel/trace/trace_output.h
new file mode 100644
index 000000000000..3b90e6ade1aa
--- /dev/null
+++ b/kernel/trace/trace_output.h
@@ -0,0 +1,63 @@
1#ifndef __TRACE_EVENTS_H
2#define __TRACE_EVENTS_H
3
4#include "trace.h"
5
6typedef enum print_line_t (*trace_print_func)(struct trace_iterator *iter,
7 int flags);
8
9struct trace_event {
10 struct hlist_node node;
11 int type;
12 trace_print_func trace;
13 trace_print_func raw;
14 trace_print_func hex;
15 trace_print_func binary;
16};
17
18extern int trace_seq_printf(struct trace_seq *s, const char *fmt, ...)
19 __attribute__ ((format (printf, 2, 3)));
20extern int
21trace_seq_bprintf(struct trace_seq *s, const char *fmt, const u32 *binary);
22extern int
23seq_print_ip_sym(struct trace_seq *s, unsigned long ip,
24 unsigned long sym_flags);
25extern ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf,
26 size_t cnt);
27int trace_seq_puts(struct trace_seq *s, const char *str);
28int trace_seq_putc(struct trace_seq *s, unsigned char c);
29int trace_seq_putmem(struct trace_seq *s, void *mem, size_t len);
30int trace_seq_putmem_hex(struct trace_seq *s, void *mem, size_t len);
31int trace_seq_path(struct trace_seq *s, struct path *path);
32int seq_print_userip_objs(const struct userstack_entry *entry,
33 struct trace_seq *s, unsigned long sym_flags);
34int seq_print_user_ip(struct trace_seq *s, struct mm_struct *mm,
35 unsigned long ip, unsigned long sym_flags);
36
37int trace_print_context(struct trace_iterator *iter);
38int trace_print_lat_context(struct trace_iterator *iter);
39
40struct trace_event *ftrace_find_event(int type);
41int register_ftrace_event(struct trace_event *event);
42int unregister_ftrace_event(struct trace_event *event);
43
44enum print_line_t trace_nop_print(struct trace_iterator *iter, int flags);
45
46#define MAX_MEMHEX_BYTES 8
47#define HEX_CHARS (MAX_MEMHEX_BYTES*2 + 1)
48
49#define SEQ_PUT_FIELD_RET(s, x) \
50do { \
51 if (!trace_seq_putmem(s, &(x), sizeof(x))) \
52 return TRACE_TYPE_PARTIAL_LINE; \
53} while (0)
54
55#define SEQ_PUT_HEX_FIELD_RET(s, x) \
56do { \
57 BUILD_BUG_ON(sizeof(x) > MAX_MEMHEX_BYTES); \
58 if (!trace_seq_putmem_hex(s, &(x), sizeof(x))) \
59 return TRACE_TYPE_PARTIAL_LINE; \
60} while (0)
61
62#endif
63
diff --git a/kernel/trace/trace_power.c b/kernel/trace/trace_power.c
index 7bda248daf55..91ce672fb037 100644
--- a/kernel/trace/trace_power.c
+++ b/kernel/trace/trace_power.c
@@ -11,24 +11,126 @@
11 11
12#include <linux/init.h> 12#include <linux/init.h>
13#include <linux/debugfs.h> 13#include <linux/debugfs.h>
14#include <linux/ftrace.h> 14#include <trace/power.h>
15#include <linux/kallsyms.h> 15#include <linux/kallsyms.h>
16#include <linux/module.h> 16#include <linux/module.h>
17 17
18#include "trace.h" 18#include "trace.h"
19#include "trace_output.h"
19 20
20static struct trace_array *power_trace; 21static struct trace_array *power_trace;
21static int __read_mostly trace_power_enabled; 22static int __read_mostly trace_power_enabled;
22 23
24static void probe_power_start(struct power_trace *it, unsigned int type,
25 unsigned int level)
26{
27 if (!trace_power_enabled)
28 return;
29
30 memset(it, 0, sizeof(struct power_trace));
31 it->state = level;
32 it->type = type;
33 it->stamp = ktime_get();
34}
35
36
37static void probe_power_end(struct power_trace *it)
38{
39 struct ring_buffer_event *event;
40 struct trace_power *entry;
41 struct trace_array_cpu *data;
42 struct trace_array *tr = power_trace;
43
44 if (!trace_power_enabled)
45 return;
46
47 preempt_disable();
48 it->end = ktime_get();
49 data = tr->data[smp_processor_id()];
50
51 event = trace_buffer_lock_reserve(tr, TRACE_POWER,
52 sizeof(*entry), 0, 0);
53 if (!event)
54 goto out;
55 entry = ring_buffer_event_data(event);
56 entry->state_data = *it;
57 trace_buffer_unlock_commit(tr, event, 0, 0);
58 out:
59 preempt_enable();
60}
61
62static void probe_power_mark(struct power_trace *it, unsigned int type,
63 unsigned int level)
64{
65 struct ring_buffer_event *event;
66 struct trace_power *entry;
67 struct trace_array_cpu *data;
68 struct trace_array *tr = power_trace;
69
70 if (!trace_power_enabled)
71 return;
72
73 memset(it, 0, sizeof(struct power_trace));
74 it->state = level;
75 it->type = type;
76 it->stamp = ktime_get();
77 preempt_disable();
78 it->end = it->stamp;
79 data = tr->data[smp_processor_id()];
80
81 event = trace_buffer_lock_reserve(tr, TRACE_POWER,
82 sizeof(*entry), 0, 0);
83 if (!event)
84 goto out;
85 entry = ring_buffer_event_data(event);
86 entry->state_data = *it;
87 trace_buffer_unlock_commit(tr, event, 0, 0);
88 out:
89 preempt_enable();
90}
91
92static int tracing_power_register(void)
93{
94 int ret;
95
96 ret = register_trace_power_start(probe_power_start);
97 if (ret) {
98 pr_info("power trace: Couldn't activate tracepoint"
99 " probe to trace_power_start\n");
100 return ret;
101 }
102 ret = register_trace_power_end(probe_power_end);
103 if (ret) {
104 pr_info("power trace: Couldn't activate tracepoint"
105 " probe to trace_power_end\n");
106 goto fail_start;
107 }
108 ret = register_trace_power_mark(probe_power_mark);
109 if (ret) {
110 pr_info("power trace: Couldn't activate tracepoint"
111 " probe to trace_power_mark\n");
112 goto fail_end;
113 }
114 return ret;
115fail_end:
116 unregister_trace_power_end(probe_power_end);
117fail_start:
118 unregister_trace_power_start(probe_power_start);
119 return ret;
120}
23 121
24static void start_power_trace(struct trace_array *tr) 122static void start_power_trace(struct trace_array *tr)
25{ 123{
26 trace_power_enabled = 1; 124 trace_power_enabled = 1;
125 tracing_power_register();
27} 126}
28 127
29static void stop_power_trace(struct trace_array *tr) 128static void stop_power_trace(struct trace_array *tr)
30{ 129{
31 trace_power_enabled = 0; 130 trace_power_enabled = 0;
131 unregister_trace_power_start(probe_power_start);
132 unregister_trace_power_end(probe_power_end);
133 unregister_trace_power_mark(probe_power_mark);
32} 134}
33 135
34 136
@@ -38,6 +140,7 @@ static int power_trace_init(struct trace_array *tr)
38 power_trace = tr; 140 power_trace = tr;
39 141
40 trace_power_enabled = 1; 142 trace_power_enabled = 1;
143 tracing_power_register();
41 144
42 for_each_cpu(cpu, cpu_possible_mask) 145 for_each_cpu(cpu, cpu_possible_mask)
43 tracing_reset(tr, cpu); 146 tracing_reset(tr, cpu);
@@ -94,86 +197,3 @@ static int init_power_trace(void)
94 return register_tracer(&power_tracer); 197 return register_tracer(&power_tracer);
95} 198}
96device_initcall(init_power_trace); 199device_initcall(init_power_trace);
97
98void trace_power_start(struct power_trace *it, unsigned int type,
99 unsigned int level)
100{
101 if (!trace_power_enabled)
102 return;
103
104 memset(it, 0, sizeof(struct power_trace));
105 it->state = level;
106 it->type = type;
107 it->stamp = ktime_get();
108}
109EXPORT_SYMBOL_GPL(trace_power_start);
110
111
112void trace_power_end(struct power_trace *it)
113{
114 struct ring_buffer_event *event;
115 struct trace_power *entry;
116 struct trace_array_cpu *data;
117 unsigned long irq_flags;
118 struct trace_array *tr = power_trace;
119
120 if (!trace_power_enabled)
121 return;
122
123 preempt_disable();
124 it->end = ktime_get();
125 data = tr->data[smp_processor_id()];
126
127 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
128 &irq_flags);
129 if (!event)
130 goto out;
131 entry = ring_buffer_event_data(event);
132 tracing_generic_entry_update(&entry->ent, 0, 0);
133 entry->ent.type = TRACE_POWER;
134 entry->state_data = *it;
135 ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
136
137 trace_wake_up();
138
139 out:
140 preempt_enable();
141}
142EXPORT_SYMBOL_GPL(trace_power_end);
143
144void trace_power_mark(struct power_trace *it, unsigned int type,
145 unsigned int level)
146{
147 struct ring_buffer_event *event;
148 struct trace_power *entry;
149 struct trace_array_cpu *data;
150 unsigned long irq_flags;
151 struct trace_array *tr = power_trace;
152
153 if (!trace_power_enabled)
154 return;
155
156 memset(it, 0, sizeof(struct power_trace));
157 it->state = level;
158 it->type = type;
159 it->stamp = ktime_get();
160 preempt_disable();
161 it->end = it->stamp;
162 data = tr->data[smp_processor_id()];
163
164 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
165 &irq_flags);
166 if (!event)
167 goto out;
168 entry = ring_buffer_event_data(event);
169 tracing_generic_entry_update(&entry->ent, 0, 0);
170 entry->ent.type = TRACE_POWER;
171 entry->state_data = *it;
172 ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
173
174 trace_wake_up();
175
176 out:
177 preempt_enable();
178}
179EXPORT_SYMBOL_GPL(trace_power_mark);
diff --git a/kernel/trace/trace_printk.c b/kernel/trace/trace_printk.c
new file mode 100644
index 000000000000..a50aea22e929
--- /dev/null
+++ b/kernel/trace/trace_printk.c
@@ -0,0 +1,138 @@
1/*
2 * trace binary printk
3 *
4 * Copyright (C) 2008 Lai Jiangshan <laijs@cn.fujitsu.com>
5 *
6 */
7#include <linux/kernel.h>
8#include <linux/ftrace.h>
9#include <linux/string.h>
10#include <linux/ctype.h>
11#include <linux/list.h>
12#include <linux/mutex.h>
13#include <linux/slab.h>
14#include <linux/module.h>
15#include <linux/seq_file.h>
16#include <linux/fs.h>
17#include <linux/marker.h>
18#include <linux/uaccess.h>
19
20#include "trace.h"
21
22#ifdef CONFIG_MODULES
23
24/*
25 * modules trace_printk()'s formats are autosaved in struct trace_bprintk_fmt
26 * which are queued on trace_bprintk_fmt_list.
27 */
28static LIST_HEAD(trace_bprintk_fmt_list);
29
30/* serialize accesses to trace_bprintk_fmt_list */
31static DEFINE_MUTEX(btrace_mutex);
32
33struct trace_bprintk_fmt {
34 struct list_head list;
35 char fmt[0];
36};
37
38static inline struct trace_bprintk_fmt *lookup_format(const char *fmt)
39{
40 struct trace_bprintk_fmt *pos;
41 list_for_each_entry(pos, &trace_bprintk_fmt_list, list) {
42 if (!strcmp(pos->fmt, fmt))
43 return pos;
44 }
45 return NULL;
46}
47
48static
49void hold_module_trace_bprintk_format(const char **start, const char **end)
50{
51 const char **iter;
52
53 mutex_lock(&btrace_mutex);
54 for (iter = start; iter < end; iter++) {
55 struct trace_bprintk_fmt *tb_fmt = lookup_format(*iter);
56 if (tb_fmt) {
57 *iter = tb_fmt->fmt;
58 continue;
59 }
60
61 tb_fmt = kmalloc(offsetof(struct trace_bprintk_fmt, fmt)
62 + strlen(*iter) + 1, GFP_KERNEL);
63 if (tb_fmt) {
64 list_add_tail(&tb_fmt->list, &trace_bprintk_fmt_list);
65 strcpy(tb_fmt->fmt, *iter);
66 *iter = tb_fmt->fmt;
67 } else
68 *iter = NULL;
69 }
70 mutex_unlock(&btrace_mutex);
71}
72
73static int module_trace_bprintk_format_notify(struct notifier_block *self,
74 unsigned long val, void *data)
75{
76 struct module *mod = data;
77 if (mod->num_trace_bprintk_fmt) {
78 const char **start = mod->trace_bprintk_fmt_start;
79 const char **end = start + mod->num_trace_bprintk_fmt;
80
81 if (val == MODULE_STATE_COMING)
82 hold_module_trace_bprintk_format(start, end);
83 }
84 return 0;
85}
86
87#else /* !CONFIG_MODULES */
88__init static int
89module_trace_bprintk_format_notify(struct notifier_block *self,
90 unsigned long val, void *data)
91{
92 return 0;
93}
94#endif /* CONFIG_MODULES */
95
96
97__initdata_or_module static
98struct notifier_block module_trace_bprintk_format_nb = {
99 .notifier_call = module_trace_bprintk_format_notify,
100};
101
102int __trace_printk(unsigned long ip, const char *fmt, ...)
103 {
104 int ret;
105 va_list ap;
106
107 if (unlikely(!fmt))
108 return 0;
109
110 if (!(trace_flags & TRACE_ITER_PRINTK))
111 return 0;
112
113 va_start(ap, fmt);
114 ret = trace_vprintk(ip, task_curr_ret_stack(current), fmt, ap);
115 va_end(ap);
116 return ret;
117}
118EXPORT_SYMBOL_GPL(__trace_printk);
119
120int __ftrace_vprintk(unsigned long ip, const char *fmt, va_list ap)
121 {
122 if (unlikely(!fmt))
123 return 0;
124
125 if (!(trace_flags & TRACE_ITER_PRINTK))
126 return 0;
127
128 return trace_vprintk(ip, task_curr_ret_stack(current), fmt, ap);
129}
130EXPORT_SYMBOL_GPL(__ftrace_vprintk);
131
132
133static __init int init_trace_printk(void)
134{
135 return register_module_notifier(&module_trace_bprintk_format_nb);
136}
137
138early_initcall(init_trace_printk);
diff --git a/kernel/trace/trace_sched_switch.c b/kernel/trace/trace_sched_switch.c
index df175cb4564f..77132c2cf3d9 100644
--- a/kernel/trace/trace_sched_switch.c
+++ b/kernel/trace/trace_sched_switch.c
@@ -43,7 +43,7 @@ probe_sched_switch(struct rq *__rq, struct task_struct *prev,
43 data = ctx_trace->data[cpu]; 43 data = ctx_trace->data[cpu];
44 44
45 if (likely(!atomic_read(&data->disabled))) 45 if (likely(!atomic_read(&data->disabled)))
46 tracing_sched_switch_trace(ctx_trace, data, prev, next, flags, pc); 46 tracing_sched_switch_trace(ctx_trace, prev, next, flags, pc);
47 47
48 local_irq_restore(flags); 48 local_irq_restore(flags);
49} 49}
@@ -66,7 +66,7 @@ probe_sched_wakeup(struct rq *__rq, struct task_struct *wakee, int success)
66 data = ctx_trace->data[cpu]; 66 data = ctx_trace->data[cpu];
67 67
68 if (likely(!atomic_read(&data->disabled))) 68 if (likely(!atomic_read(&data->disabled)))
69 tracing_sched_wakeup_trace(ctx_trace, data, wakee, current, 69 tracing_sched_wakeup_trace(ctx_trace, wakee, current,
70 flags, pc); 70 flags, pc);
71 71
72 local_irq_restore(flags); 72 local_irq_restore(flags);
@@ -93,7 +93,7 @@ static int tracing_sched_register(void)
93 ret = register_trace_sched_switch(probe_sched_switch); 93 ret = register_trace_sched_switch(probe_sched_switch);
94 if (ret) { 94 if (ret) {
95 pr_info("sched trace: Couldn't activate tracepoint" 95 pr_info("sched trace: Couldn't activate tracepoint"
96 " probe to kernel_sched_schedule\n"); 96 " probe to kernel_sched_switch\n");
97 goto fail_deprobe_wake_new; 97 goto fail_deprobe_wake_new;
98 } 98 }
99 99
@@ -185,12 +185,6 @@ void tracing_sched_switch_assign_trace(struct trace_array *tr)
185 ctx_trace = tr; 185 ctx_trace = tr;
186} 186}
187 187
188static void start_sched_trace(struct trace_array *tr)
189{
190 tracing_reset_online_cpus(tr);
191 tracing_start_sched_switch_record();
192}
193
194static void stop_sched_trace(struct trace_array *tr) 188static void stop_sched_trace(struct trace_array *tr)
195{ 189{
196 tracing_stop_sched_switch_record(); 190 tracing_stop_sched_switch_record();
@@ -199,7 +193,7 @@ static void stop_sched_trace(struct trace_array *tr)
199static int sched_switch_trace_init(struct trace_array *tr) 193static int sched_switch_trace_init(struct trace_array *tr)
200{ 194{
201 ctx_trace = tr; 195 ctx_trace = tr;
202 start_sched_trace(tr); 196 tracing_start_sched_switch_record();
203 return 0; 197 return 0;
204} 198}
205 199
@@ -227,6 +221,7 @@ static struct tracer sched_switch_trace __read_mostly =
227 .reset = sched_switch_trace_reset, 221 .reset = sched_switch_trace_reset,
228 .start = sched_switch_trace_start, 222 .start = sched_switch_trace_start,
229 .stop = sched_switch_trace_stop, 223 .stop = sched_switch_trace_stop,
224 .wait_pipe = poll_wait_pipe,
230#ifdef CONFIG_FTRACE_SELFTEST 225#ifdef CONFIG_FTRACE_SELFTEST
231 .selftest = trace_selftest_startup_sched_switch, 226 .selftest = trace_selftest_startup_sched_switch,
232#endif 227#endif
diff --git a/kernel/trace/trace_sched_wakeup.c b/kernel/trace/trace_sched_wakeup.c
index 42ae1e77b6b3..3c5ad6b2ec84 100644
--- a/kernel/trace/trace_sched_wakeup.c
+++ b/kernel/trace/trace_sched_wakeup.c
@@ -25,12 +25,15 @@ static int __read_mostly tracer_enabled;
25static struct task_struct *wakeup_task; 25static struct task_struct *wakeup_task;
26static int wakeup_cpu; 26static int wakeup_cpu;
27static unsigned wakeup_prio = -1; 27static unsigned wakeup_prio = -1;
28static int wakeup_rt;
28 29
29static raw_spinlock_t wakeup_lock = 30static raw_spinlock_t wakeup_lock =
30 (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED; 31 (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
31 32
32static void __wakeup_reset(struct trace_array *tr); 33static void __wakeup_reset(struct trace_array *tr);
33 34
35static int save_lat_flag;
36
34#ifdef CONFIG_FUNCTION_TRACER 37#ifdef CONFIG_FUNCTION_TRACER
35/* 38/*
36 * irqsoff uses its own tracer function to keep the overhead down: 39 * irqsoff uses its own tracer function to keep the overhead down:
@@ -71,7 +74,7 @@ wakeup_tracer_call(unsigned long ip, unsigned long parent_ip)
71 if (task_cpu(wakeup_task) != cpu) 74 if (task_cpu(wakeup_task) != cpu)
72 goto unlock; 75 goto unlock;
73 76
74 trace_function(tr, data, ip, parent_ip, flags, pc); 77 trace_function(tr, ip, parent_ip, flags, pc);
75 78
76 unlock: 79 unlock:
77 __raw_spin_unlock(&wakeup_lock); 80 __raw_spin_unlock(&wakeup_lock);
@@ -151,7 +154,8 @@ probe_wakeup_sched_switch(struct rq *rq, struct task_struct *prev,
151 if (unlikely(!tracer_enabled || next != wakeup_task)) 154 if (unlikely(!tracer_enabled || next != wakeup_task))
152 goto out_unlock; 155 goto out_unlock;
153 156
154 trace_function(wakeup_trace, data, CALLER_ADDR1, CALLER_ADDR2, flags, pc); 157 trace_function(wakeup_trace, CALLER_ADDR1, CALLER_ADDR2, flags, pc);
158 tracing_sched_switch_trace(wakeup_trace, prev, next, flags, pc);
155 159
156 /* 160 /*
157 * usecs conversion is slow so we try to delay the conversion 161 * usecs conversion is slow so we try to delay the conversion
@@ -182,13 +186,10 @@ out:
182 186
183static void __wakeup_reset(struct trace_array *tr) 187static void __wakeup_reset(struct trace_array *tr)
184{ 188{
185 struct trace_array_cpu *data;
186 int cpu; 189 int cpu;
187 190
188 for_each_possible_cpu(cpu) { 191 for_each_possible_cpu(cpu)
189 data = tr->data[cpu];
190 tracing_reset(tr, cpu); 192 tracing_reset(tr, cpu);
191 }
192 193
193 wakeup_cpu = -1; 194 wakeup_cpu = -1;
194 wakeup_prio = -1; 195 wakeup_prio = -1;
@@ -213,6 +214,7 @@ static void wakeup_reset(struct trace_array *tr)
213static void 214static void
214probe_wakeup(struct rq *rq, struct task_struct *p, int success) 215probe_wakeup(struct rq *rq, struct task_struct *p, int success)
215{ 216{
217 struct trace_array_cpu *data;
216 int cpu = smp_processor_id(); 218 int cpu = smp_processor_id();
217 unsigned long flags; 219 unsigned long flags;
218 long disabled; 220 long disabled;
@@ -224,7 +226,7 @@ probe_wakeup(struct rq *rq, struct task_struct *p, int success)
224 tracing_record_cmdline(p); 226 tracing_record_cmdline(p);
225 tracing_record_cmdline(current); 227 tracing_record_cmdline(current);
226 228
227 if (likely(!rt_task(p)) || 229 if ((wakeup_rt && !rt_task(p)) ||
228 p->prio >= wakeup_prio || 230 p->prio >= wakeup_prio ||
229 p->prio >= current->prio) 231 p->prio >= current->prio)
230 return; 232 return;
@@ -252,9 +254,10 @@ probe_wakeup(struct rq *rq, struct task_struct *p, int success)
252 254
253 local_save_flags(flags); 255 local_save_flags(flags);
254 256
255 wakeup_trace->data[wakeup_cpu]->preempt_timestamp = ftrace_now(cpu); 257 data = wakeup_trace->data[wakeup_cpu];
256 trace_function(wakeup_trace, wakeup_trace->data[wakeup_cpu], 258 data->preempt_timestamp = ftrace_now(cpu);
257 CALLER_ADDR1, CALLER_ADDR2, flags, pc); 259 tracing_sched_wakeup_trace(wakeup_trace, p, current, flags, pc);
260 trace_function(wakeup_trace, CALLER_ADDR1, CALLER_ADDR2, flags, pc);
258 261
259out_locked: 262out_locked:
260 __raw_spin_unlock(&wakeup_lock); 263 __raw_spin_unlock(&wakeup_lock);
@@ -262,12 +265,6 @@ out:
262 atomic_dec(&wakeup_trace->data[cpu]->disabled); 265 atomic_dec(&wakeup_trace->data[cpu]->disabled);
263} 266}
264 267
265/*
266 * save_tracer_enabled is used to save the state of the tracer_enabled
267 * variable when we disable it when we open a trace output file.
268 */
269static int save_tracer_enabled;
270
271static void start_wakeup_tracer(struct trace_array *tr) 268static void start_wakeup_tracer(struct trace_array *tr)
272{ 269{
273 int ret; 270 int ret;
@@ -289,7 +286,7 @@ static void start_wakeup_tracer(struct trace_array *tr)
289 ret = register_trace_sched_switch(probe_wakeup_sched_switch); 286 ret = register_trace_sched_switch(probe_wakeup_sched_switch);
290 if (ret) { 287 if (ret) {
291 pr_info("sched trace: Couldn't activate tracepoint" 288 pr_info("sched trace: Couldn't activate tracepoint"
292 " probe to kernel_sched_schedule\n"); 289 " probe to kernel_sched_switch\n");
293 goto fail_deprobe_wake_new; 290 goto fail_deprobe_wake_new;
294 } 291 }
295 292
@@ -306,13 +303,10 @@ static void start_wakeup_tracer(struct trace_array *tr)
306 303
307 register_ftrace_function(&trace_ops); 304 register_ftrace_function(&trace_ops);
308 305
309 if (tracing_is_enabled()) { 306 if (tracing_is_enabled())
310 tracer_enabled = 1; 307 tracer_enabled = 1;
311 save_tracer_enabled = 1; 308 else
312 } else {
313 tracer_enabled = 0; 309 tracer_enabled = 0;
314 save_tracer_enabled = 0;
315 }
316 310
317 return; 311 return;
318fail_deprobe_wake_new: 312fail_deprobe_wake_new:
@@ -324,54 +318,54 @@ fail_deprobe:
324static void stop_wakeup_tracer(struct trace_array *tr) 318static void stop_wakeup_tracer(struct trace_array *tr)
325{ 319{
326 tracer_enabled = 0; 320 tracer_enabled = 0;
327 save_tracer_enabled = 0;
328 unregister_ftrace_function(&trace_ops); 321 unregister_ftrace_function(&trace_ops);
329 unregister_trace_sched_switch(probe_wakeup_sched_switch); 322 unregister_trace_sched_switch(probe_wakeup_sched_switch);
330 unregister_trace_sched_wakeup_new(probe_wakeup); 323 unregister_trace_sched_wakeup_new(probe_wakeup);
331 unregister_trace_sched_wakeup(probe_wakeup); 324 unregister_trace_sched_wakeup(probe_wakeup);
332} 325}
333 326
334static int wakeup_tracer_init(struct trace_array *tr) 327static int __wakeup_tracer_init(struct trace_array *tr)
335{ 328{
329 save_lat_flag = trace_flags & TRACE_ITER_LATENCY_FMT;
330 trace_flags |= TRACE_ITER_LATENCY_FMT;
331
336 tracing_max_latency = 0; 332 tracing_max_latency = 0;
337 wakeup_trace = tr; 333 wakeup_trace = tr;
338 start_wakeup_tracer(tr); 334 start_wakeup_tracer(tr);
339 return 0; 335 return 0;
340} 336}
341 337
338static int wakeup_tracer_init(struct trace_array *tr)
339{
340 wakeup_rt = 0;
341 return __wakeup_tracer_init(tr);
342}
343
344static int wakeup_rt_tracer_init(struct trace_array *tr)
345{
346 wakeup_rt = 1;
347 return __wakeup_tracer_init(tr);
348}
349
342static void wakeup_tracer_reset(struct trace_array *tr) 350static void wakeup_tracer_reset(struct trace_array *tr)
343{ 351{
344 stop_wakeup_tracer(tr); 352 stop_wakeup_tracer(tr);
345 /* make sure we put back any tasks we are tracing */ 353 /* make sure we put back any tasks we are tracing */
346 wakeup_reset(tr); 354 wakeup_reset(tr);
355
356 if (!save_lat_flag)
357 trace_flags &= ~TRACE_ITER_LATENCY_FMT;
347} 358}
348 359
349static void wakeup_tracer_start(struct trace_array *tr) 360static void wakeup_tracer_start(struct trace_array *tr)
350{ 361{
351 wakeup_reset(tr); 362 wakeup_reset(tr);
352 tracer_enabled = 1; 363 tracer_enabled = 1;
353 save_tracer_enabled = 1;
354} 364}
355 365
356static void wakeup_tracer_stop(struct trace_array *tr) 366static void wakeup_tracer_stop(struct trace_array *tr)
357{ 367{
358 tracer_enabled = 0; 368 tracer_enabled = 0;
359 save_tracer_enabled = 0;
360}
361
362static void wakeup_tracer_open(struct trace_iterator *iter)
363{
364 /* stop the trace while dumping */
365 tracer_enabled = 0;
366}
367
368static void wakeup_tracer_close(struct trace_iterator *iter)
369{
370 /* forget about any processes we were recording */
371 if (save_tracer_enabled) {
372 wakeup_reset(iter->tr);
373 tracer_enabled = 1;
374 }
375} 369}
376 370
377static struct tracer wakeup_tracer __read_mostly = 371static struct tracer wakeup_tracer __read_mostly =
@@ -381,8 +375,20 @@ static struct tracer wakeup_tracer __read_mostly =
381 .reset = wakeup_tracer_reset, 375 .reset = wakeup_tracer_reset,
382 .start = wakeup_tracer_start, 376 .start = wakeup_tracer_start,
383 .stop = wakeup_tracer_stop, 377 .stop = wakeup_tracer_stop,
384 .open = wakeup_tracer_open, 378 .print_max = 1,
385 .close = wakeup_tracer_close, 379#ifdef CONFIG_FTRACE_SELFTEST
380 .selftest = trace_selftest_startup_wakeup,
381#endif
382};
383
384static struct tracer wakeup_rt_tracer __read_mostly =
385{
386 .name = "wakeup_rt",
387 .init = wakeup_rt_tracer_init,
388 .reset = wakeup_tracer_reset,
389 .start = wakeup_tracer_start,
390 .stop = wakeup_tracer_stop,
391 .wait_pipe = poll_wait_pipe,
386 .print_max = 1, 392 .print_max = 1,
387#ifdef CONFIG_FTRACE_SELFTEST 393#ifdef CONFIG_FTRACE_SELFTEST
388 .selftest = trace_selftest_startup_wakeup, 394 .selftest = trace_selftest_startup_wakeup,
@@ -397,6 +403,10 @@ __init static int init_wakeup_tracer(void)
397 if (ret) 403 if (ret)
398 return ret; 404 return ret;
399 405
406 ret = register_tracer(&wakeup_rt_tracer);
407 if (ret)
408 return ret;
409
400 return 0; 410 return 0;
401} 411}
402device_initcall(init_wakeup_tracer); 412device_initcall(init_wakeup_tracer);
diff --git a/kernel/trace/trace_selftest.c b/kernel/trace/trace_selftest.c
index bc8e80a86bca..f907a2b29028 100644
--- a/kernel/trace/trace_selftest.c
+++ b/kernel/trace/trace_selftest.c
@@ -1,5 +1,6 @@
1/* Include in trace.c */ 1/* Include in trace.c */
2 2
3#include <linux/stringify.h>
3#include <linux/kthread.h> 4#include <linux/kthread.h>
4#include <linux/delay.h> 5#include <linux/delay.h>
5 6
@@ -9,11 +10,12 @@ static inline int trace_valid_entry(struct trace_entry *entry)
9 case TRACE_FN: 10 case TRACE_FN:
10 case TRACE_CTX: 11 case TRACE_CTX:
11 case TRACE_WAKE: 12 case TRACE_WAKE:
12 case TRACE_CONT:
13 case TRACE_STACK: 13 case TRACE_STACK:
14 case TRACE_PRINT: 14 case TRACE_PRINT:
15 case TRACE_SPECIAL: 15 case TRACE_SPECIAL:
16 case TRACE_BRANCH: 16 case TRACE_BRANCH:
17 case TRACE_GRAPH_ENT:
18 case TRACE_GRAPH_RET:
17 return 1; 19 return 1;
18 } 20 }
19 return 0; 21 return 0;
@@ -99,9 +101,6 @@ static inline void warn_failed_init_tracer(struct tracer *trace, int init_ret)
99 101
100#ifdef CONFIG_DYNAMIC_FTRACE 102#ifdef CONFIG_DYNAMIC_FTRACE
101 103
102#define __STR(x) #x
103#define STR(x) __STR(x)
104
105/* Test dynamic code modification and ftrace filters */ 104/* Test dynamic code modification and ftrace filters */
106int trace_selftest_startup_dynamic_tracing(struct tracer *trace, 105int trace_selftest_startup_dynamic_tracing(struct tracer *trace,
107 struct trace_array *tr, 106 struct trace_array *tr,
@@ -125,17 +124,17 @@ int trace_selftest_startup_dynamic_tracing(struct tracer *trace,
125 func(); 124 func();
126 125
127 /* 126 /*
128 * Some archs *cough*PowerPC*cough* add charachters to the 127 * Some archs *cough*PowerPC*cough* add characters to the
129 * start of the function names. We simply put a '*' to 128 * start of the function names. We simply put a '*' to
130 * accomodate them. 129 * accommodate them.
131 */ 130 */
132 func_name = "*" STR(DYN_FTRACE_TEST_NAME); 131 func_name = "*" __stringify(DYN_FTRACE_TEST_NAME);
133 132
134 /* filter only on our function */ 133 /* filter only on our function */
135 ftrace_set_filter(func_name, strlen(func_name), 1); 134 ftrace_set_filter(func_name, strlen(func_name), 1);
136 135
137 /* enable tracing */ 136 /* enable tracing */
138 ret = trace->init(tr); 137 ret = tracer_init(trace, tr);
139 if (ret) { 138 if (ret) {
140 warn_failed_init_tracer(trace, ret); 139 warn_failed_init_tracer(trace, ret);
141 goto out; 140 goto out;
@@ -209,7 +208,7 @@ trace_selftest_startup_function(struct tracer *trace, struct trace_array *tr)
209 ftrace_enabled = 1; 208 ftrace_enabled = 1;
210 tracer_enabled = 1; 209 tracer_enabled = 1;
211 210
212 ret = trace->init(tr); 211 ret = tracer_init(trace, tr);
213 if (ret) { 212 if (ret) {
214 warn_failed_init_tracer(trace, ret); 213 warn_failed_init_tracer(trace, ret);
215 goto out; 214 goto out;
@@ -247,6 +246,54 @@ trace_selftest_startup_function(struct tracer *trace, struct trace_array *tr)
247} 246}
248#endif /* CONFIG_FUNCTION_TRACER */ 247#endif /* CONFIG_FUNCTION_TRACER */
249 248
249
250#ifdef CONFIG_FUNCTION_GRAPH_TRACER
251/*
252 * Pretty much the same than for the function tracer from which the selftest
253 * has been borrowed.
254 */
255int
256trace_selftest_startup_function_graph(struct tracer *trace,
257 struct trace_array *tr)
258{
259 int ret;
260 unsigned long count;
261
262 ret = tracer_init(trace, tr);
263 if (ret) {
264 warn_failed_init_tracer(trace, ret);
265 goto out;
266 }
267
268 /* Sleep for a 1/10 of a second */
269 msleep(100);
270
271 tracing_stop();
272
273 /* check the trace buffer */
274 ret = trace_test_buffer(tr, &count);
275
276 trace->reset(tr);
277 tracing_start();
278
279 if (!ret && !count) {
280 printk(KERN_CONT ".. no entries found ..");
281 ret = -1;
282 goto out;
283 }
284
285 /* Don't test dynamic tracing, the function tracer already did */
286
287out:
288 /* Stop it if we failed */
289 if (ret)
290 ftrace_graph_stop();
291
292 return ret;
293}
294#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
295
296
250#ifdef CONFIG_IRQSOFF_TRACER 297#ifdef CONFIG_IRQSOFF_TRACER
251int 298int
252trace_selftest_startup_irqsoff(struct tracer *trace, struct trace_array *tr) 299trace_selftest_startup_irqsoff(struct tracer *trace, struct trace_array *tr)
@@ -256,7 +303,7 @@ trace_selftest_startup_irqsoff(struct tracer *trace, struct trace_array *tr)
256 int ret; 303 int ret;
257 304
258 /* start the tracing */ 305 /* start the tracing */
259 ret = trace->init(tr); 306 ret = tracer_init(trace, tr);
260 if (ret) { 307 if (ret) {
261 warn_failed_init_tracer(trace, ret); 308 warn_failed_init_tracer(trace, ret);
262 return ret; 309 return ret;
@@ -310,7 +357,7 @@ trace_selftest_startup_preemptoff(struct tracer *trace, struct trace_array *tr)
310 } 357 }
311 358
312 /* start the tracing */ 359 /* start the tracing */
313 ret = trace->init(tr); 360 ret = tracer_init(trace, tr);
314 if (ret) { 361 if (ret) {
315 warn_failed_init_tracer(trace, ret); 362 warn_failed_init_tracer(trace, ret);
316 return ret; 363 return ret;
@@ -364,7 +411,7 @@ trace_selftest_startup_preemptirqsoff(struct tracer *trace, struct trace_array *
364 } 411 }
365 412
366 /* start the tracing */ 413 /* start the tracing */
367 ret = trace->init(tr); 414 ret = tracer_init(trace, tr);
368 if (ret) { 415 if (ret) {
369 warn_failed_init_tracer(trace, ret); 416 warn_failed_init_tracer(trace, ret);
370 goto out; 417 goto out;
@@ -496,7 +543,7 @@ trace_selftest_startup_wakeup(struct tracer *trace, struct trace_array *tr)
496 wait_for_completion(&isrt); 543 wait_for_completion(&isrt);
497 544
498 /* start the tracing */ 545 /* start the tracing */
499 ret = trace->init(tr); 546 ret = tracer_init(trace, tr);
500 if (ret) { 547 if (ret) {
501 warn_failed_init_tracer(trace, ret); 548 warn_failed_init_tracer(trace, ret);
502 return ret; 549 return ret;
@@ -557,7 +604,7 @@ trace_selftest_startup_sched_switch(struct tracer *trace, struct trace_array *tr
557 int ret; 604 int ret;
558 605
559 /* start the tracing */ 606 /* start the tracing */
560 ret = trace->init(tr); 607 ret = tracer_init(trace, tr);
561 if (ret) { 608 if (ret) {
562 warn_failed_init_tracer(trace, ret); 609 warn_failed_init_tracer(trace, ret);
563 return ret; 610 return ret;
@@ -589,10 +636,10 @@ trace_selftest_startup_sysprof(struct tracer *trace, struct trace_array *tr)
589 int ret; 636 int ret;
590 637
591 /* start the tracing */ 638 /* start the tracing */
592 ret = trace->init(tr); 639 ret = tracer_init(trace, tr);
593 if (ret) { 640 if (ret) {
594 warn_failed_init_tracer(trace, ret); 641 warn_failed_init_tracer(trace, ret);
595 return 0; 642 return ret;
596 } 643 }
597 644
598 /* Sleep for a 1/10 of a second */ 645 /* Sleep for a 1/10 of a second */
@@ -604,6 +651,11 @@ trace_selftest_startup_sysprof(struct tracer *trace, struct trace_array *tr)
604 trace->reset(tr); 651 trace->reset(tr);
605 tracing_start(); 652 tracing_start();
606 653
654 if (!ret && !count) {
655 printk(KERN_CONT ".. no entries found ..");
656 ret = -1;
657 }
658
607 return ret; 659 return ret;
608} 660}
609#endif /* CONFIG_SYSPROF_TRACER */ 661#endif /* CONFIG_SYSPROF_TRACER */
@@ -616,7 +668,7 @@ trace_selftest_startup_branch(struct tracer *trace, struct trace_array *tr)
616 int ret; 668 int ret;
617 669
618 /* start the tracing */ 670 /* start the tracing */
619 ret = trace->init(tr); 671 ret = tracer_init(trace, tr);
620 if (ret) { 672 if (ret) {
621 warn_failed_init_tracer(trace, ret); 673 warn_failed_init_tracer(trace, ret);
622 return ret; 674 return ret;
@@ -631,6 +683,11 @@ trace_selftest_startup_branch(struct tracer *trace, struct trace_array *tr)
631 trace->reset(tr); 683 trace->reset(tr);
632 tracing_start(); 684 tracing_start();
633 685
686 if (!ret && !count) {
687 printk(KERN_CONT ".. no entries found ..");
688 ret = -1;
689 }
690
634 return ret; 691 return ret;
635} 692}
636#endif /* CONFIG_BRANCH_TRACER */ 693#endif /* CONFIG_BRANCH_TRACER */
diff --git a/kernel/trace/trace_stat.c b/kernel/trace/trace_stat.c
new file mode 100644
index 000000000000..39310e3434ee
--- /dev/null
+++ b/kernel/trace/trace_stat.c
@@ -0,0 +1,319 @@
1/*
2 * Infrastructure for statistic tracing (histogram output).
3 *
4 * Copyright (C) 2008 Frederic Weisbecker <fweisbec@gmail.com>
5 *
6 * Based on the code from trace_branch.c which is
7 * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com>
8 *
9 */
10
11
12#include <linux/list.h>
13#include <linux/debugfs.h>
14#include "trace_stat.h"
15#include "trace.h"
16
17
18/* List of stat entries from a tracer */
19struct trace_stat_list {
20 struct list_head list;
21 void *stat;
22};
23
24/* A stat session is the stats output in one file */
25struct tracer_stat_session {
26 struct list_head session_list;
27 struct tracer_stat *ts;
28 struct list_head stat_list;
29 struct mutex stat_mutex;
30 struct dentry *file;
31};
32
33/* All of the sessions currently in use. Each stat file embed one session */
34static LIST_HEAD(all_stat_sessions);
35static DEFINE_MUTEX(all_stat_sessions_mutex);
36
37/* The root directory for all stat files */
38static struct dentry *stat_dir;
39
40
41static void reset_stat_session(struct tracer_stat_session *session)
42{
43 struct trace_stat_list *node, *next;
44
45 list_for_each_entry_safe(node, next, &session->stat_list, list)
46 kfree(node);
47
48 INIT_LIST_HEAD(&session->stat_list);
49}
50
51static void destroy_session(struct tracer_stat_session *session)
52{
53 debugfs_remove(session->file);
54 reset_stat_session(session);
55 mutex_destroy(&session->stat_mutex);
56 kfree(session);
57}
58
59/*
60 * For tracers that don't provide a stat_cmp callback.
61 * This one will force an immediate insertion on tail of
62 * the list.
63 */
64static int dummy_cmp(void *p1, void *p2)
65{
66 return 1;
67}
68
69/*
70 * Initialize the stat list at each trace_stat file opening.
71 * All of these copies and sorting are required on all opening
72 * since the stats could have changed between two file sessions.
73 */
74static int stat_seq_init(struct tracer_stat_session *session)
75{
76 struct trace_stat_list *iter_entry, *new_entry;
77 struct tracer_stat *ts = session->ts;
78 void *prev_stat;
79 int ret = 0;
80 int i;
81
82 mutex_lock(&session->stat_mutex);
83 reset_stat_session(session);
84
85 if (!ts->stat_cmp)
86 ts->stat_cmp = dummy_cmp;
87
88 /*
89 * The first entry. Actually this is the second, but the first
90 * one (the stat_list head) is pointless.
91 */
92 new_entry = kmalloc(sizeof(struct trace_stat_list), GFP_KERNEL);
93 if (!new_entry) {
94 ret = -ENOMEM;
95 goto exit;
96 }
97
98 INIT_LIST_HEAD(&new_entry->list);
99
100 list_add(&new_entry->list, &session->stat_list);
101
102 new_entry->stat = ts->stat_start();
103 prev_stat = new_entry->stat;
104
105 /*
106 * Iterate over the tracer stat entries and store them in a sorted
107 * list.
108 */
109 for (i = 1; ; i++) {
110 new_entry = kmalloc(sizeof(struct trace_stat_list), GFP_KERNEL);
111 if (!new_entry) {
112 ret = -ENOMEM;
113 goto exit_free_list;
114 }
115
116 INIT_LIST_HEAD(&new_entry->list);
117 new_entry->stat = ts->stat_next(prev_stat, i);
118
119 /* End of insertion */
120 if (!new_entry->stat)
121 break;
122
123 list_for_each_entry(iter_entry, &session->stat_list, list) {
124
125 /* Insertion with a descendent sorting */
126 if (ts->stat_cmp(new_entry->stat,
127 iter_entry->stat) > 0) {
128
129 list_add_tail(&new_entry->list,
130 &iter_entry->list);
131 break;
132
133 /* The current smaller value */
134 } else if (list_is_last(&iter_entry->list,
135 &session->stat_list)) {
136 list_add(&new_entry->list, &iter_entry->list);
137 break;
138 }
139 }
140
141 prev_stat = new_entry->stat;
142 }
143exit:
144 mutex_unlock(&session->stat_mutex);
145 return ret;
146
147exit_free_list:
148 reset_stat_session(session);
149 mutex_unlock(&session->stat_mutex);
150 return ret;
151}
152
153
154static void *stat_seq_start(struct seq_file *s, loff_t *pos)
155{
156 struct tracer_stat_session *session = s->private;
157
158 /* Prevent from tracer switch or stat_list modification */
159 mutex_lock(&session->stat_mutex);
160
161 /* If we are in the beginning of the file, print the headers */
162 if (!*pos && session->ts->stat_headers)
163 session->ts->stat_headers(s);
164
165 return seq_list_start(&session->stat_list, *pos);
166}
167
168static void *stat_seq_next(struct seq_file *s, void *p, loff_t *pos)
169{
170 struct tracer_stat_session *session = s->private;
171
172 return seq_list_next(p, &session->stat_list, pos);
173}
174
175static void stat_seq_stop(struct seq_file *s, void *p)
176{
177 struct tracer_stat_session *session = s->private;
178 mutex_unlock(&session->stat_mutex);
179}
180
181static int stat_seq_show(struct seq_file *s, void *v)
182{
183 struct tracer_stat_session *session = s->private;
184 struct trace_stat_list *l = list_entry(v, struct trace_stat_list, list);
185
186 return session->ts->stat_show(s, l->stat);
187}
188
189static const struct seq_operations trace_stat_seq_ops = {
190 .start = stat_seq_start,
191 .next = stat_seq_next,
192 .stop = stat_seq_stop,
193 .show = stat_seq_show
194};
195
196/* The session stat is refilled and resorted at each stat file opening */
197static int tracing_stat_open(struct inode *inode, struct file *file)
198{
199 int ret;
200
201 struct tracer_stat_session *session = inode->i_private;
202
203 ret = seq_open(file, &trace_stat_seq_ops);
204 if (!ret) {
205 struct seq_file *m = file->private_data;
206 m->private = session;
207 ret = stat_seq_init(session);
208 }
209
210 return ret;
211}
212
213/*
214 * Avoid consuming memory with our now useless list.
215 */
216static int tracing_stat_release(struct inode *i, struct file *f)
217{
218 struct tracer_stat_session *session = i->i_private;
219
220 mutex_lock(&session->stat_mutex);
221 reset_stat_session(session);
222 mutex_unlock(&session->stat_mutex);
223
224 return 0;
225}
226
227static const struct file_operations tracing_stat_fops = {
228 .open = tracing_stat_open,
229 .read = seq_read,
230 .llseek = seq_lseek,
231 .release = tracing_stat_release
232};
233
234static int tracing_stat_init(void)
235{
236 struct dentry *d_tracing;
237
238 d_tracing = tracing_init_dentry();
239
240 stat_dir = debugfs_create_dir("trace_stat", d_tracing);
241 if (!stat_dir)
242 pr_warning("Could not create debugfs "
243 "'trace_stat' entry\n");
244 return 0;
245}
246
247static int init_stat_file(struct tracer_stat_session *session)
248{
249 if (!stat_dir && tracing_stat_init())
250 return -ENODEV;
251
252 session->file = debugfs_create_file(session->ts->name, 0644,
253 stat_dir,
254 session, &tracing_stat_fops);
255 if (!session->file)
256 return -ENOMEM;
257 return 0;
258}
259
260int register_stat_tracer(struct tracer_stat *trace)
261{
262 struct tracer_stat_session *session, *node, *tmp;
263 int ret;
264
265 if (!trace)
266 return -EINVAL;
267
268 if (!trace->stat_start || !trace->stat_next || !trace->stat_show)
269 return -EINVAL;
270
271 /* Already registered? */
272 mutex_lock(&all_stat_sessions_mutex);
273 list_for_each_entry_safe(node, tmp, &all_stat_sessions, session_list) {
274 if (node->ts == trace) {
275 mutex_unlock(&all_stat_sessions_mutex);
276 return -EINVAL;
277 }
278 }
279 mutex_unlock(&all_stat_sessions_mutex);
280
281 /* Init the session */
282 session = kmalloc(sizeof(struct tracer_stat_session), GFP_KERNEL);
283 if (!session)
284 return -ENOMEM;
285
286 session->ts = trace;
287 INIT_LIST_HEAD(&session->session_list);
288 INIT_LIST_HEAD(&session->stat_list);
289 mutex_init(&session->stat_mutex);
290 session->file = NULL;
291
292 ret = init_stat_file(session);
293 if (ret) {
294 destroy_session(session);
295 return ret;
296 }
297
298 /* Register */
299 mutex_lock(&all_stat_sessions_mutex);
300 list_add_tail(&session->session_list, &all_stat_sessions);
301 mutex_unlock(&all_stat_sessions_mutex);
302
303 return 0;
304}
305
306void unregister_stat_tracer(struct tracer_stat *trace)
307{
308 struct tracer_stat_session *node, *tmp;
309
310 mutex_lock(&all_stat_sessions_mutex);
311 list_for_each_entry_safe(node, tmp, &all_stat_sessions, session_list) {
312 if (node->ts == trace) {
313 list_del(&node->session_list);
314 destroy_session(node);
315 break;
316 }
317 }
318 mutex_unlock(&all_stat_sessions_mutex);
319}
diff --git a/kernel/trace/trace_stat.h b/kernel/trace/trace_stat.h
new file mode 100644
index 000000000000..202274cf7f3d
--- /dev/null
+++ b/kernel/trace/trace_stat.h
@@ -0,0 +1,31 @@
1#ifndef __TRACE_STAT_H
2#define __TRACE_STAT_H
3
4#include <linux/seq_file.h>
5
6/*
7 * If you want to provide a stat file (one-shot statistics), fill
8 * an iterator with stat_start/stat_next and a stat_show callbacks.
9 * The others callbacks are optional.
10 */
11struct tracer_stat {
12 /* The name of your stat file */
13 const char *name;
14 /* Iteration over statistic entries */
15 void *(*stat_start)(void);
16 void *(*stat_next)(void *prev, int idx);
17 /* Compare two entries for stats sorting */
18 int (*stat_cmp)(void *p1, void *p2);
19 /* Print a stat entry */
20 int (*stat_show)(struct seq_file *s, void *p);
21 /* Print the headers of your stat entries */
22 int (*stat_headers)(struct seq_file *s);
23};
24
25/*
26 * Destroy or create a stat file
27 */
28extern int register_stat_tracer(struct tracer_stat *trace);
29extern void unregister_stat_tracer(struct tracer_stat *trace);
30
31#endif /* __TRACE_STAT_H */
diff --git a/kernel/trace/trace_sysprof.c b/kernel/trace/trace_sysprof.c
index eaca5ad803ff..91fd19c2149f 100644
--- a/kernel/trace/trace_sysprof.c
+++ b/kernel/trace/trace_sysprof.c
@@ -88,7 +88,7 @@ static void backtrace_address(void *data, unsigned long addr, int reliable)
88 } 88 }
89} 89}
90 90
91const static struct stacktrace_ops backtrace_ops = { 91static const struct stacktrace_ops backtrace_ops = {
92 .warning = backtrace_warning, 92 .warning = backtrace_warning,
93 .warning_symbol = backtrace_warning_symbol, 93 .warning_symbol = backtrace_warning_symbol,
94 .stack = backtrace_stack, 94 .stack = backtrace_stack,
@@ -226,15 +226,6 @@ static void stop_stack_timers(void)
226 stop_stack_timer(cpu); 226 stop_stack_timer(cpu);
227} 227}
228 228
229static void start_stack_trace(struct trace_array *tr)
230{
231 mutex_lock(&sample_timer_lock);
232 tracing_reset_online_cpus(tr);
233 start_stack_timers();
234 tracer_enabled = 1;
235 mutex_unlock(&sample_timer_lock);
236}
237
238static void stop_stack_trace(struct trace_array *tr) 229static void stop_stack_trace(struct trace_array *tr)
239{ 230{
240 mutex_lock(&sample_timer_lock); 231 mutex_lock(&sample_timer_lock);
@@ -247,12 +238,18 @@ static int stack_trace_init(struct trace_array *tr)
247{ 238{
248 sysprof_trace = tr; 239 sysprof_trace = tr;
249 240
250 start_stack_trace(tr); 241 tracing_start_cmdline_record();
242
243 mutex_lock(&sample_timer_lock);
244 start_stack_timers();
245 tracer_enabled = 1;
246 mutex_unlock(&sample_timer_lock);
251 return 0; 247 return 0;
252} 248}
253 249
254static void stack_trace_reset(struct trace_array *tr) 250static void stack_trace_reset(struct trace_array *tr)
255{ 251{
252 tracing_stop_cmdline_record();
256 stop_stack_trace(tr); 253 stop_stack_trace(tr);
257} 254}
258 255
@@ -317,7 +314,7 @@ sysprof_sample_write(struct file *filp, const char __user *ubuf,
317 return cnt; 314 return cnt;
318} 315}
319 316
320static struct file_operations sysprof_sample_fops = { 317static const struct file_operations sysprof_sample_fops = {
321 .read = sysprof_sample_read, 318 .read = sysprof_sample_read,
322 .write = sysprof_sample_write, 319 .write = sysprof_sample_write,
323}; 320};
@@ -330,5 +327,5 @@ void init_tracer_sysprof_debugfs(struct dentry *d_tracer)
330 d_tracer, NULL, &sysprof_sample_fops); 327 d_tracer, NULL, &sysprof_sample_fops);
331 if (entry) 328 if (entry)
332 return; 329 return;
333 pr_warning("Could not create debugfs 'dyn_ftrace_total_info' entry\n"); 330 pr_warning("Could not create debugfs 'sysprof_sample_period' entry\n");
334} 331}
diff --git a/kernel/trace/trace_workqueue.c b/kernel/trace/trace_workqueue.c
new file mode 100644
index 000000000000..739fdacf873b
--- /dev/null
+++ b/kernel/trace/trace_workqueue.c
@@ -0,0 +1,280 @@
1/*
2 * Workqueue statistical tracer.
3 *
4 * Copyright (C) 2008 Frederic Weisbecker <fweisbec@gmail.com>
5 *
6 */
7
8
9#include <trace/workqueue.h>
10#include <linux/list.h>
11#include <linux/percpu.h>
12#include "trace_stat.h"
13#include "trace.h"
14
15
16/* A cpu workqueue thread */
17struct cpu_workqueue_stats {
18 struct list_head list;
19/* Useful to know if we print the cpu headers */
20 bool first_entry;
21 int cpu;
22 pid_t pid;
23/* Can be inserted from interrupt or user context, need to be atomic */
24 atomic_t inserted;
25/*
26 * Don't need to be atomic, works are serialized in a single workqueue thread
27 * on a single CPU.
28 */
29 unsigned int executed;
30};
31
32/* List of workqueue threads on one cpu */
33struct workqueue_global_stats {
34 struct list_head list;
35 spinlock_t lock;
36};
37
38/* Don't need a global lock because allocated before the workqueues, and
39 * never freed.
40 */
41static DEFINE_PER_CPU(struct workqueue_global_stats, all_workqueue_stat);
42#define workqueue_cpu_stat(cpu) (&per_cpu(all_workqueue_stat, cpu))
43
44/* Insertion of a work */
45static void
46probe_workqueue_insertion(struct task_struct *wq_thread,
47 struct work_struct *work)
48{
49 int cpu = cpumask_first(&wq_thread->cpus_allowed);
50 struct cpu_workqueue_stats *node, *next;
51 unsigned long flags;
52
53 spin_lock_irqsave(&workqueue_cpu_stat(cpu)->lock, flags);
54 list_for_each_entry_safe(node, next, &workqueue_cpu_stat(cpu)->list,
55 list) {
56 if (node->pid == wq_thread->pid) {
57 atomic_inc(&node->inserted);
58 goto found;
59 }
60 }
61 pr_debug("trace_workqueue: entry not found\n");
62found:
63 spin_unlock_irqrestore(&workqueue_cpu_stat(cpu)->lock, flags);
64}
65
66/* Execution of a work */
67static void
68probe_workqueue_execution(struct task_struct *wq_thread,
69 struct work_struct *work)
70{
71 int cpu = cpumask_first(&wq_thread->cpus_allowed);
72 struct cpu_workqueue_stats *node, *next;
73 unsigned long flags;
74
75 spin_lock_irqsave(&workqueue_cpu_stat(cpu)->lock, flags);
76 list_for_each_entry_safe(node, next, &workqueue_cpu_stat(cpu)->list,
77 list) {
78 if (node->pid == wq_thread->pid) {
79 node->executed++;
80 goto found;
81 }
82 }
83 pr_debug("trace_workqueue: entry not found\n");
84found:
85 spin_unlock_irqrestore(&workqueue_cpu_stat(cpu)->lock, flags);
86}
87
88/* Creation of a cpu workqueue thread */
89static void probe_workqueue_creation(struct task_struct *wq_thread, int cpu)
90{
91 struct cpu_workqueue_stats *cws;
92 unsigned long flags;
93
94 WARN_ON(cpu < 0);
95
96 /* Workqueues are sometimes created in atomic context */
97 cws = kzalloc(sizeof(struct cpu_workqueue_stats), GFP_ATOMIC);
98 if (!cws) {
99 pr_warning("trace_workqueue: not enough memory\n");
100 return;
101 }
102 INIT_LIST_HEAD(&cws->list);
103 cws->cpu = cpu;
104
105 cws->pid = wq_thread->pid;
106
107 spin_lock_irqsave(&workqueue_cpu_stat(cpu)->lock, flags);
108 if (list_empty(&workqueue_cpu_stat(cpu)->list))
109 cws->first_entry = true;
110 list_add_tail(&cws->list, &workqueue_cpu_stat(cpu)->list);
111 spin_unlock_irqrestore(&workqueue_cpu_stat(cpu)->lock, flags);
112}
113
114/* Destruction of a cpu workqueue thread */
115static void probe_workqueue_destruction(struct task_struct *wq_thread)
116{
117 /* Workqueue only execute on one cpu */
118 int cpu = cpumask_first(&wq_thread->cpus_allowed);
119 struct cpu_workqueue_stats *node, *next;
120 unsigned long flags;
121
122 spin_lock_irqsave(&workqueue_cpu_stat(cpu)->lock, flags);
123 list_for_each_entry_safe(node, next, &workqueue_cpu_stat(cpu)->list,
124 list) {
125 if (node->pid == wq_thread->pid) {
126 list_del(&node->list);
127 kfree(node);
128 goto found;
129 }
130 }
131
132 pr_debug("trace_workqueue: don't find workqueue to destroy\n");
133found:
134 spin_unlock_irqrestore(&workqueue_cpu_stat(cpu)->lock, flags);
135
136}
137
138static struct cpu_workqueue_stats *workqueue_stat_start_cpu(int cpu)
139{
140 unsigned long flags;
141 struct cpu_workqueue_stats *ret = NULL;
142
143
144 spin_lock_irqsave(&workqueue_cpu_stat(cpu)->lock, flags);
145
146 if (!list_empty(&workqueue_cpu_stat(cpu)->list))
147 ret = list_entry(workqueue_cpu_stat(cpu)->list.next,
148 struct cpu_workqueue_stats, list);
149
150 spin_unlock_irqrestore(&workqueue_cpu_stat(cpu)->lock, flags);
151
152 return ret;
153}
154
155static void *workqueue_stat_start(void)
156{
157 int cpu;
158 void *ret = NULL;
159
160 for_each_possible_cpu(cpu) {
161 ret = workqueue_stat_start_cpu(cpu);
162 if (ret)
163 return ret;
164 }
165 return NULL;
166}
167
168static void *workqueue_stat_next(void *prev, int idx)
169{
170 struct cpu_workqueue_stats *prev_cws = prev;
171 int cpu = prev_cws->cpu;
172 unsigned long flags;
173 void *ret = NULL;
174
175 spin_lock_irqsave(&workqueue_cpu_stat(cpu)->lock, flags);
176 if (list_is_last(&prev_cws->list, &workqueue_cpu_stat(cpu)->list)) {
177 spin_unlock_irqrestore(&workqueue_cpu_stat(cpu)->lock, flags);
178 do {
179 cpu = cpumask_next(cpu, cpu_possible_mask);
180 if (cpu >= nr_cpu_ids)
181 return NULL;
182 } while (!(ret = workqueue_stat_start_cpu(cpu)));
183 return ret;
184 }
185 spin_unlock_irqrestore(&workqueue_cpu_stat(cpu)->lock, flags);
186
187 return list_entry(prev_cws->list.next, struct cpu_workqueue_stats,
188 list);
189}
190
191static int workqueue_stat_show(struct seq_file *s, void *p)
192{
193 struct cpu_workqueue_stats *cws = p;
194 unsigned long flags;
195 int cpu = cws->cpu;
196 struct task_struct *tsk = find_task_by_vpid(cws->pid);
197
198 seq_printf(s, "%3d %6d %6u %s\n", cws->cpu,
199 atomic_read(&cws->inserted),
200 cws->executed,
201 tsk ? tsk->comm : "<...>");
202
203 spin_lock_irqsave(&workqueue_cpu_stat(cpu)->lock, flags);
204 if (&cws->list == workqueue_cpu_stat(cpu)->list.next)
205 seq_printf(s, "\n");
206 spin_unlock_irqrestore(&workqueue_cpu_stat(cpu)->lock, flags);
207
208 return 0;
209}
210
211static int workqueue_stat_headers(struct seq_file *s)
212{
213 seq_printf(s, "# CPU INSERTED EXECUTED NAME\n");
214 seq_printf(s, "# | | | |\n\n");
215 return 0;
216}
217
218struct tracer_stat workqueue_stats __read_mostly = {
219 .name = "workqueues",
220 .stat_start = workqueue_stat_start,
221 .stat_next = workqueue_stat_next,
222 .stat_show = workqueue_stat_show,
223 .stat_headers = workqueue_stat_headers
224};
225
226
227int __init stat_workqueue_init(void)
228{
229 if (register_stat_tracer(&workqueue_stats)) {
230 pr_warning("Unable to register workqueue stat tracer\n");
231 return 1;
232 }
233
234 return 0;
235}
236fs_initcall(stat_workqueue_init);
237
238/*
239 * Workqueues are created very early, just after pre-smp initcalls.
240 * So we must register our tracepoints at this stage.
241 */
242int __init trace_workqueue_early_init(void)
243{
244 int ret, cpu;
245
246 ret = register_trace_workqueue_insertion(probe_workqueue_insertion);
247 if (ret)
248 goto out;
249
250 ret = register_trace_workqueue_execution(probe_workqueue_execution);
251 if (ret)
252 goto no_insertion;
253
254 ret = register_trace_workqueue_creation(probe_workqueue_creation);
255 if (ret)
256 goto no_execution;
257
258 ret = register_trace_workqueue_destruction(probe_workqueue_destruction);
259 if (ret)
260 goto no_creation;
261
262 for_each_possible_cpu(cpu) {
263 spin_lock_init(&workqueue_cpu_stat(cpu)->lock);
264 INIT_LIST_HEAD(&workqueue_cpu_stat(cpu)->list);
265 }
266
267 return 0;
268
269no_creation:
270 unregister_trace_workqueue_creation(probe_workqueue_creation);
271no_execution:
272 unregister_trace_workqueue_execution(probe_workqueue_execution);
273no_insertion:
274 unregister_trace_workqueue_insertion(probe_workqueue_insertion);
275out:
276 pr_warning("trace_workqueue: unable to trace workqueues\n");
277
278 return 1;
279}
280early_initcall(trace_workqueue_early_init);
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index 1f0c509b40d3..e53ee18ef431 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -33,6 +33,7 @@
33#include <linux/kallsyms.h> 33#include <linux/kallsyms.h>
34#include <linux/debug_locks.h> 34#include <linux/debug_locks.h>
35#include <linux/lockdep.h> 35#include <linux/lockdep.h>
36#include <trace/workqueue.h>
36 37
37/* 38/*
38 * The per-CPU workqueue (if single thread, we always use the first 39 * The per-CPU workqueue (if single thread, we always use the first
@@ -125,9 +126,13 @@ struct cpu_workqueue_struct *get_wq_data(struct work_struct *work)
125 return (void *) (atomic_long_read(&work->data) & WORK_STRUCT_WQ_DATA_MASK); 126 return (void *) (atomic_long_read(&work->data) & WORK_STRUCT_WQ_DATA_MASK);
126} 127}
127 128
129DEFINE_TRACE(workqueue_insertion);
130
128static void insert_work(struct cpu_workqueue_struct *cwq, 131static void insert_work(struct cpu_workqueue_struct *cwq,
129 struct work_struct *work, struct list_head *head) 132 struct work_struct *work, struct list_head *head)
130{ 133{
134 trace_workqueue_insertion(cwq->thread, work);
135
131 set_wq_data(work, cwq); 136 set_wq_data(work, cwq);
132 /* 137 /*
133 * Ensure that we get the right work->data if we see the 138 * Ensure that we get the right work->data if we see the
@@ -259,6 +264,8 @@ int queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
259} 264}
260EXPORT_SYMBOL_GPL(queue_delayed_work_on); 265EXPORT_SYMBOL_GPL(queue_delayed_work_on);
261 266
267DEFINE_TRACE(workqueue_execution);
268
262static void run_workqueue(struct cpu_workqueue_struct *cwq) 269static void run_workqueue(struct cpu_workqueue_struct *cwq)
263{ 270{
264 spin_lock_irq(&cwq->lock); 271 spin_lock_irq(&cwq->lock);
@@ -284,7 +291,7 @@ static void run_workqueue(struct cpu_workqueue_struct *cwq)
284 */ 291 */
285 struct lockdep_map lockdep_map = work->lockdep_map; 292 struct lockdep_map lockdep_map = work->lockdep_map;
286#endif 293#endif
287 294 trace_workqueue_execution(cwq->thread, work);
288 cwq->current_work = work; 295 cwq->current_work = work;
289 list_del_init(cwq->worklist.next); 296 list_del_init(cwq->worklist.next);
290 spin_unlock_irq(&cwq->lock); 297 spin_unlock_irq(&cwq->lock);
@@ -765,6 +772,8 @@ init_cpu_workqueue(struct workqueue_struct *wq, int cpu)
765 return cwq; 772 return cwq;
766} 773}
767 774
775DEFINE_TRACE(workqueue_creation);
776
768static int create_workqueue_thread(struct cpu_workqueue_struct *cwq, int cpu) 777static int create_workqueue_thread(struct cpu_workqueue_struct *cwq, int cpu)
769{ 778{
770 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 }; 779 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
@@ -787,6 +796,8 @@ static int create_workqueue_thread(struct cpu_workqueue_struct *cwq, int cpu)
787 sched_setscheduler_nocheck(p, SCHED_FIFO, &param); 796 sched_setscheduler_nocheck(p, SCHED_FIFO, &param);
788 cwq->thread = p; 797 cwq->thread = p;
789 798
799 trace_workqueue_creation(cwq->thread, cpu);
800
790 return 0; 801 return 0;
791} 802}
792 803
@@ -868,6 +879,8 @@ struct workqueue_struct *__create_workqueue_key(const char *name,
868} 879}
869EXPORT_SYMBOL_GPL(__create_workqueue_key); 880EXPORT_SYMBOL_GPL(__create_workqueue_key);
870 881
882DEFINE_TRACE(workqueue_destruction);
883
871static void cleanup_workqueue_thread(struct cpu_workqueue_struct *cwq) 884static void cleanup_workqueue_thread(struct cpu_workqueue_struct *cwq)
872{ 885{
873 /* 886 /*
@@ -891,6 +904,7 @@ static void cleanup_workqueue_thread(struct cpu_workqueue_struct *cwq)
891 * checks list_empty(), and a "normal" queue_work() can't use 904 * checks list_empty(), and a "normal" queue_work() can't use
892 * a dead CPU. 905 * a dead CPU.
893 */ 906 */
907 trace_workqueue_destruction(cwq->thread);
894 kthread_stop(cwq->thread); 908 kthread_stop(cwq->thread);
895 cwq->thread = NULL; 909 cwq->thread = NULL;
896} 910}
generated by cgit v1.2.2 (git 2.25.0) at 2025-12-30 16:50:39 -0500