aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/trace
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/trace')
-rw-r--r--kernel/trace/Makefile1
-rw-r--r--kernel/trace/ftrace.c580
-rw-r--r--kernel/trace/trace.c161
-rw-r--r--kernel/trace/trace.h18
-rw-r--r--kernel/trace/trace_functions.c4
-rw-r--r--kernel/trace/trace_irqsoff.c16
-rw-r--r--kernel/trace/trace_mmiotrace.c295
-rw-r--r--kernel/trace/trace_sched_switch.c49
-rw-r--r--kernel/trace/trace_sched_wakeup.c67
-rw-r--r--kernel/trace/trace_selftest.c1
10 files changed, 980 insertions, 212 deletions
diff --git a/kernel/trace/Makefile b/kernel/trace/Makefile
index 7aec123ec1d8..71d17de17288 100644
--- a/kernel/trace/Makefile
+++ b/kernel/trace/Makefile
@@ -19,5 +19,6 @@ obj-$(CONFIG_FTRACE) += trace_functions.o
19obj-$(CONFIG_IRQSOFF_TRACER) += trace_irqsoff.o 19obj-$(CONFIG_IRQSOFF_TRACER) += trace_irqsoff.o
20obj-$(CONFIG_PREEMPT_TRACER) += trace_irqsoff.o 20obj-$(CONFIG_PREEMPT_TRACER) += trace_irqsoff.o
21obj-$(CONFIG_SCHED_TRACER) += trace_sched_wakeup.o 21obj-$(CONFIG_SCHED_TRACER) += trace_sched_wakeup.o
22obj-$(CONFIG_MMIOTRACE) += trace_mmiotrace.o
22 23
23libftrace-y := ftrace.o 24libftrace-y := ftrace.o
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
index 89bd9a6f52ec..0f271c45cd02 100644
--- a/kernel/trace/ftrace.c
+++ b/kernel/trace/ftrace.c
@@ -21,12 +21,15 @@
21#include <linux/hardirq.h> 21#include <linux/hardirq.h>
22#include <linux/kthread.h> 22#include <linux/kthread.h>
23#include <linux/uaccess.h> 23#include <linux/uaccess.h>
24#include <linux/kprobes.h>
24#include <linux/ftrace.h> 25#include <linux/ftrace.h>
25#include <linux/sysctl.h> 26#include <linux/sysctl.h>
26#include <linux/ctype.h> 27#include <linux/ctype.h>
27#include <linux/hash.h> 28#include <linux/hash.h>
28#include <linux/list.h> 29#include <linux/list.h>
29 30
31#include <asm/ftrace.h>
32
30#include "trace.h" 33#include "trace.h"
31 34
32/* ftrace_enabled is a method to turn ftrace on or off */ 35/* ftrace_enabled is a method to turn ftrace on or off */
@@ -50,7 +53,7 @@ static struct ftrace_ops ftrace_list_end __read_mostly =
50static struct ftrace_ops *ftrace_list __read_mostly = &ftrace_list_end; 53static struct ftrace_ops *ftrace_list __read_mostly = &ftrace_list_end;
51ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub; 54ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
52 55
53void ftrace_list_func(unsigned long ip, unsigned long parent_ip) 56static void ftrace_list_func(unsigned long ip, unsigned long parent_ip)
54{ 57{
55 struct ftrace_ops *op = ftrace_list; 58 struct ftrace_ops *op = ftrace_list;
56 59
@@ -151,8 +154,6 @@ static int __unregister_ftrace_function(struct ftrace_ops *ops)
151#ifdef CONFIG_DYNAMIC_FTRACE 154#ifdef CONFIG_DYNAMIC_FTRACE
152 155
153static struct task_struct *ftraced_task; 156static struct task_struct *ftraced_task;
154static DECLARE_WAIT_QUEUE_HEAD(ftraced_waiters);
155static unsigned long ftraced_iteration_counter;
156 157
157enum { 158enum {
158 FTRACE_ENABLE_CALLS = (1 << 0), 159 FTRACE_ENABLE_CALLS = (1 << 0),
@@ -163,6 +164,8 @@ enum {
163}; 164};
164 165
165static int ftrace_filtered; 166static int ftrace_filtered;
167static int tracing_on;
168static int frozen_record_count;
166 169
167static struct hlist_head ftrace_hash[FTRACE_HASHSIZE]; 170static struct hlist_head ftrace_hash[FTRACE_HASHSIZE];
168 171
@@ -170,7 +173,7 @@ static DEFINE_PER_CPU(int, ftrace_shutdown_disable_cpu);
170 173
171static DEFINE_SPINLOCK(ftrace_shutdown_lock); 174static DEFINE_SPINLOCK(ftrace_shutdown_lock);
172static DEFINE_MUTEX(ftraced_lock); 175static DEFINE_MUTEX(ftraced_lock);
173static DEFINE_MUTEX(ftrace_filter_lock); 176static DEFINE_MUTEX(ftrace_regex_lock);
174 177
175struct ftrace_page { 178struct ftrace_page {
176 struct ftrace_page *next; 179 struct ftrace_page *next;
@@ -189,11 +192,77 @@ static struct ftrace_page *ftrace_pages;
189 192
190static int ftraced_trigger; 193static int ftraced_trigger;
191static int ftraced_suspend; 194static int ftraced_suspend;
195static int ftraced_stop;
192 196
193static int ftrace_record_suspend; 197static int ftrace_record_suspend;
194 198
195static struct dyn_ftrace *ftrace_free_records; 199static struct dyn_ftrace *ftrace_free_records;
196 200
201
202#ifdef CONFIG_KPROBES
203static inline void freeze_record(struct dyn_ftrace *rec)
204{
205 if (!(rec->flags & FTRACE_FL_FROZEN)) {
206 rec->flags |= FTRACE_FL_FROZEN;
207 frozen_record_count++;
208 }
209}
210
211static inline void unfreeze_record(struct dyn_ftrace *rec)
212{
213 if (rec->flags & FTRACE_FL_FROZEN) {
214 rec->flags &= ~FTRACE_FL_FROZEN;
215 frozen_record_count--;
216 }
217}
218
219static inline int record_frozen(struct dyn_ftrace *rec)
220{
221 return rec->flags & FTRACE_FL_FROZEN;
222}
223#else
224# define freeze_record(rec) ({ 0; })
225# define unfreeze_record(rec) ({ 0; })
226# define record_frozen(rec) ({ 0; })
227#endif /* CONFIG_KPROBES */
228
229int skip_trace(unsigned long ip)
230{
231 unsigned long fl;
232 struct dyn_ftrace *rec;
233 struct hlist_node *t;
234 struct hlist_head *head;
235
236 if (frozen_record_count == 0)
237 return 0;
238
239 head = &ftrace_hash[hash_long(ip, FTRACE_HASHBITS)];
240 hlist_for_each_entry_rcu(rec, t, head, node) {
241 if (rec->ip == ip) {
242 if (record_frozen(rec)) {
243 if (rec->flags & FTRACE_FL_FAILED)
244 return 1;
245
246 if (!(rec->flags & FTRACE_FL_CONVERTED))
247 return 1;
248
249 if (!tracing_on || !ftrace_enabled)
250 return 1;
251
252 if (ftrace_filtered) {
253 fl = rec->flags & (FTRACE_FL_FILTER |
254 FTRACE_FL_NOTRACE);
255 if (!fl || (fl & FTRACE_FL_NOTRACE))
256 return 1;
257 }
258 }
259 break;
260 }
261 }
262
263 return 0;
264}
265
197static inline int 266static inline int
198ftrace_ip_in_hash(unsigned long ip, unsigned long key) 267ftrace_ip_in_hash(unsigned long ip, unsigned long key)
199{ 268{
@@ -201,7 +270,7 @@ ftrace_ip_in_hash(unsigned long ip, unsigned long key)
201 struct hlist_node *t; 270 struct hlist_node *t;
202 int found = 0; 271 int found = 0;
203 272
204 hlist_for_each_entry(p, t, &ftrace_hash[key], node) { 273 hlist_for_each_entry_rcu(p, t, &ftrace_hash[key], node) {
205 if (p->ip == ip) { 274 if (p->ip == ip) {
206 found = 1; 275 found = 1;
207 break; 276 break;
@@ -214,7 +283,13 @@ ftrace_ip_in_hash(unsigned long ip, unsigned long key)
214static inline void 283static inline void
215ftrace_add_hash(struct dyn_ftrace *node, unsigned long key) 284ftrace_add_hash(struct dyn_ftrace *node, unsigned long key)
216{ 285{
217 hlist_add_head(&node->node, &ftrace_hash[key]); 286 hlist_add_head_rcu(&node->node, &ftrace_hash[key]);
287}
288
289/* called from kstop_machine */
290static inline void ftrace_del_hash(struct dyn_ftrace *node)
291{
292 hlist_del(&node->node);
218} 293}
219 294
220static void ftrace_free_rec(struct dyn_ftrace *rec) 295static void ftrace_free_rec(struct dyn_ftrace *rec)
@@ -301,13 +376,6 @@ ftrace_record_ip(unsigned long ip)
301 if (ftrace_ip_in_hash(ip, key)) 376 if (ftrace_ip_in_hash(ip, key))
302 goto out_unlock; 377 goto out_unlock;
303 378
304 /*
305 * There's a slight race that the ftraced will update the
306 * hash and reset here. If it is already converted, skip it.
307 */
308 if (ftrace_ip_converted(ip))
309 goto out_unlock;
310
311 node = ftrace_alloc_dyn_node(ip); 379 node = ftrace_alloc_dyn_node(ip);
312 if (!node) 380 if (!node)
313 goto out_unlock; 381 goto out_unlock;
@@ -331,19 +399,16 @@ ftrace_record_ip(unsigned long ip)
331} 399}
332 400
333#define FTRACE_ADDR ((long)(ftrace_caller)) 401#define FTRACE_ADDR ((long)(ftrace_caller))
334#define MCOUNT_ADDR ((long)(mcount))
335 402
336static void 403static int
337__ftrace_replace_code(struct dyn_ftrace *rec, 404__ftrace_replace_code(struct dyn_ftrace *rec,
338 unsigned char *old, unsigned char *new, int enable) 405 unsigned char *old, unsigned char *new, int enable)
339{ 406{
340 unsigned long ip; 407 unsigned long ip, fl;
341 int failed;
342 408
343 ip = rec->ip; 409 ip = rec->ip;
344 410
345 if (ftrace_filtered && enable) { 411 if (ftrace_filtered && enable) {
346 unsigned long fl;
347 /* 412 /*
348 * If filtering is on: 413 * If filtering is on:
349 * 414 *
@@ -356,20 +421,29 @@ __ftrace_replace_code(struct dyn_ftrace *rec,
356 * If this record is not set to be filtered 421 * If this record is not set to be filtered
357 * and it is not enabled do nothing. 422 * and it is not enabled do nothing.
358 * 423 *
424 * If this record is set not to trace then
425 * do nothing.
426 *
427 * If this record is set not to trace and
428 * it is enabled then disable it.
429 *
359 * If this record is not set to be filtered and 430 * If this record is not set to be filtered and
360 * it is enabled, disable it. 431 * it is enabled, disable it.
361 */ 432 */
362 fl = rec->flags & (FTRACE_FL_FILTER | FTRACE_FL_ENABLED); 433
434 fl = rec->flags & (FTRACE_FL_FILTER | FTRACE_FL_NOTRACE |
435 FTRACE_FL_ENABLED);
363 436
364 if ((fl == (FTRACE_FL_FILTER | FTRACE_FL_ENABLED)) || 437 if ((fl == (FTRACE_FL_FILTER | FTRACE_FL_ENABLED)) ||
365 (fl == 0)) 438 (fl == (FTRACE_FL_FILTER | FTRACE_FL_NOTRACE)) ||
366 return; 439 !fl || (fl == FTRACE_FL_NOTRACE))
440 return 0;
367 441
368 /* 442 /*
369 * If it is enabled disable it, 443 * If it is enabled disable it,
370 * otherwise enable it! 444 * otherwise enable it!
371 */ 445 */
372 if (fl == FTRACE_FL_ENABLED) { 446 if (fl & FTRACE_FL_ENABLED) {
373 /* swap new and old */ 447 /* swap new and old */
374 new = old; 448 new = old;
375 old = ftrace_call_replace(ip, FTRACE_ADDR); 449 old = ftrace_call_replace(ip, FTRACE_ADDR);
@@ -380,41 +454,39 @@ __ftrace_replace_code(struct dyn_ftrace *rec,
380 } 454 }
381 } else { 455 } else {
382 456
383 if (enable) 457 if (enable) {
458 /*
459 * If this record is set not to trace and is
460 * not enabled, do nothing.
461 */
462 fl = rec->flags & (FTRACE_FL_NOTRACE | FTRACE_FL_ENABLED);
463 if (fl == FTRACE_FL_NOTRACE)
464 return 0;
465
384 new = ftrace_call_replace(ip, FTRACE_ADDR); 466 new = ftrace_call_replace(ip, FTRACE_ADDR);
385 else 467 } else
386 old = ftrace_call_replace(ip, FTRACE_ADDR); 468 old = ftrace_call_replace(ip, FTRACE_ADDR);
387 469
388 if (enable) { 470 if (enable) {
389 if (rec->flags & FTRACE_FL_ENABLED) 471 if (rec->flags & FTRACE_FL_ENABLED)
390 return; 472 return 0;
391 rec->flags |= FTRACE_FL_ENABLED; 473 rec->flags |= FTRACE_FL_ENABLED;
392 } else { 474 } else {
393 if (!(rec->flags & FTRACE_FL_ENABLED)) 475 if (!(rec->flags & FTRACE_FL_ENABLED))
394 return; 476 return 0;
395 rec->flags &= ~FTRACE_FL_ENABLED; 477 rec->flags &= ~FTRACE_FL_ENABLED;
396 } 478 }
397 } 479 }
398 480
399 failed = ftrace_modify_code(ip, old, new); 481 return ftrace_modify_code(ip, old, new);
400 if (failed) {
401 unsigned long key;
402 /* It is possible that the function hasn't been converted yet */
403 key = hash_long(ip, FTRACE_HASHBITS);
404 if (!ftrace_ip_in_hash(ip, key)) {
405 rec->flags |= FTRACE_FL_FAILED;
406 ftrace_free_rec(rec);
407 }
408
409 }
410} 482}
411 483
412static void ftrace_replace_code(int enable) 484static void ftrace_replace_code(int enable)
413{ 485{
486 int i, failed;
414 unsigned char *new = NULL, *old = NULL; 487 unsigned char *new = NULL, *old = NULL;
415 struct dyn_ftrace *rec; 488 struct dyn_ftrace *rec;
416 struct ftrace_page *pg; 489 struct ftrace_page *pg;
417 int i;
418 490
419 if (enable) 491 if (enable)
420 old = ftrace_nop_replace(); 492 old = ftrace_nop_replace();
@@ -429,7 +501,23 @@ static void ftrace_replace_code(int enable)
429 if (rec->flags & FTRACE_FL_FAILED) 501 if (rec->flags & FTRACE_FL_FAILED)
430 continue; 502 continue;
431 503
432 __ftrace_replace_code(rec, old, new, enable); 504 /* ignore updates to this record's mcount site */
505 if (get_kprobe((void *)rec->ip)) {
506 freeze_record(rec);
507 continue;
508 } else {
509 unfreeze_record(rec);
510 }
511
512 failed = __ftrace_replace_code(rec, old, new, enable);
513 if (failed && (rec->flags & FTRACE_FL_CONVERTED)) {
514 rec->flags |= FTRACE_FL_FAILED;
515 if ((system_state == SYSTEM_BOOTING) ||
516 !core_kernel_text(rec->ip)) {
517 ftrace_del_hash(rec);
518 ftrace_free_rec(rec);
519 }
520 }
433 } 521 }
434 } 522 }
435} 523}
@@ -443,7 +531,7 @@ static void ftrace_shutdown_replenish(void)
443 ftrace_pages->next = (void *)get_zeroed_page(GFP_KERNEL); 531 ftrace_pages->next = (void *)get_zeroed_page(GFP_KERNEL);
444} 532}
445 533
446static void 534static int
447ftrace_code_disable(struct dyn_ftrace *rec) 535ftrace_code_disable(struct dyn_ftrace *rec)
448{ 536{
449 unsigned long ip; 537 unsigned long ip;
@@ -458,19 +546,30 @@ ftrace_code_disable(struct dyn_ftrace *rec)
458 failed = ftrace_modify_code(ip, call, nop); 546 failed = ftrace_modify_code(ip, call, nop);
459 if (failed) { 547 if (failed) {
460 rec->flags |= FTRACE_FL_FAILED; 548 rec->flags |= FTRACE_FL_FAILED;
461 ftrace_free_rec(rec); 549 return 0;
462 } 550 }
551 return 1;
463} 552}
464 553
554static int __ftrace_update_code(void *ignore);
555
465static int __ftrace_modify_code(void *data) 556static int __ftrace_modify_code(void *data)
466{ 557{
467 unsigned long addr; 558 unsigned long addr;
468 int *command = data; 559 int *command = data;
469 560
470 if (*command & FTRACE_ENABLE_CALLS) 561 if (*command & FTRACE_ENABLE_CALLS) {
562 /*
563 * Update any recorded ips now that we have the
564 * machine stopped
565 */
566 __ftrace_update_code(NULL);
471 ftrace_replace_code(1); 567 ftrace_replace_code(1);
472 else if (*command & FTRACE_DISABLE_CALLS) 568 tracing_on = 1;
569 } else if (*command & FTRACE_DISABLE_CALLS) {
473 ftrace_replace_code(0); 570 ftrace_replace_code(0);
571 tracing_on = 0;
572 }
474 573
475 if (*command & FTRACE_UPDATE_TRACE_FUNC) 574 if (*command & FTRACE_UPDATE_TRACE_FUNC)
476 ftrace_update_ftrace_func(ftrace_trace_function); 575 ftrace_update_ftrace_func(ftrace_trace_function);
@@ -491,6 +590,25 @@ static void ftrace_run_update_code(int command)
491 stop_machine_run(__ftrace_modify_code, &command, NR_CPUS); 590 stop_machine_run(__ftrace_modify_code, &command, NR_CPUS);
492} 591}
493 592
593void ftrace_disable_daemon(void)
594{
595 /* Stop the daemon from calling kstop_machine */
596 mutex_lock(&ftraced_lock);
597 ftraced_stop = 1;
598 mutex_unlock(&ftraced_lock);
599
600 ftrace_force_update();
601}
602
603void ftrace_enable_daemon(void)
604{
605 mutex_lock(&ftraced_lock);
606 ftraced_stop = 0;
607 mutex_unlock(&ftraced_lock);
608
609 ftrace_force_update();
610}
611
494static ftrace_func_t saved_ftrace_func; 612static ftrace_func_t saved_ftrace_func;
495 613
496static void ftrace_startup(void) 614static void ftrace_startup(void)
@@ -583,14 +701,14 @@ unsigned long ftrace_update_tot_cnt;
583 701
584static int __ftrace_update_code(void *ignore) 702static int __ftrace_update_code(void *ignore)
585{ 703{
586 struct dyn_ftrace *p; 704 int i, save_ftrace_enabled;
587 struct hlist_head head;
588 struct hlist_node *t;
589 int save_ftrace_enabled;
590 cycle_t start, stop; 705 cycle_t start, stop;
591 int i; 706 struct dyn_ftrace *p;
707 struct hlist_node *t, *n;
708 struct hlist_head *head, temp_list;
592 709
593 /* Don't be recording funcs now */ 710 /* Don't be recording funcs now */
711 ftrace_record_suspend++;
594 save_ftrace_enabled = ftrace_enabled; 712 save_ftrace_enabled = ftrace_enabled;
595 ftrace_enabled = 0; 713 ftrace_enabled = 0;
596 714
@@ -599,35 +717,79 @@ static int __ftrace_update_code(void *ignore)
599 717
600 /* No locks needed, the machine is stopped! */ 718 /* No locks needed, the machine is stopped! */
601 for (i = 0; i < FTRACE_HASHSIZE; i++) { 719 for (i = 0; i < FTRACE_HASHSIZE; i++) {
602 if (hlist_empty(&ftrace_hash[i])) 720 INIT_HLIST_HEAD(&temp_list);
603 continue; 721 head = &ftrace_hash[i];
604
605 head = ftrace_hash[i];
606 INIT_HLIST_HEAD(&ftrace_hash[i]);
607 722
608 /* all CPUS are stopped, we are safe to modify code */ 723 /* all CPUS are stopped, we are safe to modify code */
609 hlist_for_each_entry(p, t, &head, node) { 724 hlist_for_each_entry_safe(p, t, n, head, node) {
610 ftrace_code_disable(p); 725 /* Skip over failed records which have not been
611 ftrace_update_cnt++; 726 * freed. */
727 if (p->flags & FTRACE_FL_FAILED)
728 continue;
729
730 /* Unconverted records are always at the head of the
731 * hash bucket. Once we encounter a converted record,
732 * simply skip over to the next bucket. Saves ftraced
733 * some processor cycles (ftrace does its bid for
734 * global warming :-p ). */
735 if (p->flags & (FTRACE_FL_CONVERTED))
736 break;
737
738 /* Ignore updates to this record's mcount site.
739 * Reintroduce this record at the head of this
740 * bucket to attempt to "convert" it again if
741 * the kprobe on it is unregistered before the
742 * next run. */
743 if (get_kprobe((void *)p->ip)) {
744 ftrace_del_hash(p);
745 INIT_HLIST_NODE(&p->node);
746 hlist_add_head(&p->node, &temp_list);
747 freeze_record(p);
748 continue;
749 } else {
750 unfreeze_record(p);
751 }
752
753 /* convert record (i.e, patch mcount-call with NOP) */
754 if (ftrace_code_disable(p)) {
755 p->flags |= FTRACE_FL_CONVERTED;
756 ftrace_update_cnt++;
757 } else {
758 if ((system_state == SYSTEM_BOOTING) ||
759 !core_kernel_text(p->ip)) {
760 ftrace_del_hash(p);
761 ftrace_free_rec(p);
762 }
763 }
612 } 764 }
613 765
766 hlist_for_each_entry_safe(p, t, n, &temp_list, node) {
767 hlist_del(&p->node);
768 INIT_HLIST_NODE(&p->node);
769 hlist_add_head(&p->node, head);
770 }
614 } 771 }
615 772
616 stop = ftrace_now(raw_smp_processor_id()); 773 stop = ftrace_now(raw_smp_processor_id());
617 ftrace_update_time = stop - start; 774 ftrace_update_time = stop - start;
618 ftrace_update_tot_cnt += ftrace_update_cnt; 775 ftrace_update_tot_cnt += ftrace_update_cnt;
776 ftraced_trigger = 0;
619 777
620 ftrace_enabled = save_ftrace_enabled; 778 ftrace_enabled = save_ftrace_enabled;
779 ftrace_record_suspend--;
621 780
622 return 0; 781 return 0;
623} 782}
624 783
625static void ftrace_update_code(void) 784static int ftrace_update_code(void)
626{ 785{
627 if (unlikely(ftrace_disabled)) 786 if (unlikely(ftrace_disabled) ||
628 return; 787 !ftrace_enabled || !ftraced_trigger)
788 return 0;
629 789
630 stop_machine_run(__ftrace_update_code, NULL, NR_CPUS); 790 stop_machine_run(__ftrace_update_code, NULL, NR_CPUS);
791
792 return 1;
631} 793}
632 794
633static int ftraced(void *ignore) 795static int ftraced(void *ignore)
@@ -646,14 +808,13 @@ static int ftraced(void *ignore)
646 808
647 mutex_lock(&ftrace_sysctl_lock); 809 mutex_lock(&ftrace_sysctl_lock);
648 mutex_lock(&ftraced_lock); 810 mutex_lock(&ftraced_lock);
649 if (ftrace_enabled && ftraced_trigger && !ftraced_suspend) { 811 if (!ftraced_suspend && !ftraced_stop &&
650 ftrace_record_suspend++; 812 ftrace_update_code()) {
651 ftrace_update_code();
652 usecs = nsecs_to_usecs(ftrace_update_time); 813 usecs = nsecs_to_usecs(ftrace_update_time);
653 if (ftrace_update_tot_cnt > 100000) { 814 if (ftrace_update_tot_cnt > 100000) {
654 ftrace_update_tot_cnt = 0; 815 ftrace_update_tot_cnt = 0;
655 pr_info("hm, dftrace overflow: %lu change%s" 816 pr_info("hm, dftrace overflow: %lu change%s"
656 " (%lu total) in %lu usec%s\n", 817 " (%lu total) in %lu usec%s\n",
657 ftrace_update_cnt, 818 ftrace_update_cnt,
658 ftrace_update_cnt != 1 ? "s" : "", 819 ftrace_update_cnt != 1 ? "s" : "",
659 ftrace_update_tot_cnt, 820 ftrace_update_tot_cnt,
@@ -661,15 +822,10 @@ static int ftraced(void *ignore)
661 ftrace_disabled = 1; 822 ftrace_disabled = 1;
662 WARN_ON_ONCE(1); 823 WARN_ON_ONCE(1);
663 } 824 }
664 ftraced_trigger = 0;
665 ftrace_record_suspend--;
666 } 825 }
667 ftraced_iteration_counter++;
668 mutex_unlock(&ftraced_lock); 826 mutex_unlock(&ftraced_lock);
669 mutex_unlock(&ftrace_sysctl_lock); 827 mutex_unlock(&ftrace_sysctl_lock);
670 828
671 wake_up_interruptible(&ftraced_waiters);
672
673 ftrace_shutdown_replenish(); 829 ftrace_shutdown_replenish();
674 } 830 }
675 __set_current_state(TASK_RUNNING); 831 __set_current_state(TASK_RUNNING);
@@ -721,6 +877,8 @@ static int __init ftrace_dyn_table_alloc(void)
721enum { 877enum {
722 FTRACE_ITER_FILTER = (1 << 0), 878 FTRACE_ITER_FILTER = (1 << 0),
723 FTRACE_ITER_CONT = (1 << 1), 879 FTRACE_ITER_CONT = (1 << 1),
880 FTRACE_ITER_NOTRACE = (1 << 2),
881 FTRACE_ITER_FAILURES = (1 << 3),
724}; 882};
725 883
726#define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */ 884#define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
@@ -752,9 +910,18 @@ t_next(struct seq_file *m, void *v, loff_t *pos)
752 } 910 }
753 } else { 911 } else {
754 rec = &iter->pg->records[iter->idx++]; 912 rec = &iter->pg->records[iter->idx++];
755 if ((rec->flags & FTRACE_FL_FAILED) || 913 if ((!(iter->flags & FTRACE_ITER_FAILURES) &&
914 (rec->flags & FTRACE_FL_FAILED)) ||
915
916 ((iter->flags & FTRACE_ITER_FAILURES) &&
917 (!(rec->flags & FTRACE_FL_FAILED) ||
918 (rec->flags & FTRACE_FL_FREE))) ||
919
756 ((iter->flags & FTRACE_ITER_FILTER) && 920 ((iter->flags & FTRACE_ITER_FILTER) &&
757 !(rec->flags & FTRACE_FL_FILTER))) { 921 !(rec->flags & FTRACE_FL_FILTER)) ||
922
923 ((iter->flags & FTRACE_ITER_NOTRACE) &&
924 !(rec->flags & FTRACE_FL_NOTRACE))) {
758 rec = NULL; 925 rec = NULL;
759 goto retry; 926 goto retry;
760 } 927 }
@@ -847,22 +1014,42 @@ int ftrace_avail_release(struct inode *inode, struct file *file)
847 return 0; 1014 return 0;
848} 1015}
849 1016
850static void ftrace_filter_reset(void) 1017static int
1018ftrace_failures_open(struct inode *inode, struct file *file)
1019{
1020 int ret;
1021 struct seq_file *m;
1022 struct ftrace_iterator *iter;
1023
1024 ret = ftrace_avail_open(inode, file);
1025 if (!ret) {
1026 m = (struct seq_file *)file->private_data;
1027 iter = (struct ftrace_iterator *)m->private;
1028 iter->flags = FTRACE_ITER_FAILURES;
1029 }
1030
1031 return ret;
1032}
1033
1034
1035static void ftrace_filter_reset(int enable)
851{ 1036{
852 struct ftrace_page *pg; 1037 struct ftrace_page *pg;
853 struct dyn_ftrace *rec; 1038 struct dyn_ftrace *rec;
1039 unsigned long type = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
854 unsigned i; 1040 unsigned i;
855 1041
856 /* keep kstop machine from running */ 1042 /* keep kstop machine from running */
857 preempt_disable(); 1043 preempt_disable();
858 ftrace_filtered = 0; 1044 if (enable)
1045 ftrace_filtered = 0;
859 pg = ftrace_pages_start; 1046 pg = ftrace_pages_start;
860 while (pg) { 1047 while (pg) {
861 for (i = 0; i < pg->index; i++) { 1048 for (i = 0; i < pg->index; i++) {
862 rec = &pg->records[i]; 1049 rec = &pg->records[i];
863 if (rec->flags & FTRACE_FL_FAILED) 1050 if (rec->flags & FTRACE_FL_FAILED)
864 continue; 1051 continue;
865 rec->flags &= ~FTRACE_FL_FILTER; 1052 rec->flags &= ~type;
866 } 1053 }
867 pg = pg->next; 1054 pg = pg->next;
868 } 1055 }
@@ -870,7 +1057,7 @@ static void ftrace_filter_reset(void)
870} 1057}
871 1058
872static int 1059static int
873ftrace_filter_open(struct inode *inode, struct file *file) 1060ftrace_regex_open(struct inode *inode, struct file *file, int enable)
874{ 1061{
875 struct ftrace_iterator *iter; 1062 struct ftrace_iterator *iter;
876 int ret = 0; 1063 int ret = 0;
@@ -882,15 +1069,16 @@ ftrace_filter_open(struct inode *inode, struct file *file)
882 if (!iter) 1069 if (!iter)
883 return -ENOMEM; 1070 return -ENOMEM;
884 1071
885 mutex_lock(&ftrace_filter_lock); 1072 mutex_lock(&ftrace_regex_lock);
886 if ((file->f_mode & FMODE_WRITE) && 1073 if ((file->f_mode & FMODE_WRITE) &&
887 !(file->f_flags & O_APPEND)) 1074 !(file->f_flags & O_APPEND))
888 ftrace_filter_reset(); 1075 ftrace_filter_reset(enable);
889 1076
890 if (file->f_mode & FMODE_READ) { 1077 if (file->f_mode & FMODE_READ) {
891 iter->pg = ftrace_pages_start; 1078 iter->pg = ftrace_pages_start;
892 iter->pos = -1; 1079 iter->pos = -1;
893 iter->flags = FTRACE_ITER_FILTER; 1080 iter->flags = enable ? FTRACE_ITER_FILTER :
1081 FTRACE_ITER_NOTRACE;
894 1082
895 ret = seq_open(file, &show_ftrace_seq_ops); 1083 ret = seq_open(file, &show_ftrace_seq_ops);
896 if (!ret) { 1084 if (!ret) {
@@ -900,13 +1088,25 @@ ftrace_filter_open(struct inode *inode, struct file *file)
900 kfree(iter); 1088 kfree(iter);
901 } else 1089 } else
902 file->private_data = iter; 1090 file->private_data = iter;
903 mutex_unlock(&ftrace_filter_lock); 1091 mutex_unlock(&ftrace_regex_lock);
904 1092
905 return ret; 1093 return ret;
906} 1094}
907 1095
1096static int
1097ftrace_filter_open(struct inode *inode, struct file *file)
1098{
1099 return ftrace_regex_open(inode, file, 1);
1100}
1101
1102static int
1103ftrace_notrace_open(struct inode *inode, struct file *file)
1104{
1105 return ftrace_regex_open(inode, file, 0);
1106}
1107
908static ssize_t 1108static ssize_t
909ftrace_filter_read(struct file *file, char __user *ubuf, 1109ftrace_regex_read(struct file *file, char __user *ubuf,
910 size_t cnt, loff_t *ppos) 1110 size_t cnt, loff_t *ppos)
911{ 1111{
912 if (file->f_mode & FMODE_READ) 1112 if (file->f_mode & FMODE_READ)
@@ -916,7 +1116,7 @@ ftrace_filter_read(struct file *file, char __user *ubuf,
916} 1116}
917 1117
918static loff_t 1118static loff_t
919ftrace_filter_lseek(struct file *file, loff_t offset, int origin) 1119ftrace_regex_lseek(struct file *file, loff_t offset, int origin)
920{ 1120{
921 loff_t ret; 1121 loff_t ret;
922 1122
@@ -936,13 +1136,14 @@ enum {
936}; 1136};
937 1137
938static void 1138static void
939ftrace_match(unsigned char *buff, int len) 1139ftrace_match(unsigned char *buff, int len, int enable)
940{ 1140{
941 char str[KSYM_SYMBOL_LEN]; 1141 char str[KSYM_SYMBOL_LEN];
942 char *search = NULL; 1142 char *search = NULL;
943 struct ftrace_page *pg; 1143 struct ftrace_page *pg;
944 struct dyn_ftrace *rec; 1144 struct dyn_ftrace *rec;
945 int type = MATCH_FULL; 1145 int type = MATCH_FULL;
1146 unsigned long flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
946 unsigned i, match = 0, search_len = 0; 1147 unsigned i, match = 0, search_len = 0;
947 1148
948 for (i = 0; i < len; i++) { 1149 for (i = 0; i < len; i++) {
@@ -966,7 +1167,8 @@ ftrace_match(unsigned char *buff, int len)
966 1167
967 /* keep kstop machine from running */ 1168 /* keep kstop machine from running */
968 preempt_disable(); 1169 preempt_disable();
969 ftrace_filtered = 1; 1170 if (enable)
1171 ftrace_filtered = 1;
970 pg = ftrace_pages_start; 1172 pg = ftrace_pages_start;
971 while (pg) { 1173 while (pg) {
972 for (i = 0; i < pg->index; i++) { 1174 for (i = 0; i < pg->index; i++) {
@@ -997,7 +1199,7 @@ ftrace_match(unsigned char *buff, int len)
997 break; 1199 break;
998 } 1200 }
999 if (matched) 1201 if (matched)
1000 rec->flags |= FTRACE_FL_FILTER; 1202 rec->flags |= flag;
1001 } 1203 }
1002 pg = pg->next; 1204 pg = pg->next;
1003 } 1205 }
@@ -1005,8 +1207,8 @@ ftrace_match(unsigned char *buff, int len)
1005} 1207}
1006 1208
1007static ssize_t 1209static ssize_t
1008ftrace_filter_write(struct file *file, const char __user *ubuf, 1210ftrace_regex_write(struct file *file, const char __user *ubuf,
1009 size_t cnt, loff_t *ppos) 1211 size_t cnt, loff_t *ppos, int enable)
1010{ 1212{
1011 struct ftrace_iterator *iter; 1213 struct ftrace_iterator *iter;
1012 char ch; 1214 char ch;
@@ -1016,7 +1218,7 @@ ftrace_filter_write(struct file *file, const char __user *ubuf,
1016 if (!cnt || cnt < 0) 1218 if (!cnt || cnt < 0)
1017 return 0; 1219 return 0;
1018 1220
1019 mutex_lock(&ftrace_filter_lock); 1221 mutex_lock(&ftrace_regex_lock);
1020 1222
1021 if (file->f_mode & FMODE_READ) { 1223 if (file->f_mode & FMODE_READ) {
1022 struct seq_file *m = file->private_data; 1224 struct seq_file *m = file->private_data;
@@ -1045,7 +1247,6 @@ ftrace_filter_write(struct file *file, const char __user *ubuf,
1045 cnt--; 1247 cnt--;
1046 } 1248 }
1047 1249
1048
1049 if (isspace(ch)) { 1250 if (isspace(ch)) {
1050 file->f_pos += read; 1251 file->f_pos += read;
1051 ret = read; 1252 ret = read;
@@ -1072,7 +1273,7 @@ ftrace_filter_write(struct file *file, const char __user *ubuf,
1072 if (isspace(ch)) { 1273 if (isspace(ch)) {
1073 iter->filtered++; 1274 iter->filtered++;
1074 iter->buffer[iter->buffer_idx] = 0; 1275 iter->buffer[iter->buffer_idx] = 0;
1075 ftrace_match(iter->buffer, iter->buffer_idx); 1276 ftrace_match(iter->buffer, iter->buffer_idx, enable);
1076 iter->buffer_idx = 0; 1277 iter->buffer_idx = 0;
1077 } else 1278 } else
1078 iter->flags |= FTRACE_ITER_CONT; 1279 iter->flags |= FTRACE_ITER_CONT;
@@ -1082,11 +1283,39 @@ ftrace_filter_write(struct file *file, const char __user *ubuf,
1082 1283
1083 ret = read; 1284 ret = read;
1084 out: 1285 out:
1085 mutex_unlock(&ftrace_filter_lock); 1286 mutex_unlock(&ftrace_regex_lock);
1086 1287
1087 return ret; 1288 return ret;
1088} 1289}
1089 1290
1291static ssize_t
1292ftrace_filter_write(struct file *file, const char __user *ubuf,
1293 size_t cnt, loff_t *ppos)
1294{
1295 return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
1296}
1297
1298static ssize_t
1299ftrace_notrace_write(struct file *file, const char __user *ubuf,
1300 size_t cnt, loff_t *ppos)
1301{
1302 return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
1303}
1304
1305static void
1306ftrace_set_regex(unsigned char *buf, int len, int reset, int enable)
1307{
1308 if (unlikely(ftrace_disabled))
1309 return;
1310
1311 mutex_lock(&ftrace_regex_lock);
1312 if (reset)
1313 ftrace_filter_reset(enable);
1314 if (buf)
1315 ftrace_match(buf, len, enable);
1316 mutex_unlock(&ftrace_regex_lock);
1317}
1318
1090/** 1319/**
1091 * ftrace_set_filter - set a function to filter on in ftrace 1320 * ftrace_set_filter - set a function to filter on in ftrace
1092 * @buf - the string that holds the function filter text. 1321 * @buf - the string that holds the function filter text.
@@ -1098,24 +1327,31 @@ ftrace_filter_write(struct file *file, const char __user *ubuf,
1098 */ 1327 */
1099void ftrace_set_filter(unsigned char *buf, int len, int reset) 1328void ftrace_set_filter(unsigned char *buf, int len, int reset)
1100{ 1329{
1101 if (unlikely(ftrace_disabled)) 1330 ftrace_set_regex(buf, len, reset, 1);
1102 return; 1331}
1103 1332
1104 mutex_lock(&ftrace_filter_lock); 1333/**
1105 if (reset) 1334 * ftrace_set_notrace - set a function to not trace in ftrace
1106 ftrace_filter_reset(); 1335 * @buf - the string that holds the function notrace text.
1107 if (buf) 1336 * @len - the length of the string.
1108 ftrace_match(buf, len); 1337 * @reset - non zero to reset all filters before applying this filter.
1109 mutex_unlock(&ftrace_filter_lock); 1338 *
1339 * Notrace Filters denote which functions should not be enabled when tracing
1340 * is enabled. If @buf is NULL and reset is set, all functions will be enabled
1341 * for tracing.
1342 */
1343void ftrace_set_notrace(unsigned char *buf, int len, int reset)
1344{
1345 ftrace_set_regex(buf, len, reset, 0);
1110} 1346}
1111 1347
1112static int 1348static int
1113ftrace_filter_release(struct inode *inode, struct file *file) 1349ftrace_regex_release(struct inode *inode, struct file *file, int enable)
1114{ 1350{
1115 struct seq_file *m = (struct seq_file *)file->private_data; 1351 struct seq_file *m = (struct seq_file *)file->private_data;
1116 struct ftrace_iterator *iter; 1352 struct ftrace_iterator *iter;
1117 1353
1118 mutex_lock(&ftrace_filter_lock); 1354 mutex_lock(&ftrace_regex_lock);
1119 if (file->f_mode & FMODE_READ) { 1355 if (file->f_mode & FMODE_READ) {
1120 iter = m->private; 1356 iter = m->private;
1121 1357
@@ -1126,7 +1362,7 @@ ftrace_filter_release(struct inode *inode, struct file *file)
1126 if (iter->buffer_idx) { 1362 if (iter->buffer_idx) {
1127 iter->filtered++; 1363 iter->filtered++;
1128 iter->buffer[iter->buffer_idx] = 0; 1364 iter->buffer[iter->buffer_idx] = 0;
1129 ftrace_match(iter->buffer, iter->buffer_idx); 1365 ftrace_match(iter->buffer, iter->buffer_idx, enable);
1130 } 1366 }
1131 1367
1132 mutex_lock(&ftrace_sysctl_lock); 1368 mutex_lock(&ftrace_sysctl_lock);
@@ -1137,10 +1373,71 @@ ftrace_filter_release(struct inode *inode, struct file *file)
1137 mutex_unlock(&ftrace_sysctl_lock); 1373 mutex_unlock(&ftrace_sysctl_lock);
1138 1374
1139 kfree(iter); 1375 kfree(iter);
1140 mutex_unlock(&ftrace_filter_lock); 1376 mutex_unlock(&ftrace_regex_lock);
1141 return 0; 1377 return 0;
1142} 1378}
1143 1379
1380static int
1381ftrace_filter_release(struct inode *inode, struct file *file)
1382{
1383 return ftrace_regex_release(inode, file, 1);
1384}
1385
1386static int
1387ftrace_notrace_release(struct inode *inode, struct file *file)
1388{
1389 return ftrace_regex_release(inode, file, 0);
1390}
1391
1392static ssize_t
1393ftraced_read(struct file *filp, char __user *ubuf,
1394 size_t cnt, loff_t *ppos)
1395{
1396 /* don't worry about races */
1397 char *buf = ftraced_stop ? "disabled\n" : "enabled\n";
1398 int r = strlen(buf);
1399
1400 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
1401}
1402
1403static ssize_t
1404ftraced_write(struct file *filp, const char __user *ubuf,
1405 size_t cnt, loff_t *ppos)
1406{
1407 char buf[64];
1408 long val;
1409 int ret;
1410
1411 if (cnt >= sizeof(buf))
1412 return -EINVAL;
1413
1414 if (copy_from_user(&buf, ubuf, cnt))
1415 return -EFAULT;
1416
1417 if (strncmp(buf, "enable", 6) == 0)
1418 val = 1;
1419 else if (strncmp(buf, "disable", 7) == 0)
1420 val = 0;
1421 else {
1422 buf[cnt] = 0;
1423
1424 ret = strict_strtoul(buf, 10, &val);
1425 if (ret < 0)
1426 return ret;
1427
1428 val = !!val;
1429 }
1430
1431 if (val)
1432 ftrace_enable_daemon();
1433 else
1434 ftrace_disable_daemon();
1435
1436 filp->f_pos += cnt;
1437
1438 return cnt;
1439}
1440
1144static struct file_operations ftrace_avail_fops = { 1441static struct file_operations ftrace_avail_fops = {
1145 .open = ftrace_avail_open, 1442 .open = ftrace_avail_open,
1146 .read = seq_read, 1443 .read = seq_read,
@@ -1148,59 +1445,57 @@ static struct file_operations ftrace_avail_fops = {
1148 .release = ftrace_avail_release, 1445 .release = ftrace_avail_release,
1149}; 1446};
1150 1447
1448static struct file_operations ftrace_failures_fops = {
1449 .open = ftrace_failures_open,
1450 .read = seq_read,
1451 .llseek = seq_lseek,
1452 .release = ftrace_avail_release,
1453};
1454
1151static struct file_operations ftrace_filter_fops = { 1455static struct file_operations ftrace_filter_fops = {
1152 .open = ftrace_filter_open, 1456 .open = ftrace_filter_open,
1153 .read = ftrace_filter_read, 1457 .read = ftrace_regex_read,
1154 .write = ftrace_filter_write, 1458 .write = ftrace_filter_write,
1155 .llseek = ftrace_filter_lseek, 1459 .llseek = ftrace_regex_lseek,
1156 .release = ftrace_filter_release, 1460 .release = ftrace_filter_release,
1157}; 1461};
1158 1462
1463static struct file_operations ftrace_notrace_fops = {
1464 .open = ftrace_notrace_open,
1465 .read = ftrace_regex_read,
1466 .write = ftrace_notrace_write,
1467 .llseek = ftrace_regex_lseek,
1468 .release = ftrace_notrace_release,
1469};
1470
1471static struct file_operations ftraced_fops = {
1472 .open = tracing_open_generic,
1473 .read = ftraced_read,
1474 .write = ftraced_write,
1475};
1476
1159/** 1477/**
1160 * ftrace_force_update - force an update to all recording ftrace functions 1478 * ftrace_force_update - force an update to all recording ftrace functions
1161 *
1162 * The ftrace dynamic update daemon only wakes up once a second.
1163 * There may be cases where an update needs to be done immediately
1164 * for tests or internal kernel tracing to begin. This function
1165 * wakes the daemon to do an update and will not return until the
1166 * update is complete.
1167 */ 1479 */
1168int ftrace_force_update(void) 1480int ftrace_force_update(void)
1169{ 1481{
1170 unsigned long last_counter;
1171 DECLARE_WAITQUEUE(wait, current);
1172 int ret = 0; 1482 int ret = 0;
1173 1483
1174 if (unlikely(ftrace_disabled)) 1484 if (unlikely(ftrace_disabled))
1175 return -ENODEV; 1485 return -ENODEV;
1176 1486
1487 mutex_lock(&ftrace_sysctl_lock);
1177 mutex_lock(&ftraced_lock); 1488 mutex_lock(&ftraced_lock);
1178 last_counter = ftraced_iteration_counter;
1179 1489
1180 set_current_state(TASK_INTERRUPTIBLE); 1490 /*
1181 add_wait_queue(&ftraced_waiters, &wait); 1491 * If ftraced_trigger is not set, then there is nothing
1182 1492 * to update.
1183 if (unlikely(!ftraced_task)) { 1493 */
1184 ret = -ENODEV; 1494 if (ftraced_trigger && !ftrace_update_code())
1185 goto out; 1495 ret = -EBUSY;
1186 }
1187
1188 do {
1189 mutex_unlock(&ftraced_lock);
1190 wake_up_process(ftraced_task);
1191 schedule();
1192 mutex_lock(&ftraced_lock);
1193 if (signal_pending(current)) {
1194 ret = -EINTR;
1195 break;
1196 }
1197 set_current_state(TASK_INTERRUPTIBLE);
1198 } while (last_counter == ftraced_iteration_counter);
1199 1496
1200 out:
1201 mutex_unlock(&ftraced_lock); 1497 mutex_unlock(&ftraced_lock);
1202 remove_wait_queue(&ftraced_waiters, &wait); 1498 mutex_unlock(&ftrace_sysctl_lock);
1203 set_current_state(TASK_RUNNING);
1204 1499
1205 return ret; 1500 return ret;
1206} 1501}
@@ -1234,11 +1529,28 @@ static __init int ftrace_init_debugfs(void)
1234 pr_warning("Could not create debugfs " 1529 pr_warning("Could not create debugfs "
1235 "'available_filter_functions' entry\n"); 1530 "'available_filter_functions' entry\n");
1236 1531
1532 entry = debugfs_create_file("failures", 0444,
1533 d_tracer, NULL, &ftrace_failures_fops);
1534 if (!entry)
1535 pr_warning("Could not create debugfs 'failures' entry\n");
1536
1237 entry = debugfs_create_file("set_ftrace_filter", 0644, d_tracer, 1537 entry = debugfs_create_file("set_ftrace_filter", 0644, d_tracer,
1238 NULL, &ftrace_filter_fops); 1538 NULL, &ftrace_filter_fops);
1239 if (!entry) 1539 if (!entry)
1240 pr_warning("Could not create debugfs " 1540 pr_warning("Could not create debugfs "
1241 "'set_ftrace_filter' entry\n"); 1541 "'set_ftrace_filter' entry\n");
1542
1543 entry = debugfs_create_file("set_ftrace_notrace", 0644, d_tracer,
1544 NULL, &ftrace_notrace_fops);
1545 if (!entry)
1546 pr_warning("Could not create debugfs "
1547 "'set_ftrace_notrace' entry\n");
1548
1549 entry = debugfs_create_file("ftraced_enabled", 0644, d_tracer,
1550 NULL, &ftraced_fops);
1551 if (!entry)
1552 pr_warning("Could not create debugfs "
1553 "'ftraced_enabled' entry\n");
1242 return 0; 1554 return 0;
1243} 1555}
1244 1556
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index 95b7c48a9a1d..e46de641ea44 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -27,6 +27,7 @@
27#include <linux/poll.h> 27#include <linux/poll.h>
28#include <linux/gfp.h> 28#include <linux/gfp.h>
29#include <linux/fs.h> 29#include <linux/fs.h>
30#include <linux/kprobes.h>
30#include <linux/writeback.h> 31#include <linux/writeback.h>
31 32
32#include <linux/stacktrace.h> 33#include <linux/stacktrace.h>
@@ -42,11 +43,6 @@ static cpumask_t __read_mostly tracing_buffer_mask;
42#define for_each_tracing_cpu(cpu) \ 43#define for_each_tracing_cpu(cpu) \
43 for_each_cpu_mask(cpu, tracing_buffer_mask) 44 for_each_cpu_mask(cpu, tracing_buffer_mask)
44 45
45/* dummy trace to disable tracing */
46static struct tracer no_tracer __read_mostly = {
47 .name = "none",
48};
49
50static int trace_alloc_page(void); 46static int trace_alloc_page(void);
51static int trace_free_page(void); 47static int trace_free_page(void);
52 48
@@ -134,6 +130,23 @@ static DECLARE_WAIT_QUEUE_HEAD(trace_wait);
134/* trace_flags holds iter_ctrl options */ 130/* trace_flags holds iter_ctrl options */
135unsigned long trace_flags = TRACE_ITER_PRINT_PARENT; 131unsigned long trace_flags = TRACE_ITER_PRINT_PARENT;
136 132
133static notrace void no_trace_init(struct trace_array *tr)
134{
135 int cpu;
136
137 if(tr->ctrl)
138 for_each_online_cpu(cpu)
139 tracing_reset(tr->data[cpu]);
140 tracer_enabled = 0;
141}
142
143/* dummy trace to disable tracing */
144static struct tracer no_tracer __read_mostly = {
145 .name = "none",
146 .init = no_trace_init
147};
148
149
137/** 150/**
138 * trace_wake_up - wake up tasks waiting for trace input 151 * trace_wake_up - wake up tasks waiting for trace input
139 * 152 *
@@ -249,24 +262,32 @@ __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
249 tracing_record_cmdline(current); 262 tracing_record_cmdline(current);
250} 263}
251 264
265#define CHECK_COND(cond) \
266 if (unlikely(cond)) { \
267 tracing_disabled = 1; \
268 WARN_ON(1); \
269 return -1; \
270 }
271
252/** 272/**
253 * check_pages - integrity check of trace buffers 273 * check_pages - integrity check of trace buffers
254 * 274 *
255 * As a safty measure we check to make sure the data pages have not 275 * As a safty measure we check to make sure the data pages have not
256 * been corrupted. TODO: configure to disable this because it adds 276 * been corrupted.
257 * a bit of overhead.
258 */ 277 */
259void check_pages(struct trace_array_cpu *data) 278int check_pages(struct trace_array_cpu *data)
260{ 279{
261 struct page *page, *tmp; 280 struct page *page, *tmp;
262 281
263 BUG_ON(data->trace_pages.next->prev != &data->trace_pages); 282 CHECK_COND(data->trace_pages.next->prev != &data->trace_pages);
264 BUG_ON(data->trace_pages.prev->next != &data->trace_pages); 283 CHECK_COND(data->trace_pages.prev->next != &data->trace_pages);
265 284
266 list_for_each_entry_safe(page, tmp, &data->trace_pages, lru) { 285 list_for_each_entry_safe(page, tmp, &data->trace_pages, lru) {
267 BUG_ON(page->lru.next->prev != &page->lru); 286 CHECK_COND(page->lru.next->prev != &page->lru);
268 BUG_ON(page->lru.prev->next != &page->lru); 287 CHECK_COND(page->lru.prev->next != &page->lru);
269 } 288 }
289
290 return 0;
270} 291}
271 292
272/** 293/**
@@ -280,7 +301,6 @@ void *head_page(struct trace_array_cpu *data)
280{ 301{
281 struct page *page; 302 struct page *page;
282 303
283 check_pages(data);
284 if (list_empty(&data->trace_pages)) 304 if (list_empty(&data->trace_pages))
285 return NULL; 305 return NULL;
286 306
@@ -645,9 +665,6 @@ static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
645static int cmdline_idx; 665static int cmdline_idx;
646static DEFINE_SPINLOCK(trace_cmdline_lock); 666static DEFINE_SPINLOCK(trace_cmdline_lock);
647 667
648/* trace in all context switches */
649atomic_t trace_record_cmdline_enabled __read_mostly;
650
651/* temporary disable recording */ 668/* temporary disable recording */
652atomic_t trace_record_cmdline_disabled __read_mostly; 669atomic_t trace_record_cmdline_disabled __read_mostly;
653 670
@@ -831,6 +848,48 @@ ftrace(struct trace_array *tr, struct trace_array_cpu *data,
831 trace_function(tr, data, ip, parent_ip, flags); 848 trace_function(tr, data, ip, parent_ip, flags);
832} 849}
833 850
851#ifdef CONFIG_MMIOTRACE
852void __trace_mmiotrace_rw(struct trace_array *tr, struct trace_array_cpu *data,
853 struct mmiotrace_rw *rw)
854{
855 struct trace_entry *entry;
856 unsigned long irq_flags;
857
858 raw_local_irq_save(irq_flags);
859 __raw_spin_lock(&data->lock);
860
861 entry = tracing_get_trace_entry(tr, data);
862 tracing_generic_entry_update(entry, 0);
863 entry->type = TRACE_MMIO_RW;
864 entry->mmiorw = *rw;
865
866 __raw_spin_unlock(&data->lock);
867 raw_local_irq_restore(irq_flags);
868
869 trace_wake_up();
870}
871
872void __trace_mmiotrace_map(struct trace_array *tr, struct trace_array_cpu *data,
873 struct mmiotrace_map *map)
874{
875 struct trace_entry *entry;
876 unsigned long irq_flags;
877
878 raw_local_irq_save(irq_flags);
879 __raw_spin_lock(&data->lock);
880
881 entry = tracing_get_trace_entry(tr, data);
882 tracing_generic_entry_update(entry, 0);
883 entry->type = TRACE_MMIO_MAP;
884 entry->mmiomap = *map;
885
886 __raw_spin_unlock(&data->lock);
887 raw_local_irq_restore(irq_flags);
888
889 trace_wake_up();
890}
891#endif
892
834void __trace_stack(struct trace_array *tr, 893void __trace_stack(struct trace_array *tr,
835 struct trace_array_cpu *data, 894 struct trace_array_cpu *data,
836 unsigned long flags, 895 unsigned long flags,
@@ -934,6 +993,30 @@ tracing_sched_wakeup_trace(struct trace_array *tr,
934 trace_wake_up(); 993 trace_wake_up();
935} 994}
936 995
996void
997ftrace_special(unsigned long arg1, unsigned long arg2, unsigned long arg3)
998{
999 struct trace_array *tr = &global_trace;
1000 struct trace_array_cpu *data;
1001 unsigned long flags;
1002 long disabled;
1003 int cpu;
1004
1005 if (tracing_disabled || current_trace == &no_tracer || !tr->ctrl)
1006 return;
1007
1008 local_irq_save(flags);
1009 cpu = raw_smp_processor_id();
1010 data = tr->data[cpu];
1011 disabled = atomic_inc_return(&data->disabled);
1012
1013 if (likely(disabled == 1))
1014 __trace_special(tr, data, arg1, arg2, arg3);
1015
1016 atomic_dec(&data->disabled);
1017 local_irq_restore(flags);
1018}
1019
937#ifdef CONFIG_FTRACE 1020#ifdef CONFIG_FTRACE
938static void 1021static void
939function_trace_call(unsigned long ip, unsigned long parent_ip) 1022function_trace_call(unsigned long ip, unsigned long parent_ip)
@@ -947,6 +1030,9 @@ function_trace_call(unsigned long ip, unsigned long parent_ip)
947 if (unlikely(!tracer_enabled)) 1030 if (unlikely(!tracer_enabled))
948 return; 1031 return;
949 1032
1033 if (skip_trace(ip))
1034 return;
1035
950 local_irq_save(flags); 1036 local_irq_save(flags);
951 cpu = raw_smp_processor_id(); 1037 cpu = raw_smp_processor_id();
952 data = tr->data[cpu]; 1038 data = tr->data[cpu];
@@ -1171,6 +1257,20 @@ static void s_stop(struct seq_file *m, void *p)
1171 mutex_unlock(&trace_types_lock); 1257 mutex_unlock(&trace_types_lock);
1172} 1258}
1173 1259
1260#define KRETPROBE_MSG "[unknown/kretprobe'd]"
1261
1262#ifdef CONFIG_KRETPROBES
1263static inline int kretprobed(unsigned long addr)
1264{
1265 return addr == (unsigned long)kretprobe_trampoline;
1266}
1267#else
1268static inline int kretprobed(unsigned long addr)
1269{
1270 return 0;
1271}
1272#endif /* CONFIG_KRETPROBES */
1273
1174static int 1274static int
1175seq_print_sym_short(struct trace_seq *s, const char *fmt, unsigned long address) 1275seq_print_sym_short(struct trace_seq *s, const char *fmt, unsigned long address)
1176{ 1276{
@@ -1406,7 +1506,10 @@ print_lat_fmt(struct trace_iterator *iter, unsigned int trace_idx, int cpu)
1406 case TRACE_FN: 1506 case TRACE_FN:
1407 seq_print_ip_sym(s, entry->fn.ip, sym_flags); 1507 seq_print_ip_sym(s, entry->fn.ip, sym_flags);
1408 trace_seq_puts(s, " ("); 1508 trace_seq_puts(s, " (");
1409 seq_print_ip_sym(s, entry->fn.parent_ip, sym_flags); 1509 if (kretprobed(entry->fn.parent_ip))
1510 trace_seq_puts(s, KRETPROBE_MSG);
1511 else
1512 seq_print_ip_sym(s, entry->fn.parent_ip, sym_flags);
1410 trace_seq_puts(s, ")\n"); 1513 trace_seq_puts(s, ")\n");
1411 break; 1514 break;
1412 case TRACE_CTX: 1515 case TRACE_CTX:
@@ -1486,8 +1589,11 @@ static int print_trace_fmt(struct trace_iterator *iter)
1486 ret = trace_seq_printf(s, " <-"); 1589 ret = trace_seq_printf(s, " <-");
1487 if (!ret) 1590 if (!ret)
1488 return 0; 1591 return 0;
1489 ret = seq_print_ip_sym(s, entry->fn.parent_ip, 1592 if (kretprobed(entry->fn.parent_ip))
1490 sym_flags); 1593 ret = trace_seq_puts(s, KRETPROBE_MSG);
1594 else
1595 ret = seq_print_ip_sym(s, entry->fn.parent_ip,
1596 sym_flags);
1491 if (!ret) 1597 if (!ret)
1492 return 0; 1598 return 0;
1493 } 1599 }
@@ -2566,7 +2672,7 @@ tracing_entries_write(struct file *filp, const char __user *ubuf,
2566{ 2672{
2567 unsigned long val; 2673 unsigned long val;
2568 char buf[64]; 2674 char buf[64];
2569 int ret; 2675 int i, ret;
2570 2676
2571 if (cnt >= sizeof(buf)) 2677 if (cnt >= sizeof(buf))
2572 return -EINVAL; 2678 return -EINVAL;
@@ -2635,8 +2741,15 @@ tracing_entries_write(struct file *filp, const char __user *ubuf,
2635 trace_free_page(); 2741 trace_free_page();
2636 } 2742 }
2637 2743
2744 /* check integrity */
2745 for_each_tracing_cpu(i)
2746 check_pages(global_trace.data[i]);
2747
2638 filp->f_pos += cnt; 2748 filp->f_pos += cnt;
2639 2749
2750 /* If check pages failed, return ENOMEM */
2751 if (tracing_disabled)
2752 cnt = -ENOMEM;
2640 out: 2753 out:
2641 max_tr.entries = global_trace.entries; 2754 max_tr.entries = global_trace.entries;
2642 mutex_unlock(&trace_types_lock); 2755 mutex_unlock(&trace_types_lock);
@@ -2930,8 +3043,6 @@ __init static int tracer_alloc_buffers(void)
2930 int ret = -ENOMEM; 3043 int ret = -ENOMEM;
2931 int i; 3044 int i;
2932 3045
2933 global_trace.ctrl = tracer_enabled;
2934
2935 /* TODO: make the number of buffers hot pluggable with CPUS */ 3046 /* TODO: make the number of buffers hot pluggable with CPUS */
2936 tracing_nr_buffers = num_possible_cpus(); 3047 tracing_nr_buffers = num_possible_cpus();
2937 tracing_buffer_mask = cpu_possible_map; 3048 tracing_buffer_mask = cpu_possible_map;
@@ -2988,9 +3099,8 @@ __init static int tracer_alloc_buffers(void)
2988 } 3099 }
2989 max_tr.entries = global_trace.entries; 3100 max_tr.entries = global_trace.entries;
2990 3101
2991 pr_info("tracer: %d pages allocated for %ld", 3102 pr_info("tracer: %d pages allocated for %ld entries of %ld bytes\n",
2992 pages, trace_nr_entries); 3103 pages, trace_nr_entries, (long)TRACE_ENTRY_SIZE);
2993 pr_info(" entries of %ld bytes\n", (long)TRACE_ENTRY_SIZE);
2994 pr_info(" actual entries %ld\n", global_trace.entries); 3104 pr_info(" actual entries %ld\n", global_trace.entries);
2995 3105
2996 tracer_init_debugfs(); 3106 tracer_init_debugfs();
@@ -3001,6 +3111,7 @@ __init static int tracer_alloc_buffers(void)
3001 current_trace = &no_tracer; 3111 current_trace = &no_tracer;
3002 3112
3003 /* All seems OK, enable tracing */ 3113 /* All seems OK, enable tracing */
3114 global_trace.ctrl = tracer_enabled;
3004 tracing_disabled = 0; 3115 tracing_disabled = 0;
3005 3116
3006 return 0; 3117 return 0;
diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h
index b7f85d9c80d7..8cb215b239d5 100644
--- a/kernel/trace/trace.h
+++ b/kernel/trace/trace.h
@@ -5,6 +5,7 @@
5#include <asm/atomic.h> 5#include <asm/atomic.h>
6#include <linux/sched.h> 6#include <linux/sched.h>
7#include <linux/clocksource.h> 7#include <linux/clocksource.h>
8#include <linux/mmiotrace.h>
8 9
9enum trace_type { 10enum trace_type {
10 __TRACE_FIRST_TYPE = 0, 11 __TRACE_FIRST_TYPE = 0,
@@ -14,6 +15,8 @@ enum trace_type {
14 TRACE_WAKE, 15 TRACE_WAKE,
15 TRACE_STACK, 16 TRACE_STACK,
16 TRACE_SPECIAL, 17 TRACE_SPECIAL,
18 TRACE_MMIO_RW,
19 TRACE_MMIO_MAP,
17 20
18 __TRACE_LAST_TYPE 21 __TRACE_LAST_TYPE
19}; 22};
@@ -75,6 +78,8 @@ struct trace_entry {
75 struct ctx_switch_entry ctx; 78 struct ctx_switch_entry ctx;
76 struct special_entry special; 79 struct special_entry special;
77 struct stack_entry stack; 80 struct stack_entry stack;
81 struct mmiotrace_rw mmiorw;
82 struct mmiotrace_map mmiomap;
78 }; 83 };
79}; 84};
80 85
@@ -220,6 +225,8 @@ void trace_function(struct trace_array *tr,
220 225
221void tracing_start_function_trace(void); 226void tracing_start_function_trace(void);
222void tracing_stop_function_trace(void); 227void tracing_stop_function_trace(void);
228void tracing_start_cmdline_record(void);
229void tracing_stop_cmdline_record(void);
223int register_tracer(struct tracer *type); 230int register_tracer(struct tracer *type);
224void unregister_tracer(struct tracer *type); 231void unregister_tracer(struct tracer *type);
225 232
@@ -228,8 +235,6 @@ extern unsigned long nsecs_to_usecs(unsigned long nsecs);
228extern unsigned long tracing_max_latency; 235extern unsigned long tracing_max_latency;
229extern unsigned long tracing_thresh; 236extern unsigned long tracing_thresh;
230 237
231extern atomic_t trace_record_cmdline_enabled;
232
233void update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu); 238void update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu);
234void update_max_tr_single(struct trace_array *tr, 239void update_max_tr_single(struct trace_array *tr,
235 struct task_struct *tsk, int cpu); 240 struct task_struct *tsk, int cpu);
@@ -257,6 +262,15 @@ extern unsigned long ftrace_update_tot_cnt;
257extern int DYN_FTRACE_TEST_NAME(void); 262extern int DYN_FTRACE_TEST_NAME(void);
258#endif 263#endif
259 264
265#ifdef CONFIG_MMIOTRACE
266extern void __trace_mmiotrace_rw(struct trace_array *tr,
267 struct trace_array_cpu *data,
268 struct mmiotrace_rw *rw);
269extern void __trace_mmiotrace_map(struct trace_array *tr,
270 struct trace_array_cpu *data,
271 struct mmiotrace_map *map);
272#endif
273
260#ifdef CONFIG_FTRACE_STARTUP_TEST 274#ifdef CONFIG_FTRACE_STARTUP_TEST
261#ifdef CONFIG_FTRACE 275#ifdef CONFIG_FTRACE
262extern int trace_selftest_startup_function(struct tracer *trace, 276extern int trace_selftest_startup_function(struct tracer *trace,
diff --git a/kernel/trace/trace_functions.c b/kernel/trace/trace_functions.c
index 0a084656d7cf..7ee7dcd76b7d 100644
--- a/kernel/trace/trace_functions.c
+++ b/kernel/trace/trace_functions.c
@@ -29,14 +29,14 @@ static void function_reset(struct trace_array *tr)
29static void start_function_trace(struct trace_array *tr) 29static void start_function_trace(struct trace_array *tr)
30{ 30{
31 function_reset(tr); 31 function_reset(tr);
32 atomic_inc(&trace_record_cmdline_enabled); 32 tracing_start_cmdline_record();
33 tracing_start_function_trace(); 33 tracing_start_function_trace();
34} 34}
35 35
36static void stop_function_trace(struct trace_array *tr) 36static void stop_function_trace(struct trace_array *tr)
37{ 37{
38 tracing_stop_function_trace(); 38 tracing_stop_function_trace();
39 atomic_dec(&trace_record_cmdline_enabled); 39 tracing_stop_cmdline_record();
40} 40}
41 41
42static void function_trace_init(struct trace_array *tr) 42static void function_trace_init(struct trace_array *tr)
diff --git a/kernel/trace/trace_irqsoff.c b/kernel/trace/trace_irqsoff.c
index 761f3ec66c50..421d6fe3650e 100644
--- a/kernel/trace/trace_irqsoff.c
+++ b/kernel/trace/trace_irqsoff.c
@@ -165,22 +165,6 @@ check_critical_timing(struct trace_array *tr,
165 165
166 update_max_tr_single(tr, current, cpu); 166 update_max_tr_single(tr, current, cpu);
167 167
168 if (!runqueue_is_locked()) {
169 if (tracing_thresh) {
170 printk(KERN_INFO "(%16s-%-5d|#%d): %lu us critical"
171 " section violates %lu us threshold.\n",
172 current->comm, current->pid,
173 raw_smp_processor_id(),
174 latency, nsecs_to_usecs(tracing_thresh));
175 } else {
176 printk(KERN_INFO "(%16s-%-5d|#%d): new %lu us"
177 " maximum-latency critical section.\n",
178 current->comm, current->pid,
179 raw_smp_processor_id(),
180 latency);
181 }
182 }
183
184 max_sequence++; 168 max_sequence++;
185 169
186out_unlock: 170out_unlock:
diff --git a/kernel/trace/trace_mmiotrace.c b/kernel/trace/trace_mmiotrace.c
new file mode 100644
index 000000000000..b13dc19dcbb4
--- /dev/null
+++ b/kernel/trace/trace_mmiotrace.c
@@ -0,0 +1,295 @@
1/*
2 * Memory mapped I/O tracing
3 *
4 * Copyright (C) 2008 Pekka Paalanen <pq@iki.fi>
5 */
6
7#define DEBUG 1
8
9#include <linux/kernel.h>
10#include <linux/mmiotrace.h>
11#include <linux/pci.h>
12
13#include "trace.h"
14
15struct header_iter {
16 struct pci_dev *dev;
17};
18
19static struct trace_array *mmio_trace_array;
20static bool overrun_detected;
21
22static void mmio_reset_data(struct trace_array *tr)
23{
24 int cpu;
25
26 overrun_detected = false;
27 tr->time_start = ftrace_now(tr->cpu);
28
29 for_each_online_cpu(cpu)
30 tracing_reset(tr->data[cpu]);
31}
32
33static void mmio_trace_init(struct trace_array *tr)
34{
35 pr_debug("in %s\n", __func__);
36 mmio_trace_array = tr;
37 if (tr->ctrl) {
38 mmio_reset_data(tr);
39 enable_mmiotrace();
40 }
41}
42
43static void mmio_trace_reset(struct trace_array *tr)
44{
45 pr_debug("in %s\n", __func__);
46 if (tr->ctrl)
47 disable_mmiotrace();
48 mmio_reset_data(tr);
49 mmio_trace_array = NULL;
50}
51
52static void mmio_trace_ctrl_update(struct trace_array *tr)
53{
54 pr_debug("in %s\n", __func__);
55 if (tr->ctrl) {
56 mmio_reset_data(tr);
57 enable_mmiotrace();
58 } else {
59 disable_mmiotrace();
60 }
61}
62
63static int mmio_print_pcidev(struct trace_seq *s, const struct pci_dev *dev)
64{
65 int ret = 0;
66 int i;
67 resource_size_t start, end;
68 const struct pci_driver *drv = pci_dev_driver(dev);
69
70 /* XXX: incomplete checks for trace_seq_printf() return value */
71 ret += trace_seq_printf(s, "PCIDEV %02x%02x %04x%04x %x",
72 dev->bus->number, dev->devfn,
73 dev->vendor, dev->device, dev->irq);
74 /*
75 * XXX: is pci_resource_to_user() appropriate, since we are
76 * supposed to interpret the __ioremap() phys_addr argument based on
77 * these printed values?
78 */
79 for (i = 0; i < 7; i++) {
80 pci_resource_to_user(dev, i, &dev->resource[i], &start, &end);
81 ret += trace_seq_printf(s, " %llx",
82 (unsigned long long)(start |
83 (dev->resource[i].flags & PCI_REGION_FLAG_MASK)));
84 }
85 for (i = 0; i < 7; i++) {
86 pci_resource_to_user(dev, i, &dev->resource[i], &start, &end);
87 ret += trace_seq_printf(s, " %llx",
88 dev->resource[i].start < dev->resource[i].end ?
89 (unsigned long long)(end - start) + 1 : 0);
90 }
91 if (drv)
92 ret += trace_seq_printf(s, " %s\n", drv->name);
93 else
94 ret += trace_seq_printf(s, " \n");
95 return ret;
96}
97
98static void destroy_header_iter(struct header_iter *hiter)
99{
100 if (!hiter)
101 return;
102 pci_dev_put(hiter->dev);
103 kfree(hiter);
104}
105
106static void mmio_pipe_open(struct trace_iterator *iter)
107{
108 struct header_iter *hiter;
109 struct trace_seq *s = &iter->seq;
110
111 trace_seq_printf(s, "VERSION 20070824\n");
112
113 hiter = kzalloc(sizeof(*hiter), GFP_KERNEL);
114 if (!hiter)
115 return;
116
117 hiter->dev = pci_get_device(PCI_ANY_ID, PCI_ANY_ID, NULL);
118 iter->private = hiter;
119}
120
121/* XXX: This is not called when the pipe is closed! */
122static void mmio_close(struct trace_iterator *iter)
123{
124 struct header_iter *hiter = iter->private;
125 destroy_header_iter(hiter);
126 iter->private = NULL;
127}
128
129static unsigned long count_overruns(struct trace_iterator *iter)
130{
131 int cpu;
132 unsigned long cnt = 0;
133 for_each_online_cpu(cpu) {
134 cnt += iter->overrun[cpu];
135 iter->overrun[cpu] = 0;
136 }
137 return cnt;
138}
139
140static ssize_t mmio_read(struct trace_iterator *iter, struct file *filp,
141 char __user *ubuf, size_t cnt, loff_t *ppos)
142{
143 ssize_t ret;
144 struct header_iter *hiter = iter->private;
145 struct trace_seq *s = &iter->seq;
146 unsigned long n;
147
148 n = count_overruns(iter);
149 if (n) {
150 /* XXX: This is later than where events were lost. */
151 trace_seq_printf(s, "MARK 0.000000 Lost %lu events.\n", n);
152 if (!overrun_detected)
153 pr_warning("mmiotrace has lost events.\n");
154 overrun_detected = true;
155 goto print_out;
156 }
157
158 if (!hiter)
159 return 0;
160
161 mmio_print_pcidev(s, hiter->dev);
162 hiter->dev = pci_get_device(PCI_ANY_ID, PCI_ANY_ID, hiter->dev);
163
164 if (!hiter->dev) {
165 destroy_header_iter(hiter);
166 iter->private = NULL;
167 }
168
169print_out:
170 ret = trace_seq_to_user(s, ubuf, cnt);
171 return (ret == -EBUSY) ? 0 : ret;
172}
173
174static int mmio_print_rw(struct trace_iterator *iter)
175{
176 struct trace_entry *entry = iter->ent;
177 struct mmiotrace_rw *rw = &entry->mmiorw;
178 struct trace_seq *s = &iter->seq;
179 unsigned long long t = ns2usecs(entry->t);
180 unsigned long usec_rem = do_div(t, 1000000ULL);
181 unsigned secs = (unsigned long)t;
182 int ret = 1;
183
184 switch (entry->mmiorw.opcode) {
185 case MMIO_READ:
186 ret = trace_seq_printf(s,
187 "R %d %lu.%06lu %d 0x%llx 0x%lx 0x%lx %d\n",
188 rw->width, secs, usec_rem, rw->map_id,
189 (unsigned long long)rw->phys,
190 rw->value, rw->pc, 0);
191 break;
192 case MMIO_WRITE:
193 ret = trace_seq_printf(s,
194 "W %d %lu.%06lu %d 0x%llx 0x%lx 0x%lx %d\n",
195 rw->width, secs, usec_rem, rw->map_id,
196 (unsigned long long)rw->phys,
197 rw->value, rw->pc, 0);
198 break;
199 case MMIO_UNKNOWN_OP:
200 ret = trace_seq_printf(s,
201 "UNKNOWN %lu.%06lu %d 0x%llx %02x,%02x,%02x 0x%lx %d\n",
202 secs, usec_rem, rw->map_id,
203 (unsigned long long)rw->phys,
204 (rw->value >> 16) & 0xff, (rw->value >> 8) & 0xff,
205 (rw->value >> 0) & 0xff, rw->pc, 0);
206 break;
207 default:
208 ret = trace_seq_printf(s, "rw what?\n");
209 break;
210 }
211 if (ret)
212 return 1;
213 return 0;
214}
215
216static int mmio_print_map(struct trace_iterator *iter)
217{
218 struct trace_entry *entry = iter->ent;
219 struct mmiotrace_map *m = &entry->mmiomap;
220 struct trace_seq *s = &iter->seq;
221 unsigned long long t = ns2usecs(entry->t);
222 unsigned long usec_rem = do_div(t, 1000000ULL);
223 unsigned secs = (unsigned long)t;
224 int ret = 1;
225
226 switch (entry->mmiorw.opcode) {
227 case MMIO_PROBE:
228 ret = trace_seq_printf(s,
229 "MAP %lu.%06lu %d 0x%llx 0x%lx 0x%lx 0x%lx %d\n",
230 secs, usec_rem, m->map_id,
231 (unsigned long long)m->phys, m->virt, m->len,
232 0UL, 0);
233 break;
234 case MMIO_UNPROBE:
235 ret = trace_seq_printf(s,
236 "UNMAP %lu.%06lu %d 0x%lx %d\n",
237 secs, usec_rem, m->map_id, 0UL, 0);
238 break;
239 default:
240 ret = trace_seq_printf(s, "map what?\n");
241 break;
242 }
243 if (ret)
244 return 1;
245 return 0;
246}
247
248/* return 0 to abort printing without consuming current entry in pipe mode */
249static int mmio_print_line(struct trace_iterator *iter)
250{
251 switch (iter->ent->type) {
252 case TRACE_MMIO_RW:
253 return mmio_print_rw(iter);
254 case TRACE_MMIO_MAP:
255 return mmio_print_map(iter);
256 default:
257 return 1; /* ignore unknown entries */
258 }
259}
260
261static struct tracer mmio_tracer __read_mostly =
262{
263 .name = "mmiotrace",
264 .init = mmio_trace_init,
265 .reset = mmio_trace_reset,
266 .pipe_open = mmio_pipe_open,
267 .close = mmio_close,
268 .read = mmio_read,
269 .ctrl_update = mmio_trace_ctrl_update,
270 .print_line = mmio_print_line,
271};
272
273__init static int init_mmio_trace(void)
274{
275 return register_tracer(&mmio_tracer);
276}
277device_initcall(init_mmio_trace);
278
279void mmio_trace_rw(struct mmiotrace_rw *rw)
280{
281 struct trace_array *tr = mmio_trace_array;
282 struct trace_array_cpu *data = tr->data[smp_processor_id()];
283 __trace_mmiotrace_rw(tr, data, rw);
284}
285
286void mmio_trace_mapping(struct mmiotrace_map *map)
287{
288 struct trace_array *tr = mmio_trace_array;
289 struct trace_array_cpu *data;
290
291 preempt_disable();
292 data = tr->data[smp_processor_id()];
293 __trace_mmiotrace_map(tr, data, map);
294 preempt_enable();
295}
diff --git a/kernel/trace/trace_sched_switch.c b/kernel/trace/trace_sched_switch.c
index d25ffa5eaf2b..93a662009151 100644
--- a/kernel/trace/trace_sched_switch.c
+++ b/kernel/trace/trace_sched_switch.c
@@ -29,6 +29,9 @@ sched_switch_func(void *private, void *__rq, struct task_struct *prev,
29 long disabled; 29 long disabled;
30 int cpu; 30 int cpu;
31 31
32 tracing_record_cmdline(prev);
33 tracing_record_cmdline(next);
34
32 if (!tracer_enabled) 35 if (!tracer_enabled)
33 return; 36 return;
34 37
@@ -63,8 +66,6 @@ sched_switch_callback(void *probe_data, void *call_data,
63 prev = va_arg(*args, typeof(prev)); 66 prev = va_arg(*args, typeof(prev));
64 next = va_arg(*args, typeof(next)); 67 next = va_arg(*args, typeof(next));
65 68
66 tracing_record_cmdline(prev);
67
68 /* 69 /*
69 * If tracer_switch_func only points to the local 70 * If tracer_switch_func only points to the local
70 * switch func, it still needs the ptr passed to it. 71 * switch func, it still needs the ptr passed to it.
@@ -125,30 +126,6 @@ wake_up_callback(void *probe_data, void *call_data,
125 wakeup_func(probe_data, __rq, task, curr); 126 wakeup_func(probe_data, __rq, task, curr);
126} 127}
127 128
128void
129ftrace_special(unsigned long arg1, unsigned long arg2, unsigned long arg3)
130{
131 struct trace_array *tr = ctx_trace;
132 struct trace_array_cpu *data;
133 unsigned long flags;
134 long disabled;
135 int cpu;
136
137 if (!tracer_enabled)
138 return;
139
140 local_irq_save(flags);
141 cpu = raw_smp_processor_id();
142 data = tr->data[cpu];
143 disabled = atomic_inc_return(&data->disabled);
144
145 if (likely(disabled == 1))
146 __trace_special(tr, data, arg1, arg2, arg3);
147
148 atomic_dec(&data->disabled);
149 local_irq_restore(flags);
150}
151
152static void sched_switch_reset(struct trace_array *tr) 129static void sched_switch_reset(struct trace_array *tr)
153{ 130{
154 int cpu; 131 int cpu;
@@ -219,7 +196,7 @@ static void tracing_sched_unregister(void)
219 &ctx_trace); 196 &ctx_trace);
220} 197}
221 198
222void tracing_start_sched_switch(void) 199static void tracing_start_sched_switch(void)
223{ 200{
224 long ref; 201 long ref;
225 202
@@ -228,7 +205,7 @@ void tracing_start_sched_switch(void)
228 tracing_sched_register(); 205 tracing_sched_register();
229} 206}
230 207
231void tracing_stop_sched_switch(void) 208static void tracing_stop_sched_switch(void)
232{ 209{
233 long ref; 210 long ref;
234 211
@@ -237,18 +214,26 @@ void tracing_stop_sched_switch(void)
237 tracing_sched_unregister(); 214 tracing_sched_unregister();
238} 215}
239 216
217void tracing_start_cmdline_record(void)
218{
219 tracing_start_sched_switch();
220}
221
222void tracing_stop_cmdline_record(void)
223{
224 tracing_stop_sched_switch();
225}
226
240static void start_sched_trace(struct trace_array *tr) 227static void start_sched_trace(struct trace_array *tr)
241{ 228{
242 sched_switch_reset(tr); 229 sched_switch_reset(tr);
243 atomic_inc(&trace_record_cmdline_enabled);
244 tracer_enabled = 1; 230 tracer_enabled = 1;
245 tracing_start_sched_switch(); 231 tracing_start_cmdline_record();
246} 232}
247 233
248static void stop_sched_trace(struct trace_array *tr) 234static void stop_sched_trace(struct trace_array *tr)
249{ 235{
250 tracing_stop_sched_switch(); 236 tracing_stop_cmdline_record();
251 atomic_dec(&trace_record_cmdline_enabled);
252 tracer_enabled = 0; 237 tracer_enabled = 0;
253} 238}
254 239
diff --git a/kernel/trace/trace_sched_wakeup.c b/kernel/trace/trace_sched_wakeup.c
index 5d2fb48e47f8..bf7e91caef57 100644
--- a/kernel/trace/trace_sched_wakeup.c
+++ b/kernel/trace/trace_sched_wakeup.c
@@ -30,6 +30,69 @@ static DEFINE_SPINLOCK(wakeup_lock);
30 30
31static void __wakeup_reset(struct trace_array *tr); 31static void __wakeup_reset(struct trace_array *tr);
32 32
33#ifdef CONFIG_FTRACE
34/*
35 * irqsoff uses its own tracer function to keep the overhead down:
36 */
37static void
38wakeup_tracer_call(unsigned long ip, unsigned long parent_ip)
39{
40 struct trace_array *tr = wakeup_trace;
41 struct trace_array_cpu *data;
42 unsigned long flags;
43 long disabled;
44 int resched;
45 int cpu;
46
47 if (likely(!wakeup_task))
48 return;
49
50 resched = need_resched();
51 preempt_disable_notrace();
52
53 cpu = raw_smp_processor_id();
54 data = tr->data[cpu];
55 disabled = atomic_inc_return(&data->disabled);
56 if (unlikely(disabled != 1))
57 goto out;
58
59 spin_lock_irqsave(&wakeup_lock, flags);
60
61 if (unlikely(!wakeup_task))
62 goto unlock;
63
64 /*
65 * The task can't disappear because it needs to
66 * wake up first, and we have the wakeup_lock.
67 */
68 if (task_cpu(wakeup_task) != cpu)
69 goto unlock;
70
71 trace_function(tr, data, ip, parent_ip, flags);
72
73 unlock:
74 spin_unlock_irqrestore(&wakeup_lock, flags);
75
76 out:
77 atomic_dec(&data->disabled);
78
79 /*
80 * To prevent recursion from the scheduler, if the
81 * resched flag was set before we entered, then
82 * don't reschedule.
83 */
84 if (resched)
85 preempt_enable_no_resched_notrace();
86 else
87 preempt_enable_notrace();
88}
89
90static struct ftrace_ops trace_ops __read_mostly =
91{
92 .func = wakeup_tracer_call,
93};
94#endif /* CONFIG_FTRACE */
95
33/* 96/*
34 * Should this new latency be reported/recorded? 97 * Should this new latency be reported/recorded?
35 */ 98 */
@@ -73,7 +136,7 @@ wakeup_sched_switch(void *private, void *rq, struct task_struct *prev,
73 if (next != wakeup_task) 136 if (next != wakeup_task)
74 return; 137 return;
75 138
76 /* The task we are waitng for is waking up */ 139 /* The task we are waiting for is waking up */
77 data = tr->data[wakeup_cpu]; 140 data = tr->data[wakeup_cpu];
78 141
79 /* disable local data, not wakeup_cpu data */ 142 /* disable local data, not wakeup_cpu data */
@@ -290,6 +353,7 @@ static void start_wakeup_tracer(struct trace_array *tr)
290 smp_wmb(); 353 smp_wmb();
291 354
292 tracer_enabled = 1; 355 tracer_enabled = 1;
356 register_ftrace_function(&trace_ops);
293 357
294 return; 358 return;
295fail_deprobe_wake_new: 359fail_deprobe_wake_new:
@@ -305,6 +369,7 @@ fail_deprobe:
305static void stop_wakeup_tracer(struct trace_array *tr) 369static void stop_wakeup_tracer(struct trace_array *tr)
306{ 370{
307 tracer_enabled = 0; 371 tracer_enabled = 0;
372 unregister_ftrace_function(&trace_ops);
308 marker_probe_unregister("kernel_sched_schedule", 373 marker_probe_unregister("kernel_sched_schedule",
309 sched_switch_callback, 374 sched_switch_callback,
310 &wakeup_trace); 375 &wakeup_trace);
diff --git a/kernel/trace/trace_selftest.c b/kernel/trace/trace_selftest.c
index 5588ecc40985..0911b7e073bf 100644
--- a/kernel/trace/trace_selftest.c
+++ b/kernel/trace/trace_selftest.c
@@ -28,6 +28,7 @@ trace_test_buffer_cpu(struct trace_array *tr, struct trace_array_cpu *data)
28 page = list_entry(data->trace_pages.next, struct page, lru); 28 page = list_entry(data->trace_pages.next, struct page, lru);
29 entries = page_address(page); 29 entries = page_address(page);
30 30
31 check_pages(data);
31 if (head_page(data) != entries) 32 if (head_page(data) != entries)
32 goto failed; 33 goto failed;
33 34
generated by cgit v1.2.2 (git 2.25.0) at 2025-03-01 00:20:11 -0500