aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/trace/ftrace.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/trace/ftrace.c')
-rw-r--r--kernel/trace/ftrace.c642
1 files changed, 113 insertions, 529 deletions
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
index 4dda4f60a2a9..e60205722d0c 100644
--- a/kernel/trace/ftrace.c
+++ b/kernel/trace/ftrace.c
@@ -25,13 +25,24 @@
25#include <linux/ftrace.h> 25#include <linux/ftrace.h>
26#include <linux/sysctl.h> 26#include <linux/sysctl.h>
27#include <linux/ctype.h> 27#include <linux/ctype.h>
28#include <linux/hash.h>
29#include <linux/list.h> 28#include <linux/list.h>
30 29
31#include <asm/ftrace.h> 30#include <asm/ftrace.h>
32 31
33#include "trace.h" 32#include "trace.h"
34 33
34#define FTRACE_WARN_ON(cond) \
35 do { \
36 if (WARN_ON(cond)) \
37 ftrace_kill(); \
38 } while (0)
39
40#define FTRACE_WARN_ON_ONCE(cond) \
41 do { \
42 if (WARN_ON_ONCE(cond)) \
43 ftrace_kill(); \
44 } while (0)
45
35/* ftrace_enabled is a method to turn ftrace on or off */ 46/* ftrace_enabled is a method to turn ftrace on or off */
36int ftrace_enabled __read_mostly; 47int ftrace_enabled __read_mostly;
37static int last_ftrace_enabled; 48static int last_ftrace_enabled;
@@ -153,21 +164,8 @@ static int __unregister_ftrace_function(struct ftrace_ops *ops)
153} 164}
154 165
155#ifdef CONFIG_DYNAMIC_FTRACE 166#ifdef CONFIG_DYNAMIC_FTRACE
156
157#ifndef CONFIG_FTRACE_MCOUNT_RECORD 167#ifndef CONFIG_FTRACE_MCOUNT_RECORD
158/* 168# error Dynamic ftrace depends on MCOUNT_RECORD
159 * The hash lock is only needed when the recording of the mcount
160 * callers are dynamic. That is, by the caller themselves and
161 * not recorded via the compilation.
162 */
163static DEFINE_SPINLOCK(ftrace_hash_lock);
164#define ftrace_hash_lock(flags) spin_lock_irqsave(&ftrace_hash_lock, flags)
165#define ftrace_hash_unlock(flags) \
166 spin_unlock_irqrestore(&ftrace_hash_lock, flags)
167#else
168/* This is protected via the ftrace_lock with MCOUNT_RECORD. */
169#define ftrace_hash_lock(flags) do { (void)(flags); } while (0)
170#define ftrace_hash_unlock(flags) do { } while(0)
171#endif 169#endif
172 170
173/* 171/*
@@ -178,8 +176,6 @@ static DEFINE_SPINLOCK(ftrace_hash_lock);
178 */ 176 */
179static unsigned long mcount_addr = MCOUNT_ADDR; 177static unsigned long mcount_addr = MCOUNT_ADDR;
180 178
181static struct task_struct *ftraced_task;
182
183enum { 179enum {
184 FTRACE_ENABLE_CALLS = (1 << 0), 180 FTRACE_ENABLE_CALLS = (1 << 0),
185 FTRACE_DISABLE_CALLS = (1 << 1), 181 FTRACE_DISABLE_CALLS = (1 << 1),
@@ -189,14 +185,9 @@ enum {
189}; 185};
190 186
191static int ftrace_filtered; 187static int ftrace_filtered;
192static int tracing_on;
193static int frozen_record_count;
194 188
195static struct hlist_head ftrace_hash[FTRACE_HASHSIZE]; 189static LIST_HEAD(ftrace_new_addrs);
196 190
197static DEFINE_PER_CPU(int, ftrace_shutdown_disable_cpu);
198
199static DEFINE_MUTEX(ftraced_lock);
200static DEFINE_MUTEX(ftrace_regex_lock); 191static DEFINE_MUTEX(ftrace_regex_lock);
201 192
202struct ftrace_page { 193struct ftrace_page {
@@ -214,16 +205,13 @@ struct ftrace_page {
214static struct ftrace_page *ftrace_pages_start; 205static struct ftrace_page *ftrace_pages_start;
215static struct ftrace_page *ftrace_pages; 206static struct ftrace_page *ftrace_pages;
216 207
217static int ftraced_trigger;
218static int ftraced_suspend;
219static int ftraced_stop;
220
221static int ftrace_record_suspend;
222
223static struct dyn_ftrace *ftrace_free_records; 208static struct dyn_ftrace *ftrace_free_records;
224 209
225 210
226#ifdef CONFIG_KPROBES 211#ifdef CONFIG_KPROBES
212
213static int frozen_record_count;
214
227static inline void freeze_record(struct dyn_ftrace *rec) 215static inline void freeze_record(struct dyn_ftrace *rec)
228{ 216{
229 if (!(rec->flags & FTRACE_FL_FROZEN)) { 217 if (!(rec->flags & FTRACE_FL_FROZEN)) {
@@ -250,72 +238,6 @@ static inline int record_frozen(struct dyn_ftrace *rec)
250# define record_frozen(rec) ({ 0; }) 238# define record_frozen(rec) ({ 0; })
251#endif /* CONFIG_KPROBES */ 239#endif /* CONFIG_KPROBES */
252 240
253int skip_trace(unsigned long ip)
254{
255 unsigned long fl;
256 struct dyn_ftrace *rec;
257 struct hlist_node *t;
258 struct hlist_head *head;
259
260 if (frozen_record_count == 0)
261 return 0;
262
263 head = &ftrace_hash[hash_long(ip, FTRACE_HASHBITS)];
264 hlist_for_each_entry_rcu(rec, t, head, node) {
265 if (rec->ip == ip) {
266 if (record_frozen(rec)) {
267 if (rec->flags & FTRACE_FL_FAILED)
268 return 1;
269
270 if (!(rec->flags & FTRACE_FL_CONVERTED))
271 return 1;
272
273 if (!tracing_on || !ftrace_enabled)
274 return 1;
275
276 if (ftrace_filtered) {
277 fl = rec->flags & (FTRACE_FL_FILTER |
278 FTRACE_FL_NOTRACE);
279 if (!fl || (fl & FTRACE_FL_NOTRACE))
280 return 1;
281 }
282 }
283 break;
284 }
285 }
286
287 return 0;
288}
289
290static inline int
291ftrace_ip_in_hash(unsigned long ip, unsigned long key)
292{
293 struct dyn_ftrace *p;
294 struct hlist_node *t;
295 int found = 0;
296
297 hlist_for_each_entry_rcu(p, t, &ftrace_hash[key], node) {
298 if (p->ip == ip) {
299 found = 1;
300 break;
301 }
302 }
303
304 return found;
305}
306
307static inline void
308ftrace_add_hash(struct dyn_ftrace *node, unsigned long key)
309{
310 hlist_add_head_rcu(&node->node, &ftrace_hash[key]);
311}
312
313/* called from kstop_machine */
314static inline void ftrace_del_hash(struct dyn_ftrace *node)
315{
316 hlist_del(&node->node);
317}
318
319static void ftrace_free_rec(struct dyn_ftrace *rec) 241static void ftrace_free_rec(struct dyn_ftrace *rec)
320{ 242{
321 rec->ip = (unsigned long)ftrace_free_records; 243 rec->ip = (unsigned long)ftrace_free_records;
@@ -346,7 +268,6 @@ void ftrace_release(void *start, unsigned long size)
346 } 268 }
347 } 269 }
348 spin_unlock(&ftrace_lock); 270 spin_unlock(&ftrace_lock);
349
350} 271}
351 272
352static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip) 273static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip)
@@ -358,10 +279,8 @@ static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip)
358 rec = ftrace_free_records; 279 rec = ftrace_free_records;
359 280
360 if (unlikely(!(rec->flags & FTRACE_FL_FREE))) { 281 if (unlikely(!(rec->flags & FTRACE_FL_FREE))) {
361 WARN_ON_ONCE(1); 282 FTRACE_WARN_ON_ONCE(1);
362 ftrace_free_records = NULL; 283 ftrace_free_records = NULL;
363 ftrace_disabled = 1;
364 ftrace_enabled = 0;
365 return NULL; 284 return NULL;
366 } 285 }
367 286
@@ -371,76 +290,36 @@ static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip)
371 } 290 }
372 291
373 if (ftrace_pages->index == ENTRIES_PER_PAGE) { 292 if (ftrace_pages->index == ENTRIES_PER_PAGE) {
374 if (!ftrace_pages->next) 293 if (!ftrace_pages->next) {
375 return NULL; 294 /* allocate another page */
295 ftrace_pages->next =
296 (void *)get_zeroed_page(GFP_KERNEL);
297 if (!ftrace_pages->next)
298 return NULL;
299 }
376 ftrace_pages = ftrace_pages->next; 300 ftrace_pages = ftrace_pages->next;
377 } 301 }
378 302
379 return &ftrace_pages->records[ftrace_pages->index++]; 303 return &ftrace_pages->records[ftrace_pages->index++];
380} 304}
381 305
382static void 306static struct dyn_ftrace *
383ftrace_record_ip(unsigned long ip) 307ftrace_record_ip(unsigned long ip)
384{ 308{
385 struct dyn_ftrace *node; 309 struct dyn_ftrace *rec;
386 unsigned long flags;
387 unsigned long key;
388 int resched;
389 int cpu;
390 310
391 if (!ftrace_enabled || ftrace_disabled) 311 if (!ftrace_enabled || ftrace_disabled)
392 return; 312 return NULL;
393
394 resched = need_resched();
395 preempt_disable_notrace();
396 313
397 /* 314 rec = ftrace_alloc_dyn_node(ip);
398 * We simply need to protect against recursion. 315 if (!rec)
399 * Use the the raw version of smp_processor_id and not 316 return NULL;
400 * __get_cpu_var which can call debug hooks that can
401 * cause a recursive crash here.
402 */
403 cpu = raw_smp_processor_id();
404 per_cpu(ftrace_shutdown_disable_cpu, cpu)++;
405 if (per_cpu(ftrace_shutdown_disable_cpu, cpu) != 1)
406 goto out;
407
408 if (unlikely(ftrace_record_suspend))
409 goto out;
410
411 key = hash_long(ip, FTRACE_HASHBITS);
412
413 WARN_ON_ONCE(key >= FTRACE_HASHSIZE);
414
415 if (ftrace_ip_in_hash(ip, key))
416 goto out;
417
418 ftrace_hash_lock(flags);
419
420 /* This ip may have hit the hash before the lock */
421 if (ftrace_ip_in_hash(ip, key))
422 goto out_unlock;
423
424 node = ftrace_alloc_dyn_node(ip);
425 if (!node)
426 goto out_unlock;
427
428 node->ip = ip;
429
430 ftrace_add_hash(node, key);
431 317
432 ftraced_trigger = 1; 318 rec->ip = ip;
433 319
434 out_unlock: 320 list_add(&rec->list, &ftrace_new_addrs);
435 ftrace_hash_unlock(flags);
436 out:
437 per_cpu(ftrace_shutdown_disable_cpu, cpu)--;
438 321
439 /* prevent recursion with scheduler */ 322 return rec;
440 if (resched)
441 preempt_enable_no_resched_notrace();
442 else
443 preempt_enable_notrace();
444} 323}
445 324
446#define FTRACE_ADDR ((long)(ftrace_caller)) 325#define FTRACE_ADDR ((long)(ftrace_caller))
@@ -559,7 +438,6 @@ static void ftrace_replace_code(int enable)
559 rec->flags |= FTRACE_FL_FAILED; 438 rec->flags |= FTRACE_FL_FAILED;
560 if ((system_state == SYSTEM_BOOTING) || 439 if ((system_state == SYSTEM_BOOTING) ||
561 !core_kernel_text(rec->ip)) { 440 !core_kernel_text(rec->ip)) {
562 ftrace_del_hash(rec);
563 ftrace_free_rec(rec); 441 ftrace_free_rec(rec);
564 } 442 }
565 } 443 }
@@ -567,15 +445,6 @@ static void ftrace_replace_code(int enable)
567 } 445 }
568} 446}
569 447
570static void ftrace_shutdown_replenish(void)
571{
572 if (ftrace_pages->next)
573 return;
574
575 /* allocate another page */
576 ftrace_pages->next = (void *)get_zeroed_page(GFP_KERNEL);
577}
578
579static void print_ip_ins(const char *fmt, unsigned char *p) 448static void print_ip_ins(const char *fmt, unsigned char *p)
580{ 449{
581 int i; 450 int i;
@@ -591,23 +460,23 @@ ftrace_code_disable(struct dyn_ftrace *rec)
591{ 460{
592 unsigned long ip; 461 unsigned long ip;
593 unsigned char *nop, *call; 462 unsigned char *nop, *call;
594 int failed; 463 int ret;
595 464
596 ip = rec->ip; 465 ip = rec->ip;
597 466
598 nop = ftrace_nop_replace(); 467 nop = ftrace_nop_replace();
599 call = ftrace_call_replace(ip, mcount_addr); 468 call = ftrace_call_replace(ip, mcount_addr);
600 469
601 failed = ftrace_modify_code(ip, call, nop); 470 ret = ftrace_modify_code(ip, call, nop);
602 if (failed) { 471 if (ret) {
603 switch (failed) { 472 switch (ret) {
604 case 1: 473 case -EFAULT:
605 WARN_ON_ONCE(1); 474 FTRACE_WARN_ON_ONCE(1);
606 pr_info("ftrace faulted on modifying "); 475 pr_info("ftrace faulted on modifying ");
607 print_ip_sym(ip); 476 print_ip_sym(ip);
608 break; 477 break;
609 case 2: 478 case -EINVAL:
610 WARN_ON_ONCE(1); 479 FTRACE_WARN_ON_ONCE(1);
611 pr_info("ftrace failed to modify "); 480 pr_info("ftrace failed to modify ");
612 print_ip_sym(ip); 481 print_ip_sym(ip);
613 print_ip_ins(" expected: ", call); 482 print_ip_ins(" expected: ", call);
@@ -615,6 +484,15 @@ ftrace_code_disable(struct dyn_ftrace *rec)
615 print_ip_ins(" replace: ", nop); 484 print_ip_ins(" replace: ", nop);
616 printk(KERN_CONT "\n"); 485 printk(KERN_CONT "\n");
617 break; 486 break;
487 case -EPERM:
488 FTRACE_WARN_ON_ONCE(1);
489 pr_info("ftrace faulted on writing ");
490 print_ip_sym(ip);
491 break;
492 default:
493 FTRACE_WARN_ON_ONCE(1);
494 pr_info("ftrace faulted on unknown error ");
495 print_ip_sym(ip);
618 } 496 }
619 497
620 rec->flags |= FTRACE_FL_FAILED; 498 rec->flags |= FTRACE_FL_FAILED;
@@ -623,37 +501,18 @@ ftrace_code_disable(struct dyn_ftrace *rec)
623 return 1; 501 return 1;
624} 502}
625 503
626static int __ftrace_update_code(void *ignore);
627
628static int __ftrace_modify_code(void *data) 504static int __ftrace_modify_code(void *data)
629{ 505{
630 unsigned long addr;
631 int *command = data; 506 int *command = data;
632 507
633 if (*command & FTRACE_ENABLE_CALLS) { 508 if (*command & FTRACE_ENABLE_CALLS)
634 /*
635 * Update any recorded ips now that we have the
636 * machine stopped
637 */
638 __ftrace_update_code(NULL);
639 ftrace_replace_code(1); 509 ftrace_replace_code(1);
640 tracing_on = 1; 510 else if (*command & FTRACE_DISABLE_CALLS)
641 } else if (*command & FTRACE_DISABLE_CALLS) {
642 ftrace_replace_code(0); 511 ftrace_replace_code(0);
643 tracing_on = 0;
644 }
645 512
646 if (*command & FTRACE_UPDATE_TRACE_FUNC) 513 if (*command & FTRACE_UPDATE_TRACE_FUNC)
647 ftrace_update_ftrace_func(ftrace_trace_function); 514 ftrace_update_ftrace_func(ftrace_trace_function);
648 515
649 if (*command & FTRACE_ENABLE_MCOUNT) {
650 addr = (unsigned long)ftrace_record_ip;
651 ftrace_mcount_set(&addr);
652 } else if (*command & FTRACE_DISABLE_MCOUNT) {
653 addr = (unsigned long)ftrace_stub;
654 ftrace_mcount_set(&addr);
655 }
656
657 return 0; 516 return 0;
658} 517}
659 518
@@ -662,26 +521,9 @@ static void ftrace_run_update_code(int command)
662 stop_machine(__ftrace_modify_code, &command, NULL); 521 stop_machine(__ftrace_modify_code, &command, NULL);
663} 522}
664 523
665void ftrace_disable_daemon(void)
666{
667 /* Stop the daemon from calling kstop_machine */
668 mutex_lock(&ftraced_lock);
669 ftraced_stop = 1;
670 mutex_unlock(&ftraced_lock);
671
672 ftrace_force_update();
673}
674
675void ftrace_enable_daemon(void)
676{
677 mutex_lock(&ftraced_lock);
678 ftraced_stop = 0;
679 mutex_unlock(&ftraced_lock);
680
681 ftrace_force_update();
682}
683
684static ftrace_func_t saved_ftrace_func; 524static ftrace_func_t saved_ftrace_func;
525static int ftrace_start;
526static DEFINE_MUTEX(ftrace_start_lock);
685 527
686static void ftrace_startup(void) 528static void ftrace_startup(void)
687{ 529{
@@ -690,9 +532,9 @@ static void ftrace_startup(void)
690 if (unlikely(ftrace_disabled)) 532 if (unlikely(ftrace_disabled))
691 return; 533 return;
692 534
693 mutex_lock(&ftraced_lock); 535 mutex_lock(&ftrace_start_lock);
694 ftraced_suspend++; 536 ftrace_start++;
695 if (ftraced_suspend == 1) 537 if (ftrace_start == 1)
696 command |= FTRACE_ENABLE_CALLS; 538 command |= FTRACE_ENABLE_CALLS;
697 539
698 if (saved_ftrace_func != ftrace_trace_function) { 540 if (saved_ftrace_func != ftrace_trace_function) {
@@ -705,7 +547,7 @@ static void ftrace_startup(void)
705 547
706 ftrace_run_update_code(command); 548 ftrace_run_update_code(command);
707 out: 549 out:
708 mutex_unlock(&ftraced_lock); 550 mutex_unlock(&ftrace_start_lock);
709} 551}
710 552
711static void ftrace_shutdown(void) 553static void ftrace_shutdown(void)
@@ -715,9 +557,9 @@ static void ftrace_shutdown(void)
715 if (unlikely(ftrace_disabled)) 557 if (unlikely(ftrace_disabled))
716 return; 558 return;
717 559
718 mutex_lock(&ftraced_lock); 560 mutex_lock(&ftrace_start_lock);
719 ftraced_suspend--; 561 ftrace_start--;
720 if (!ftraced_suspend) 562 if (!ftrace_start)
721 command |= FTRACE_DISABLE_CALLS; 563 command |= FTRACE_DISABLE_CALLS;
722 564
723 if (saved_ftrace_func != ftrace_trace_function) { 565 if (saved_ftrace_func != ftrace_trace_function) {
@@ -730,7 +572,7 @@ static void ftrace_shutdown(void)
730 572
731 ftrace_run_update_code(command); 573 ftrace_run_update_code(command);
732 out: 574 out:
733 mutex_unlock(&ftraced_lock); 575 mutex_unlock(&ftrace_start_lock);
734} 576}
735 577
736static void ftrace_startup_sysctl(void) 578static void ftrace_startup_sysctl(void)
@@ -740,15 +582,15 @@ static void ftrace_startup_sysctl(void)
740 if (unlikely(ftrace_disabled)) 582 if (unlikely(ftrace_disabled))
741 return; 583 return;
742 584
743 mutex_lock(&ftraced_lock); 585 mutex_lock(&ftrace_start_lock);
744 /* Force update next time */ 586 /* Force update next time */
745 saved_ftrace_func = NULL; 587 saved_ftrace_func = NULL;
746 /* ftraced_suspend is true if we want ftrace running */ 588 /* ftrace_start is true if we want ftrace running */
747 if (ftraced_suspend) 589 if (ftrace_start)
748 command |= FTRACE_ENABLE_CALLS; 590 command |= FTRACE_ENABLE_CALLS;
749 591
750 ftrace_run_update_code(command); 592 ftrace_run_update_code(command);
751 mutex_unlock(&ftraced_lock); 593 mutex_unlock(&ftrace_start_lock);
752} 594}
753 595
754static void ftrace_shutdown_sysctl(void) 596static void ftrace_shutdown_sysctl(void)
@@ -758,112 +600,50 @@ static void ftrace_shutdown_sysctl(void)
758 if (unlikely(ftrace_disabled)) 600 if (unlikely(ftrace_disabled))
759 return; 601 return;
760 602
761 mutex_lock(&ftraced_lock); 603 mutex_lock(&ftrace_start_lock);
762 /* ftraced_suspend is true if ftrace is running */ 604 /* ftrace_start is true if ftrace is running */
763 if (ftraced_suspend) 605 if (ftrace_start)
764 command |= FTRACE_DISABLE_CALLS; 606 command |= FTRACE_DISABLE_CALLS;
765 607
766 ftrace_run_update_code(command); 608 ftrace_run_update_code(command);
767 mutex_unlock(&ftraced_lock); 609 mutex_unlock(&ftrace_start_lock);
768} 610}
769 611
770static cycle_t ftrace_update_time; 612static cycle_t ftrace_update_time;
771static unsigned long ftrace_update_cnt; 613static unsigned long ftrace_update_cnt;
772unsigned long ftrace_update_tot_cnt; 614unsigned long ftrace_update_tot_cnt;
773 615
774static int __ftrace_update_code(void *ignore) 616static int ftrace_update_code(void)
775{ 617{
776 int i, save_ftrace_enabled; 618 struct dyn_ftrace *p, *t;
777 cycle_t start, stop; 619 cycle_t start, stop;
778 struct dyn_ftrace *p;
779 struct hlist_node *t, *n;
780 struct hlist_head *head, temp_list;
781
782 /* Don't be recording funcs now */
783 ftrace_record_suspend++;
784 save_ftrace_enabled = ftrace_enabled;
785 ftrace_enabled = 0;
786 620
787 start = ftrace_now(raw_smp_processor_id()); 621 start = ftrace_now(raw_smp_processor_id());
788 ftrace_update_cnt = 0; 622 ftrace_update_cnt = 0;
789 623
790 /* No locks needed, the machine is stopped! */ 624 list_for_each_entry_safe(p, t, &ftrace_new_addrs, list) {
791 for (i = 0; i < FTRACE_HASHSIZE; i++) {
792 INIT_HLIST_HEAD(&temp_list);
793 head = &ftrace_hash[i];
794
795 /* all CPUS are stopped, we are safe to modify code */
796 hlist_for_each_entry_safe(p, t, n, head, node) {
797 /* Skip over failed records which have not been
798 * freed. */
799 if (p->flags & FTRACE_FL_FAILED)
800 continue;
801
802 /* Unconverted records are always at the head of the
803 * hash bucket. Once we encounter a converted record,
804 * simply skip over to the next bucket. Saves ftraced
805 * some processor cycles (ftrace does its bid for
806 * global warming :-p ). */
807 if (p->flags & (FTRACE_FL_CONVERTED))
808 break;
809 625
810 /* Ignore updates to this record's mcount site. 626 /* If something went wrong, bail without enabling anything */
811 * Reintroduce this record at the head of this 627 if (unlikely(ftrace_disabled))
812 * bucket to attempt to "convert" it again if 628 return -1;
813 * the kprobe on it is unregistered before the
814 * next run. */
815 if (get_kprobe((void *)p->ip)) {
816 ftrace_del_hash(p);
817 INIT_HLIST_NODE(&p->node);
818 hlist_add_head(&p->node, &temp_list);
819 freeze_record(p);
820 continue;
821 } else {
822 unfreeze_record(p);
823 }
824 629
825 /* convert record (i.e, patch mcount-call with NOP) */ 630 list_del_init(&p->list);
826 if (ftrace_code_disable(p)) {
827 p->flags |= FTRACE_FL_CONVERTED;
828 ftrace_update_cnt++;
829 } else {
830 if ((system_state == SYSTEM_BOOTING) ||
831 !core_kernel_text(p->ip)) {
832 ftrace_del_hash(p);
833 ftrace_free_rec(p);
834 }
835 }
836 }
837 631
838 hlist_for_each_entry_safe(p, t, n, &temp_list, node) { 632 /* convert record (i.e, patch mcount-call with NOP) */
839 hlist_del(&p->node); 633 if (ftrace_code_disable(p)) {
840 INIT_HLIST_NODE(&p->node); 634 p->flags |= FTRACE_FL_CONVERTED;
841 hlist_add_head(&p->node, head); 635 ftrace_update_cnt++;
842 } 636 } else
637 ftrace_free_rec(p);
843 } 638 }
844 639
845 stop = ftrace_now(raw_smp_processor_id()); 640 stop = ftrace_now(raw_smp_processor_id());
846 ftrace_update_time = stop - start; 641 ftrace_update_time = stop - start;
847 ftrace_update_tot_cnt += ftrace_update_cnt; 642 ftrace_update_tot_cnt += ftrace_update_cnt;
848 ftraced_trigger = 0;
849
850 ftrace_enabled = save_ftrace_enabled;
851 ftrace_record_suspend--;
852 643
853 return 0; 644 return 0;
854} 645}
855 646
856static int ftrace_update_code(void)
857{
858 if (unlikely(ftrace_disabled) ||
859 !ftrace_enabled || !ftraced_trigger)
860 return 0;
861
862 stop_machine(__ftrace_update_code, NULL, NULL);
863
864 return 1;
865}
866
867static int __init ftrace_dyn_table_alloc(unsigned long num_to_init) 647static int __init ftrace_dyn_table_alloc(unsigned long num_to_init)
868{ 648{
869 struct ftrace_page *pg; 649 struct ftrace_page *pg;
@@ -892,8 +672,8 @@ static int __init ftrace_dyn_table_alloc(unsigned long num_to_init)
892 pg = ftrace_pages = ftrace_pages_start; 672 pg = ftrace_pages = ftrace_pages_start;
893 673
894 cnt = num_to_init / ENTRIES_PER_PAGE; 674 cnt = num_to_init / ENTRIES_PER_PAGE;
895 pr_info("ftrace: allocating %ld hash entries in %d pages\n", 675 pr_info("ftrace: allocating %ld entries in %d pages\n",
896 num_to_init, cnt); 676 num_to_init, cnt + 1);
897 677
898 for (i = 0; i < cnt; i++) { 678 for (i = 0; i < cnt; i++) {
899 pg->next = (void *)get_zeroed_page(GFP_KERNEL); 679 pg->next = (void *)get_zeroed_page(GFP_KERNEL);
@@ -973,13 +753,11 @@ static void *t_start(struct seq_file *m, loff_t *pos)
973 void *p = NULL; 753 void *p = NULL;
974 loff_t l = -1; 754 loff_t l = -1;
975 755
976 if (*pos != iter->pos) { 756 if (*pos > iter->pos)
977 for (p = t_next(m, p, &l); p && l < *pos; p = t_next(m, p, &l)) 757 *pos = iter->pos;
978 ; 758
979 } else { 759 l = *pos;
980 l = *pos; 760 p = t_next(m, p, &l);
981 p = t_next(m, p, &l);
982 }
983 761
984 return p; 762 return p;
985} 763}
@@ -990,15 +768,21 @@ static void t_stop(struct seq_file *m, void *p)
990 768
991static int t_show(struct seq_file *m, void *v) 769static int t_show(struct seq_file *m, void *v)
992{ 770{
771 struct ftrace_iterator *iter = m->private;
993 struct dyn_ftrace *rec = v; 772 struct dyn_ftrace *rec = v;
994 char str[KSYM_SYMBOL_LEN]; 773 char str[KSYM_SYMBOL_LEN];
774 int ret = 0;
995 775
996 if (!rec) 776 if (!rec)
997 return 0; 777 return 0;
998 778
999 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str); 779 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
1000 780
1001 seq_printf(m, "%s\n", str); 781 ret = seq_printf(m, "%s\n", str);
782 if (ret < 0) {
783 iter->pos--;
784 iter->idx--;
785 }
1002 786
1003 return 0; 787 return 0;
1004} 788}
@@ -1024,7 +808,7 @@ ftrace_avail_open(struct inode *inode, struct file *file)
1024 return -ENOMEM; 808 return -ENOMEM;
1025 809
1026 iter->pg = ftrace_pages_start; 810 iter->pg = ftrace_pages_start;
1027 iter->pos = -1; 811 iter->pos = 0;
1028 812
1029 ret = seq_open(file, &show_ftrace_seq_ops); 813 ret = seq_open(file, &show_ftrace_seq_ops);
1030 if (!ret) { 814 if (!ret) {
@@ -1111,7 +895,7 @@ ftrace_regex_open(struct inode *inode, struct file *file, int enable)
1111 895
1112 if (file->f_mode & FMODE_READ) { 896 if (file->f_mode & FMODE_READ) {
1113 iter->pg = ftrace_pages_start; 897 iter->pg = ftrace_pages_start;
1114 iter->pos = -1; 898 iter->pos = 0;
1115 iter->flags = enable ? FTRACE_ITER_FILTER : 899 iter->flags = enable ? FTRACE_ITER_FILTER :
1116 FTRACE_ITER_NOTRACE; 900 FTRACE_ITER_NOTRACE;
1117 901
@@ -1401,10 +1185,10 @@ ftrace_regex_release(struct inode *inode, struct file *file, int enable)
1401 } 1185 }
1402 1186
1403 mutex_lock(&ftrace_sysctl_lock); 1187 mutex_lock(&ftrace_sysctl_lock);
1404 mutex_lock(&ftraced_lock); 1188 mutex_lock(&ftrace_start_lock);
1405 if (iter->filtered && ftraced_suspend && ftrace_enabled) 1189 if (iter->filtered && ftrace_start && ftrace_enabled)
1406 ftrace_run_update_code(FTRACE_ENABLE_CALLS); 1190 ftrace_run_update_code(FTRACE_ENABLE_CALLS);
1407 mutex_unlock(&ftraced_lock); 1191 mutex_unlock(&ftrace_start_lock);
1408 mutex_unlock(&ftrace_sysctl_lock); 1192 mutex_unlock(&ftrace_sysctl_lock);
1409 1193
1410 kfree(iter); 1194 kfree(iter);
@@ -1424,55 +1208,6 @@ ftrace_notrace_release(struct inode *inode, struct file *file)
1424 return ftrace_regex_release(inode, file, 0); 1208 return ftrace_regex_release(inode, file, 0);
1425} 1209}
1426 1210
1427static ssize_t
1428ftraced_read(struct file *filp, char __user *ubuf,
1429 size_t cnt, loff_t *ppos)
1430{
1431 /* don't worry about races */
1432 char *buf = ftraced_stop ? "disabled\n" : "enabled\n";
1433 int r = strlen(buf);
1434
1435 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
1436}
1437
1438static ssize_t
1439ftraced_write(struct file *filp, const char __user *ubuf,
1440 size_t cnt, loff_t *ppos)
1441{
1442 char buf[64];
1443 long val;
1444 int ret;
1445
1446 if (cnt >= sizeof(buf))
1447 return -EINVAL;
1448
1449 if (copy_from_user(&buf, ubuf, cnt))
1450 return -EFAULT;
1451
1452 if (strncmp(buf, "enable", 6) == 0)
1453 val = 1;
1454 else if (strncmp(buf, "disable", 7) == 0)
1455 val = 0;
1456 else {
1457 buf[cnt] = 0;
1458
1459 ret = strict_strtoul(buf, 10, &val);
1460 if (ret < 0)
1461 return ret;
1462
1463 val = !!val;
1464 }
1465
1466 if (val)
1467 ftrace_enable_daemon();
1468 else
1469 ftrace_disable_daemon();
1470
1471 filp->f_pos += cnt;
1472
1473 return cnt;
1474}
1475
1476static struct file_operations ftrace_avail_fops = { 1211static struct file_operations ftrace_avail_fops = {
1477 .open = ftrace_avail_open, 1212 .open = ftrace_avail_open,
1478 .read = seq_read, 1213 .read = seq_read,
@@ -1503,54 +1238,6 @@ static struct file_operations ftrace_notrace_fops = {
1503 .release = ftrace_notrace_release, 1238 .release = ftrace_notrace_release,
1504}; 1239};
1505 1240
1506static struct file_operations ftraced_fops = {
1507 .open = tracing_open_generic,
1508 .read = ftraced_read,
1509 .write = ftraced_write,
1510};
1511
1512/**
1513 * ftrace_force_update - force an update to all recording ftrace functions
1514 */
1515int ftrace_force_update(void)
1516{
1517 int ret = 0;
1518
1519 if (unlikely(ftrace_disabled))
1520 return -ENODEV;
1521
1522 mutex_lock(&ftrace_sysctl_lock);
1523 mutex_lock(&ftraced_lock);
1524
1525 /*
1526 * If ftraced_trigger is not set, then there is nothing
1527 * to update.
1528 */
1529 if (ftraced_trigger && !ftrace_update_code())
1530 ret = -EBUSY;
1531
1532 mutex_unlock(&ftraced_lock);
1533 mutex_unlock(&ftrace_sysctl_lock);
1534
1535 return ret;
1536}
1537
1538static void ftrace_force_shutdown(void)
1539{
1540 struct task_struct *task;
1541 int command = FTRACE_DISABLE_CALLS | FTRACE_UPDATE_TRACE_FUNC;
1542
1543 mutex_lock(&ftraced_lock);
1544 task = ftraced_task;
1545 ftraced_task = NULL;
1546 ftraced_suspend = -1;
1547 ftrace_run_update_code(command);
1548 mutex_unlock(&ftraced_lock);
1549
1550 if (task)
1551 kthread_stop(task);
1552}
1553
1554static __init int ftrace_init_debugfs(void) 1241static __init int ftrace_init_debugfs(void)
1555{ 1242{
1556 struct dentry *d_tracer; 1243 struct dentry *d_tracer;
@@ -1581,17 +1268,11 @@ static __init int ftrace_init_debugfs(void)
1581 pr_warning("Could not create debugfs " 1268 pr_warning("Could not create debugfs "
1582 "'set_ftrace_notrace' entry\n"); 1269 "'set_ftrace_notrace' entry\n");
1583 1270
1584 entry = debugfs_create_file("ftraced_enabled", 0644, d_tracer,
1585 NULL, &ftraced_fops);
1586 if (!entry)
1587 pr_warning("Could not create debugfs "
1588 "'ftraced_enabled' entry\n");
1589 return 0; 1271 return 0;
1590} 1272}
1591 1273
1592fs_initcall(ftrace_init_debugfs); 1274fs_initcall(ftrace_init_debugfs);
1593 1275
1594#ifdef CONFIG_FTRACE_MCOUNT_RECORD
1595static int ftrace_convert_nops(unsigned long *start, 1276static int ftrace_convert_nops(unsigned long *start,
1596 unsigned long *end) 1277 unsigned long *end)
1597{ 1278{
@@ -1599,20 +1280,18 @@ static int ftrace_convert_nops(unsigned long *start,
1599 unsigned long addr; 1280 unsigned long addr;
1600 unsigned long flags; 1281 unsigned long flags;
1601 1282
1283 mutex_lock(&ftrace_start_lock);
1602 p = start; 1284 p = start;
1603 while (p < end) { 1285 while (p < end) {
1604 addr = ftrace_call_adjust(*p++); 1286 addr = ftrace_call_adjust(*p++);
1605 /* should not be called from interrupt context */
1606 spin_lock(&ftrace_lock);
1607 ftrace_record_ip(addr); 1287 ftrace_record_ip(addr);
1608 spin_unlock(&ftrace_lock);
1609 ftrace_shutdown_replenish();
1610 } 1288 }
1611 1289
1612 /* p is ignored */ 1290 /* disable interrupts to prevent kstop machine */
1613 local_irq_save(flags); 1291 local_irq_save(flags);
1614 __ftrace_update_code(p); 1292 ftrace_update_code();
1615 local_irq_restore(flags); 1293 local_irq_restore(flags);
1294 mutex_unlock(&ftrace_start_lock);
1616 1295
1617 return 0; 1296 return 0;
1618} 1297}
@@ -1658,130 +1337,34 @@ void __init ftrace_init(void)
1658 failed: 1337 failed:
1659 ftrace_disabled = 1; 1338 ftrace_disabled = 1;
1660} 1339}
1661#else /* CONFIG_FTRACE_MCOUNT_RECORD */
1662static int ftraced(void *ignore)
1663{
1664 unsigned long usecs;
1665
1666 while (!kthread_should_stop()) {
1667
1668 set_current_state(TASK_INTERRUPTIBLE);
1669 1340
1670 /* check once a second */ 1341#else
1671 schedule_timeout(HZ);
1672
1673 if (unlikely(ftrace_disabled))
1674 continue;
1675
1676 mutex_lock(&ftrace_sysctl_lock);
1677 mutex_lock(&ftraced_lock);
1678 if (!ftraced_suspend && !ftraced_stop &&
1679 ftrace_update_code()) {
1680 usecs = nsecs_to_usecs(ftrace_update_time);
1681 if (ftrace_update_tot_cnt > 100000) {
1682 ftrace_update_tot_cnt = 0;
1683 pr_info("hm, dftrace overflow: %lu change%s"
1684 " (%lu total) in %lu usec%s\n",
1685 ftrace_update_cnt,
1686 ftrace_update_cnt != 1 ? "s" : "",
1687 ftrace_update_tot_cnt,
1688 usecs, usecs != 1 ? "s" : "");
1689 ftrace_disabled = 1;
1690 WARN_ON_ONCE(1);
1691 }
1692 }
1693 mutex_unlock(&ftraced_lock);
1694 mutex_unlock(&ftrace_sysctl_lock);
1695
1696 ftrace_shutdown_replenish();
1697 }
1698 __set_current_state(TASK_RUNNING);
1699 return 0;
1700}
1701 1342
1702static int __init ftrace_dynamic_init(void) 1343static int __init ftrace_nodyn_init(void)
1703{ 1344{
1704 struct task_struct *p; 1345 ftrace_enabled = 1;
1705 unsigned long addr;
1706 int ret;
1707
1708 addr = (unsigned long)ftrace_record_ip;
1709
1710 stop_machine(ftrace_dyn_arch_init, &addr, NULL);
1711
1712 /* ftrace_dyn_arch_init places the return code in addr */
1713 if (addr) {
1714 ret = (int)addr;
1715 goto failed;
1716 }
1717
1718 ret = ftrace_dyn_table_alloc(NR_TO_INIT);
1719 if (ret)
1720 goto failed;
1721
1722 p = kthread_run(ftraced, NULL, "ftraced");
1723 if (IS_ERR(p)) {
1724 ret = -1;
1725 goto failed;
1726 }
1727
1728 last_ftrace_enabled = ftrace_enabled = 1;
1729 ftraced_task = p;
1730
1731 return 0; 1346 return 0;
1732
1733 failed:
1734 ftrace_disabled = 1;
1735 return ret;
1736} 1347}
1348device_initcall(ftrace_nodyn_init);
1737 1349
1738core_initcall(ftrace_dynamic_init);
1739#endif /* CONFIG_FTRACE_MCOUNT_RECORD */
1740
1741#else
1742# define ftrace_startup() do { } while (0) 1350# define ftrace_startup() do { } while (0)
1743# define ftrace_shutdown() do { } while (0) 1351# define ftrace_shutdown() do { } while (0)
1744# define ftrace_startup_sysctl() do { } while (0) 1352# define ftrace_startup_sysctl() do { } while (0)
1745# define ftrace_shutdown_sysctl() do { } while (0) 1353# define ftrace_shutdown_sysctl() do { } while (0)
1746# define ftrace_force_shutdown() do { } while (0)
1747#endif /* CONFIG_DYNAMIC_FTRACE */ 1354#endif /* CONFIG_DYNAMIC_FTRACE */
1748 1355
1749/** 1356/**
1750 * ftrace_kill_atomic - kill ftrace from critical sections 1357 * ftrace_kill - kill ftrace
1751 * 1358 *
1752 * This function should be used by panic code. It stops ftrace 1359 * This function should be used by panic code. It stops ftrace
1753 * but in a not so nice way. If you need to simply kill ftrace 1360 * but in a not so nice way. If you need to simply kill ftrace
1754 * from a non-atomic section, use ftrace_kill. 1361 * from a non-atomic section, use ftrace_kill.
1755 */ 1362 */
1756void ftrace_kill_atomic(void)
1757{
1758 ftrace_disabled = 1;
1759 ftrace_enabled = 0;
1760#ifdef CONFIG_DYNAMIC_FTRACE
1761 ftraced_suspend = -1;
1762#endif
1763 clear_ftrace_function();
1764}
1765
1766/**
1767 * ftrace_kill - totally shutdown ftrace
1768 *
1769 * This is a safety measure. If something was detected that seems
1770 * wrong, calling this function will keep ftrace from doing
1771 * any more modifications, and updates.
1772 * used when something went wrong.
1773 */
1774void ftrace_kill(void) 1363void ftrace_kill(void)
1775{ 1364{
1776 mutex_lock(&ftrace_sysctl_lock);
1777 ftrace_disabled = 1; 1365 ftrace_disabled = 1;
1778 ftrace_enabled = 0; 1366 ftrace_enabled = 0;
1779
1780 clear_ftrace_function(); 1367 clear_ftrace_function();
1781 mutex_unlock(&ftrace_sysctl_lock);
1782
1783 /* Try to totally disable ftrace */
1784 ftrace_force_shutdown();
1785} 1368}
1786 1369
1787/** 1370/**
@@ -1870,3 +1453,4 @@ ftrace_enable_sysctl(struct ctl_table *table, int write,
1870 mutex_unlock(&ftrace_sysctl_lock); 1453 mutex_unlock(&ftrace_sysctl_lock);
1871 return ret; 1454 return ret;
1872} 1455}
1456