aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/trace/ftrace.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/trace/ftrace.c')
-rw-r--r--kernel/trace/ftrace.c608
1 files changed, 92 insertions, 516 deletions
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
index 4dda4f60a2a9..7618c528756b 100644
--- a/kernel/trace/ftrace.c
+++ b/kernel/trace/ftrace.c
@@ -25,13 +25,24 @@
25#include <linux/ftrace.h> 25#include <linux/ftrace.h>
26#include <linux/sysctl.h> 26#include <linux/sysctl.h>
27#include <linux/ctype.h> 27#include <linux/ctype.h>
28#include <linux/hash.h>
29#include <linux/list.h> 28#include <linux/list.h>
30 29
31#include <asm/ftrace.h> 30#include <asm/ftrace.h>
32 31
33#include "trace.h" 32#include "trace.h"
34 33
34#define FTRACE_WARN_ON(cond) \
35 do { \
36 if (WARN_ON(cond)) \
37 ftrace_kill(); \
38 } while (0)
39
40#define FTRACE_WARN_ON_ONCE(cond) \
41 do { \
42 if (WARN_ON_ONCE(cond)) \
43 ftrace_kill(); \
44 } while (0)
45
35/* ftrace_enabled is a method to turn ftrace on or off */ 46/* ftrace_enabled is a method to turn ftrace on or off */
36int ftrace_enabled __read_mostly; 47int ftrace_enabled __read_mostly;
37static int last_ftrace_enabled; 48static int last_ftrace_enabled;
@@ -153,21 +164,8 @@ static int __unregister_ftrace_function(struct ftrace_ops *ops)
153} 164}
154 165
155#ifdef CONFIG_DYNAMIC_FTRACE 166#ifdef CONFIG_DYNAMIC_FTRACE
156
157#ifndef CONFIG_FTRACE_MCOUNT_RECORD 167#ifndef CONFIG_FTRACE_MCOUNT_RECORD
158/* 168# error Dynamic ftrace depends on MCOUNT_RECORD
159 * The hash lock is only needed when the recording of the mcount
160 * callers are dynamic. That is, by the caller themselves and
161 * not recorded via the compilation.
162 */
163static DEFINE_SPINLOCK(ftrace_hash_lock);
164#define ftrace_hash_lock(flags) spin_lock_irqsave(&ftrace_hash_lock, flags)
165#define ftrace_hash_unlock(flags) \
166 spin_unlock_irqrestore(&ftrace_hash_lock, flags)
167#else
168/* This is protected via the ftrace_lock with MCOUNT_RECORD. */
169#define ftrace_hash_lock(flags) do { (void)(flags); } while (0)
170#define ftrace_hash_unlock(flags) do { } while(0)
171#endif 169#endif
172 170
173/* 171/*
@@ -178,8 +176,6 @@ static DEFINE_SPINLOCK(ftrace_hash_lock);
178 */ 176 */
179static unsigned long mcount_addr = MCOUNT_ADDR; 177static unsigned long mcount_addr = MCOUNT_ADDR;
180 178
181static struct task_struct *ftraced_task;
182
183enum { 179enum {
184 FTRACE_ENABLE_CALLS = (1 << 0), 180 FTRACE_ENABLE_CALLS = (1 << 0),
185 FTRACE_DISABLE_CALLS = (1 << 1), 181 FTRACE_DISABLE_CALLS = (1 << 1),
@@ -190,13 +186,9 @@ enum {
190 186
191static int ftrace_filtered; 187static int ftrace_filtered;
192static int tracing_on; 188static int tracing_on;
193static int frozen_record_count;
194 189
195static struct hlist_head ftrace_hash[FTRACE_HASHSIZE]; 190static LIST_HEAD(ftrace_new_addrs);
196 191
197static DEFINE_PER_CPU(int, ftrace_shutdown_disable_cpu);
198
199static DEFINE_MUTEX(ftraced_lock);
200static DEFINE_MUTEX(ftrace_regex_lock); 192static DEFINE_MUTEX(ftrace_regex_lock);
201 193
202struct ftrace_page { 194struct ftrace_page {
@@ -214,16 +206,13 @@ struct ftrace_page {
214static struct ftrace_page *ftrace_pages_start; 206static struct ftrace_page *ftrace_pages_start;
215static struct ftrace_page *ftrace_pages; 207static struct ftrace_page *ftrace_pages;
216 208
217static int ftraced_trigger;
218static int ftraced_suspend;
219static int ftraced_stop;
220
221static int ftrace_record_suspend;
222
223static struct dyn_ftrace *ftrace_free_records; 209static struct dyn_ftrace *ftrace_free_records;
224 210
225 211
226#ifdef CONFIG_KPROBES 212#ifdef CONFIG_KPROBES
213
214static int frozen_record_count;
215
227static inline void freeze_record(struct dyn_ftrace *rec) 216static inline void freeze_record(struct dyn_ftrace *rec)
228{ 217{
229 if (!(rec->flags & FTRACE_FL_FROZEN)) { 218 if (!(rec->flags & FTRACE_FL_FROZEN)) {
@@ -250,72 +239,6 @@ static inline int record_frozen(struct dyn_ftrace *rec)
250# define record_frozen(rec) ({ 0; }) 239# define record_frozen(rec) ({ 0; })
251#endif /* CONFIG_KPROBES */ 240#endif /* CONFIG_KPROBES */
252 241
253int skip_trace(unsigned long ip)
254{
255 unsigned long fl;
256 struct dyn_ftrace *rec;
257 struct hlist_node *t;
258 struct hlist_head *head;
259
260 if (frozen_record_count == 0)
261 return 0;
262
263 head = &ftrace_hash[hash_long(ip, FTRACE_HASHBITS)];
264 hlist_for_each_entry_rcu(rec, t, head, node) {
265 if (rec->ip == ip) {
266 if (record_frozen(rec)) {
267 if (rec->flags & FTRACE_FL_FAILED)
268 return 1;
269
270 if (!(rec->flags & FTRACE_FL_CONVERTED))
271 return 1;
272
273 if (!tracing_on || !ftrace_enabled)
274 return 1;
275
276 if (ftrace_filtered) {
277 fl = rec->flags & (FTRACE_FL_FILTER |
278 FTRACE_FL_NOTRACE);
279 if (!fl || (fl & FTRACE_FL_NOTRACE))
280 return 1;
281 }
282 }
283 break;
284 }
285 }
286
287 return 0;
288}
289
290static inline int
291ftrace_ip_in_hash(unsigned long ip, unsigned long key)
292{
293 struct dyn_ftrace *p;
294 struct hlist_node *t;
295 int found = 0;
296
297 hlist_for_each_entry_rcu(p, t, &ftrace_hash[key], node) {
298 if (p->ip == ip) {
299 found = 1;
300 break;
301 }
302 }
303
304 return found;
305}
306
307static inline void
308ftrace_add_hash(struct dyn_ftrace *node, unsigned long key)
309{
310 hlist_add_head_rcu(&node->node, &ftrace_hash[key]);
311}
312
313/* called from kstop_machine */
314static inline void ftrace_del_hash(struct dyn_ftrace *node)
315{
316 hlist_del(&node->node);
317}
318
319static void ftrace_free_rec(struct dyn_ftrace *rec) 242static void ftrace_free_rec(struct dyn_ftrace *rec)
320{ 243{
321 rec->ip = (unsigned long)ftrace_free_records; 244 rec->ip = (unsigned long)ftrace_free_records;
@@ -346,7 +269,6 @@ void ftrace_release(void *start, unsigned long size)
346 } 269 }
347 } 270 }
348 spin_unlock(&ftrace_lock); 271 spin_unlock(&ftrace_lock);
349
350} 272}
351 273
352static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip) 274static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip)
@@ -358,10 +280,8 @@ static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip)
358 rec = ftrace_free_records; 280 rec = ftrace_free_records;
359 281
360 if (unlikely(!(rec->flags & FTRACE_FL_FREE))) { 282 if (unlikely(!(rec->flags & FTRACE_FL_FREE))) {
361 WARN_ON_ONCE(1); 283 FTRACE_WARN_ON_ONCE(1);
362 ftrace_free_records = NULL; 284 ftrace_free_records = NULL;
363 ftrace_disabled = 1;
364 ftrace_enabled = 0;
365 return NULL; 285 return NULL;
366 } 286 }
367 287
@@ -371,76 +291,36 @@ static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip)
371 } 291 }
372 292
373 if (ftrace_pages->index == ENTRIES_PER_PAGE) { 293 if (ftrace_pages->index == ENTRIES_PER_PAGE) {
374 if (!ftrace_pages->next) 294 if (!ftrace_pages->next) {
375 return NULL; 295 /* allocate another page */
296 ftrace_pages->next =
297 (void *)get_zeroed_page(GFP_KERNEL);
298 if (!ftrace_pages->next)
299 return NULL;
300 }
376 ftrace_pages = ftrace_pages->next; 301 ftrace_pages = ftrace_pages->next;
377 } 302 }
378 303
379 return &ftrace_pages->records[ftrace_pages->index++]; 304 return &ftrace_pages->records[ftrace_pages->index++];
380} 305}
381 306
382static void 307static struct dyn_ftrace *
383ftrace_record_ip(unsigned long ip) 308ftrace_record_ip(unsigned long ip)
384{ 309{
385 struct dyn_ftrace *node; 310 struct dyn_ftrace *rec;
386 unsigned long flags;
387 unsigned long key;
388 int resched;
389 int cpu;
390 311
391 if (!ftrace_enabled || ftrace_disabled) 312 if (!ftrace_enabled || ftrace_disabled)
392 return; 313 return NULL;
393
394 resched = need_resched();
395 preempt_disable_notrace();
396
397 /*
398 * We simply need to protect against recursion.
399 * Use the the raw version of smp_processor_id and not
400 * __get_cpu_var which can call debug hooks that can
401 * cause a recursive crash here.
402 */
403 cpu = raw_smp_processor_id();
404 per_cpu(ftrace_shutdown_disable_cpu, cpu)++;
405 if (per_cpu(ftrace_shutdown_disable_cpu, cpu) != 1)
406 goto out;
407 314
408 if (unlikely(ftrace_record_suspend)) 315 rec = ftrace_alloc_dyn_node(ip);
409 goto out; 316 if (!rec)
410 317 return NULL;
411 key = hash_long(ip, FTRACE_HASHBITS);
412
413 WARN_ON_ONCE(key >= FTRACE_HASHSIZE);
414
415 if (ftrace_ip_in_hash(ip, key))
416 goto out;
417
418 ftrace_hash_lock(flags);
419
420 /* This ip may have hit the hash before the lock */
421 if (ftrace_ip_in_hash(ip, key))
422 goto out_unlock;
423
424 node = ftrace_alloc_dyn_node(ip);
425 if (!node)
426 goto out_unlock;
427
428 node->ip = ip;
429 318
430 ftrace_add_hash(node, key); 319 rec->ip = ip;
431 320
432 ftraced_trigger = 1; 321 list_add(&rec->list, &ftrace_new_addrs);
433 322
434 out_unlock: 323 return rec;
435 ftrace_hash_unlock(flags);
436 out:
437 per_cpu(ftrace_shutdown_disable_cpu, cpu)--;
438
439 /* prevent recursion with scheduler */
440 if (resched)
441 preempt_enable_no_resched_notrace();
442 else
443 preempt_enable_notrace();
444} 324}
445 325
446#define FTRACE_ADDR ((long)(ftrace_caller)) 326#define FTRACE_ADDR ((long)(ftrace_caller))
@@ -559,7 +439,6 @@ static void ftrace_replace_code(int enable)
559 rec->flags |= FTRACE_FL_FAILED; 439 rec->flags |= FTRACE_FL_FAILED;
560 if ((system_state == SYSTEM_BOOTING) || 440 if ((system_state == SYSTEM_BOOTING) ||
561 !core_kernel_text(rec->ip)) { 441 !core_kernel_text(rec->ip)) {
562 ftrace_del_hash(rec);
563 ftrace_free_rec(rec); 442 ftrace_free_rec(rec);
564 } 443 }
565 } 444 }
@@ -567,15 +446,6 @@ static void ftrace_replace_code(int enable)
567 } 446 }
568} 447}
569 448
570static void ftrace_shutdown_replenish(void)
571{
572 if (ftrace_pages->next)
573 return;
574
575 /* allocate another page */
576 ftrace_pages->next = (void *)get_zeroed_page(GFP_KERNEL);
577}
578
579static void print_ip_ins(const char *fmt, unsigned char *p) 449static void print_ip_ins(const char *fmt, unsigned char *p)
580{ 450{
581 int i; 451 int i;
@@ -591,23 +461,23 @@ ftrace_code_disable(struct dyn_ftrace *rec)
591{ 461{
592 unsigned long ip; 462 unsigned long ip;
593 unsigned char *nop, *call; 463 unsigned char *nop, *call;
594 int failed; 464 int ret;
595 465
596 ip = rec->ip; 466 ip = rec->ip;
597 467
598 nop = ftrace_nop_replace(); 468 nop = ftrace_nop_replace();
599 call = ftrace_call_replace(ip, mcount_addr); 469 call = ftrace_call_replace(ip, mcount_addr);
600 470
601 failed = ftrace_modify_code(ip, call, nop); 471 ret = ftrace_modify_code(ip, call, nop);
602 if (failed) { 472 if (ret) {
603 switch (failed) { 473 switch (ret) {
604 case 1: 474 case -EFAULT:
605 WARN_ON_ONCE(1); 475 FTRACE_WARN_ON_ONCE(1);
606 pr_info("ftrace faulted on modifying "); 476 pr_info("ftrace faulted on modifying ");
607 print_ip_sym(ip); 477 print_ip_sym(ip);
608 break; 478 break;
609 case 2: 479 case -EINVAL:
610 WARN_ON_ONCE(1); 480 FTRACE_WARN_ON_ONCE(1);
611 pr_info("ftrace failed to modify "); 481 pr_info("ftrace failed to modify ");
612 print_ip_sym(ip); 482 print_ip_sym(ip);
613 print_ip_ins(" expected: ", call); 483 print_ip_ins(" expected: ", call);
@@ -615,6 +485,15 @@ ftrace_code_disable(struct dyn_ftrace *rec)
615 print_ip_ins(" replace: ", nop); 485 print_ip_ins(" replace: ", nop);
616 printk(KERN_CONT "\n"); 486 printk(KERN_CONT "\n");
617 break; 487 break;
488 case -EPERM:
489 FTRACE_WARN_ON_ONCE(1);
490 pr_info("ftrace faulted on writing ");
491 print_ip_sym(ip);
492 break;
493 default:
494 FTRACE_WARN_ON_ONCE(1);
495 pr_info("ftrace faulted on unknown error ");
496 print_ip_sym(ip);
618 } 497 }
619 498
620 rec->flags |= FTRACE_FL_FAILED; 499 rec->flags |= FTRACE_FL_FAILED;
@@ -623,19 +502,11 @@ ftrace_code_disable(struct dyn_ftrace *rec)
623 return 1; 502 return 1;
624} 503}
625 504
626static int __ftrace_update_code(void *ignore);
627
628static int __ftrace_modify_code(void *data) 505static int __ftrace_modify_code(void *data)
629{ 506{
630 unsigned long addr;
631 int *command = data; 507 int *command = data;
632 508
633 if (*command & FTRACE_ENABLE_CALLS) { 509 if (*command & FTRACE_ENABLE_CALLS) {
634 /*
635 * Update any recorded ips now that we have the
636 * machine stopped
637 */
638 __ftrace_update_code(NULL);
639 ftrace_replace_code(1); 510 ftrace_replace_code(1);
640 tracing_on = 1; 511 tracing_on = 1;
641 } else if (*command & FTRACE_DISABLE_CALLS) { 512 } else if (*command & FTRACE_DISABLE_CALLS) {
@@ -646,14 +517,6 @@ static int __ftrace_modify_code(void *data)
646 if (*command & FTRACE_UPDATE_TRACE_FUNC) 517 if (*command & FTRACE_UPDATE_TRACE_FUNC)
647 ftrace_update_ftrace_func(ftrace_trace_function); 518 ftrace_update_ftrace_func(ftrace_trace_function);
648 519
649 if (*command & FTRACE_ENABLE_MCOUNT) {
650 addr = (unsigned long)ftrace_record_ip;
651 ftrace_mcount_set(&addr);
652 } else if (*command & FTRACE_DISABLE_MCOUNT) {
653 addr = (unsigned long)ftrace_stub;
654 ftrace_mcount_set(&addr);
655 }
656
657 return 0; 520 return 0;
658} 521}
659 522
@@ -662,26 +525,9 @@ static void ftrace_run_update_code(int command)
662 stop_machine(__ftrace_modify_code, &command, NULL); 525 stop_machine(__ftrace_modify_code, &command, NULL);
663} 526}
664 527
665void ftrace_disable_daemon(void)
666{
667 /* Stop the daemon from calling kstop_machine */
668 mutex_lock(&ftraced_lock);
669 ftraced_stop = 1;
670 mutex_unlock(&ftraced_lock);
671
672 ftrace_force_update();
673}
674
675void ftrace_enable_daemon(void)
676{
677 mutex_lock(&ftraced_lock);
678 ftraced_stop = 0;
679 mutex_unlock(&ftraced_lock);
680
681 ftrace_force_update();
682}
683
684static ftrace_func_t saved_ftrace_func; 528static ftrace_func_t saved_ftrace_func;
529static int ftrace_start;
530static DEFINE_MUTEX(ftrace_start_lock);
685 531
686static void ftrace_startup(void) 532static void ftrace_startup(void)
687{ 533{
@@ -690,9 +536,9 @@ static void ftrace_startup(void)
690 if (unlikely(ftrace_disabled)) 536 if (unlikely(ftrace_disabled))
691 return; 537 return;
692 538
693 mutex_lock(&ftraced_lock); 539 mutex_lock(&ftrace_start_lock);
694 ftraced_suspend++; 540 ftrace_start++;
695 if (ftraced_suspend == 1) 541 if (ftrace_start == 1)
696 command |= FTRACE_ENABLE_CALLS; 542 command |= FTRACE_ENABLE_CALLS;
697 543
698 if (saved_ftrace_func != ftrace_trace_function) { 544 if (saved_ftrace_func != ftrace_trace_function) {
@@ -705,7 +551,7 @@ static void ftrace_startup(void)
705 551
706 ftrace_run_update_code(command); 552 ftrace_run_update_code(command);
707 out: 553 out:
708 mutex_unlock(&ftraced_lock); 554 mutex_unlock(&ftrace_start_lock);
709} 555}
710 556
711static void ftrace_shutdown(void) 557static void ftrace_shutdown(void)
@@ -715,9 +561,9 @@ static void ftrace_shutdown(void)
715 if (unlikely(ftrace_disabled)) 561 if (unlikely(ftrace_disabled))
716 return; 562 return;
717 563
718 mutex_lock(&ftraced_lock); 564 mutex_lock(&ftrace_start_lock);
719 ftraced_suspend--; 565 ftrace_start--;
720 if (!ftraced_suspend) 566 if (!ftrace_start)
721 command |= FTRACE_DISABLE_CALLS; 567 command |= FTRACE_DISABLE_CALLS;
722 568
723 if (saved_ftrace_func != ftrace_trace_function) { 569 if (saved_ftrace_func != ftrace_trace_function) {
@@ -730,7 +576,7 @@ static void ftrace_shutdown(void)
730 576
731 ftrace_run_update_code(command); 577 ftrace_run_update_code(command);
732 out: 578 out:
733 mutex_unlock(&ftraced_lock); 579 mutex_unlock(&ftrace_start_lock);
734} 580}
735 581
736static void ftrace_startup_sysctl(void) 582static void ftrace_startup_sysctl(void)
@@ -740,15 +586,15 @@ static void ftrace_startup_sysctl(void)
740 if (unlikely(ftrace_disabled)) 586 if (unlikely(ftrace_disabled))
741 return; 587 return;
742 588
743 mutex_lock(&ftraced_lock); 589 mutex_lock(&ftrace_start_lock);
744 /* Force update next time */ 590 /* Force update next time */
745 saved_ftrace_func = NULL; 591 saved_ftrace_func = NULL;
746 /* ftraced_suspend is true if we want ftrace running */ 592 /* ftrace_start is true if we want ftrace running */
747 if (ftraced_suspend) 593 if (ftrace_start)
748 command |= FTRACE_ENABLE_CALLS; 594 command |= FTRACE_ENABLE_CALLS;
749 595
750 ftrace_run_update_code(command); 596 ftrace_run_update_code(command);
751 mutex_unlock(&ftraced_lock); 597 mutex_unlock(&ftrace_start_lock);
752} 598}
753 599
754static void ftrace_shutdown_sysctl(void) 600static void ftrace_shutdown_sysctl(void)
@@ -758,112 +604,50 @@ static void ftrace_shutdown_sysctl(void)
758 if (unlikely(ftrace_disabled)) 604 if (unlikely(ftrace_disabled))
759 return; 605 return;
760 606
761 mutex_lock(&ftraced_lock); 607 mutex_lock(&ftrace_start_lock);
762 /* ftraced_suspend is true if ftrace is running */ 608 /* ftrace_start is true if ftrace is running */
763 if (ftraced_suspend) 609 if (ftrace_start)
764 command |= FTRACE_DISABLE_CALLS; 610 command |= FTRACE_DISABLE_CALLS;
765 611
766 ftrace_run_update_code(command); 612 ftrace_run_update_code(command);
767 mutex_unlock(&ftraced_lock); 613 mutex_unlock(&ftrace_start_lock);
768} 614}
769 615
770static cycle_t ftrace_update_time; 616static cycle_t ftrace_update_time;
771static unsigned long ftrace_update_cnt; 617static unsigned long ftrace_update_cnt;
772unsigned long ftrace_update_tot_cnt; 618unsigned long ftrace_update_tot_cnt;
773 619
774static int __ftrace_update_code(void *ignore) 620static int ftrace_update_code(void)
775{ 621{
776 int i, save_ftrace_enabled; 622 struct dyn_ftrace *p, *t;
777 cycle_t start, stop; 623 cycle_t start, stop;
778 struct dyn_ftrace *p;
779 struct hlist_node *t, *n;
780 struct hlist_head *head, temp_list;
781
782 /* Don't be recording funcs now */
783 ftrace_record_suspend++;
784 save_ftrace_enabled = ftrace_enabled;
785 ftrace_enabled = 0;
786 624
787 start = ftrace_now(raw_smp_processor_id()); 625 start = ftrace_now(raw_smp_processor_id());
788 ftrace_update_cnt = 0; 626 ftrace_update_cnt = 0;
789 627
790 /* No locks needed, the machine is stopped! */ 628 list_for_each_entry_safe(p, t, &ftrace_new_addrs, list) {
791 for (i = 0; i < FTRACE_HASHSIZE; i++) {
792 INIT_HLIST_HEAD(&temp_list);
793 head = &ftrace_hash[i];
794
795 /* all CPUS are stopped, we are safe to modify code */
796 hlist_for_each_entry_safe(p, t, n, head, node) {
797 /* Skip over failed records which have not been
798 * freed. */
799 if (p->flags & FTRACE_FL_FAILED)
800 continue;
801
802 /* Unconverted records are always at the head of the
803 * hash bucket. Once we encounter a converted record,
804 * simply skip over to the next bucket. Saves ftraced
805 * some processor cycles (ftrace does its bid for
806 * global warming :-p ). */
807 if (p->flags & (FTRACE_FL_CONVERTED))
808 break;
809 629
810 /* Ignore updates to this record's mcount site. 630 /* If something went wrong, bail without enabling anything */
811 * Reintroduce this record at the head of this 631 if (unlikely(ftrace_disabled))
812 * bucket to attempt to "convert" it again if 632 return -1;
813 * the kprobe on it is unregistered before the
814 * next run. */
815 if (get_kprobe((void *)p->ip)) {
816 ftrace_del_hash(p);
817 INIT_HLIST_NODE(&p->node);
818 hlist_add_head(&p->node, &temp_list);
819 freeze_record(p);
820 continue;
821 } else {
822 unfreeze_record(p);
823 }
824 633
825 /* convert record (i.e, patch mcount-call with NOP) */ 634 list_del_init(&p->list);
826 if (ftrace_code_disable(p)) {
827 p->flags |= FTRACE_FL_CONVERTED;
828 ftrace_update_cnt++;
829 } else {
830 if ((system_state == SYSTEM_BOOTING) ||
831 !core_kernel_text(p->ip)) {
832 ftrace_del_hash(p);
833 ftrace_free_rec(p);
834 }
835 }
836 }
837 635
838 hlist_for_each_entry_safe(p, t, n, &temp_list, node) { 636 /* convert record (i.e, patch mcount-call with NOP) */
839 hlist_del(&p->node); 637 if (ftrace_code_disable(p)) {
840 INIT_HLIST_NODE(&p->node); 638 p->flags |= FTRACE_FL_CONVERTED;
841 hlist_add_head(&p->node, head); 639 ftrace_update_cnt++;
842 } 640 } else
641 ftrace_free_rec(p);
843 } 642 }
844 643
845 stop = ftrace_now(raw_smp_processor_id()); 644 stop = ftrace_now(raw_smp_processor_id());
846 ftrace_update_time = stop - start; 645 ftrace_update_time = stop - start;
847 ftrace_update_tot_cnt += ftrace_update_cnt; 646 ftrace_update_tot_cnt += ftrace_update_cnt;
848 ftraced_trigger = 0;
849
850 ftrace_enabled = save_ftrace_enabled;
851 ftrace_record_suspend--;
852 647
853 return 0; 648 return 0;
854} 649}
855 650
856static int ftrace_update_code(void)
857{
858 if (unlikely(ftrace_disabled) ||
859 !ftrace_enabled || !ftraced_trigger)
860 return 0;
861
862 stop_machine(__ftrace_update_code, NULL, NULL);
863
864 return 1;
865}
866
867static int __init ftrace_dyn_table_alloc(unsigned long num_to_init) 651static int __init ftrace_dyn_table_alloc(unsigned long num_to_init)
868{ 652{
869 struct ftrace_page *pg; 653 struct ftrace_page *pg;
@@ -892,7 +676,7 @@ static int __init ftrace_dyn_table_alloc(unsigned long num_to_init)
892 pg = ftrace_pages = ftrace_pages_start; 676 pg = ftrace_pages = ftrace_pages_start;
893 677
894 cnt = num_to_init / ENTRIES_PER_PAGE; 678 cnt = num_to_init / ENTRIES_PER_PAGE;
895 pr_info("ftrace: allocating %ld hash entries in %d pages\n", 679 pr_info("ftrace: allocating %ld entries in %d pages\n",
896 num_to_init, cnt); 680 num_to_init, cnt);
897 681
898 for (i = 0; i < cnt; i++) { 682 for (i = 0; i < cnt; i++) {
@@ -1401,10 +1185,10 @@ ftrace_regex_release(struct inode *inode, struct file *file, int enable)
1401 } 1185 }
1402 1186
1403 mutex_lock(&ftrace_sysctl_lock); 1187 mutex_lock(&ftrace_sysctl_lock);
1404 mutex_lock(&ftraced_lock); 1188 mutex_lock(&ftrace_start_lock);
1405 if (iter->filtered && ftraced_suspend && ftrace_enabled) 1189 if (iter->filtered && ftrace_start && ftrace_enabled)
1406 ftrace_run_update_code(FTRACE_ENABLE_CALLS); 1190 ftrace_run_update_code(FTRACE_ENABLE_CALLS);
1407 mutex_unlock(&ftraced_lock); 1191 mutex_unlock(&ftrace_start_lock);
1408 mutex_unlock(&ftrace_sysctl_lock); 1192 mutex_unlock(&ftrace_sysctl_lock);
1409 1193
1410 kfree(iter); 1194 kfree(iter);
@@ -1424,55 +1208,6 @@ ftrace_notrace_release(struct inode *inode, struct file *file)
1424 return ftrace_regex_release(inode, file, 0); 1208 return ftrace_regex_release(inode, file, 0);
1425} 1209}
1426 1210
1427static ssize_t
1428ftraced_read(struct file *filp, char __user *ubuf,
1429 size_t cnt, loff_t *ppos)
1430{
1431 /* don't worry about races */
1432 char *buf = ftraced_stop ? "disabled\n" : "enabled\n";
1433 int r = strlen(buf);
1434
1435 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
1436}
1437
1438static ssize_t
1439ftraced_write(struct file *filp, const char __user *ubuf,
1440 size_t cnt, loff_t *ppos)
1441{
1442 char buf[64];
1443 long val;
1444 int ret;
1445
1446 if (cnt >= sizeof(buf))
1447 return -EINVAL;
1448
1449 if (copy_from_user(&buf, ubuf, cnt))
1450 return -EFAULT;
1451
1452 if (strncmp(buf, "enable", 6) == 0)
1453 val = 1;
1454 else if (strncmp(buf, "disable", 7) == 0)
1455 val = 0;
1456 else {
1457 buf[cnt] = 0;
1458
1459 ret = strict_strtoul(buf, 10, &val);
1460 if (ret < 0)
1461 return ret;
1462
1463 val = !!val;
1464 }
1465
1466 if (val)
1467 ftrace_enable_daemon();
1468 else
1469 ftrace_disable_daemon();
1470
1471 filp->f_pos += cnt;
1472
1473 return cnt;
1474}
1475
1476static struct file_operations ftrace_avail_fops = { 1211static struct file_operations ftrace_avail_fops = {
1477 .open = ftrace_avail_open, 1212 .open = ftrace_avail_open,
1478 .read = seq_read, 1213 .read = seq_read,
@@ -1503,54 +1238,6 @@ static struct file_operations ftrace_notrace_fops = {
1503 .release = ftrace_notrace_release, 1238 .release = ftrace_notrace_release,
1504}; 1239};
1505 1240
1506static struct file_operations ftraced_fops = {
1507 .open = tracing_open_generic,
1508 .read = ftraced_read,
1509 .write = ftraced_write,
1510};
1511
1512/**
1513 * ftrace_force_update - force an update to all recording ftrace functions
1514 */
1515int ftrace_force_update(void)
1516{
1517 int ret = 0;
1518
1519 if (unlikely(ftrace_disabled))
1520 return -ENODEV;
1521
1522 mutex_lock(&ftrace_sysctl_lock);
1523 mutex_lock(&ftraced_lock);
1524
1525 /*
1526 * If ftraced_trigger is not set, then there is nothing
1527 * to update.
1528 */
1529 if (ftraced_trigger && !ftrace_update_code())
1530 ret = -EBUSY;
1531
1532 mutex_unlock(&ftraced_lock);
1533 mutex_unlock(&ftrace_sysctl_lock);
1534
1535 return ret;
1536}
1537
1538static void ftrace_force_shutdown(void)
1539{
1540 struct task_struct *task;
1541 int command = FTRACE_DISABLE_CALLS | FTRACE_UPDATE_TRACE_FUNC;
1542
1543 mutex_lock(&ftraced_lock);
1544 task = ftraced_task;
1545 ftraced_task = NULL;
1546 ftraced_suspend = -1;
1547 ftrace_run_update_code(command);
1548 mutex_unlock(&ftraced_lock);
1549
1550 if (task)
1551 kthread_stop(task);
1552}
1553
1554static __init int ftrace_init_debugfs(void) 1241static __init int ftrace_init_debugfs(void)
1555{ 1242{
1556 struct dentry *d_tracer; 1243 struct dentry *d_tracer;
@@ -1581,17 +1268,11 @@ static __init int ftrace_init_debugfs(void)
1581 pr_warning("Could not create debugfs " 1268 pr_warning("Could not create debugfs "
1582 "'set_ftrace_notrace' entry\n"); 1269 "'set_ftrace_notrace' entry\n");
1583 1270
1584 entry = debugfs_create_file("ftraced_enabled", 0644, d_tracer,
1585 NULL, &ftraced_fops);
1586 if (!entry)
1587 pr_warning("Could not create debugfs "
1588 "'ftraced_enabled' entry\n");
1589 return 0; 1271 return 0;
1590} 1272}
1591 1273
1592fs_initcall(ftrace_init_debugfs); 1274fs_initcall(ftrace_init_debugfs);
1593 1275
1594#ifdef CONFIG_FTRACE_MCOUNT_RECORD
1595static int ftrace_convert_nops(unsigned long *start, 1276static int ftrace_convert_nops(unsigned long *start,
1596 unsigned long *end) 1277 unsigned long *end)
1597{ 1278{
@@ -1599,20 +1280,18 @@ static int ftrace_convert_nops(unsigned long *start,
1599 unsigned long addr; 1280 unsigned long addr;
1600 unsigned long flags; 1281 unsigned long flags;
1601 1282
1283 mutex_lock(&ftrace_start_lock);
1602 p = start; 1284 p = start;
1603 while (p < end) { 1285 while (p < end) {
1604 addr = ftrace_call_adjust(*p++); 1286 addr = ftrace_call_adjust(*p++);
1605 /* should not be called from interrupt context */
1606 spin_lock(&ftrace_lock);
1607 ftrace_record_ip(addr); 1287 ftrace_record_ip(addr);
1608 spin_unlock(&ftrace_lock);
1609 ftrace_shutdown_replenish();
1610 } 1288 }
1611 1289
1612 /* p is ignored */ 1290 /* disable interrupts to prevent kstop machine */
1613 local_irq_save(flags); 1291 local_irq_save(flags);
1614 __ftrace_update_code(p); 1292 ftrace_update_code();
1615 local_irq_restore(flags); 1293 local_irq_restore(flags);
1294 mutex_unlock(&ftrace_start_lock);
1616 1295
1617 return 0; 1296 return 0;
1618} 1297}
@@ -1658,130 +1337,26 @@ void __init ftrace_init(void)
1658 failed: 1337 failed:
1659 ftrace_disabled = 1; 1338 ftrace_disabled = 1;
1660} 1339}
1661#else /* CONFIG_FTRACE_MCOUNT_RECORD */
1662static int ftraced(void *ignore)
1663{
1664 unsigned long usecs;
1665
1666 while (!kthread_should_stop()) {
1667
1668 set_current_state(TASK_INTERRUPTIBLE);
1669
1670 /* check once a second */
1671 schedule_timeout(HZ);
1672
1673 if (unlikely(ftrace_disabled))
1674 continue;
1675
1676 mutex_lock(&ftrace_sysctl_lock);
1677 mutex_lock(&ftraced_lock);
1678 if (!ftraced_suspend && !ftraced_stop &&
1679 ftrace_update_code()) {
1680 usecs = nsecs_to_usecs(ftrace_update_time);
1681 if (ftrace_update_tot_cnt > 100000) {
1682 ftrace_update_tot_cnt = 0;
1683 pr_info("hm, dftrace overflow: %lu change%s"
1684 " (%lu total) in %lu usec%s\n",
1685 ftrace_update_cnt,
1686 ftrace_update_cnt != 1 ? "s" : "",
1687 ftrace_update_tot_cnt,
1688 usecs, usecs != 1 ? "s" : "");
1689 ftrace_disabled = 1;
1690 WARN_ON_ONCE(1);
1691 }
1692 }
1693 mutex_unlock(&ftraced_lock);
1694 mutex_unlock(&ftrace_sysctl_lock);
1695
1696 ftrace_shutdown_replenish();
1697 }
1698 __set_current_state(TASK_RUNNING);
1699 return 0;
1700}
1701
1702static int __init ftrace_dynamic_init(void)
1703{
1704 struct task_struct *p;
1705 unsigned long addr;
1706 int ret;
1707
1708 addr = (unsigned long)ftrace_record_ip;
1709
1710 stop_machine(ftrace_dyn_arch_init, &addr, NULL);
1711
1712 /* ftrace_dyn_arch_init places the return code in addr */
1713 if (addr) {
1714 ret = (int)addr;
1715 goto failed;
1716 }
1717
1718 ret = ftrace_dyn_table_alloc(NR_TO_INIT);
1719 if (ret)
1720 goto failed;
1721
1722 p = kthread_run(ftraced, NULL, "ftraced");
1723 if (IS_ERR(p)) {
1724 ret = -1;
1725 goto failed;
1726 }
1727
1728 last_ftrace_enabled = ftrace_enabled = 1;
1729 ftraced_task = p;
1730
1731 return 0;
1732
1733 failed:
1734 ftrace_disabled = 1;
1735 return ret;
1736}
1737
1738core_initcall(ftrace_dynamic_init);
1739#endif /* CONFIG_FTRACE_MCOUNT_RECORD */
1740 1340
1741#else 1341#else
1742# define ftrace_startup() do { } while (0) 1342# define ftrace_startup() do { } while (0)
1743# define ftrace_shutdown() do { } while (0) 1343# define ftrace_shutdown() do { } while (0)
1744# define ftrace_startup_sysctl() do { } while (0) 1344# define ftrace_startup_sysctl() do { } while (0)
1745# define ftrace_shutdown_sysctl() do { } while (0) 1345# define ftrace_shutdown_sysctl() do { } while (0)
1746# define ftrace_force_shutdown() do { } while (0)
1747#endif /* CONFIG_DYNAMIC_FTRACE */ 1346#endif /* CONFIG_DYNAMIC_FTRACE */
1748 1347
1749/** 1348/**
1750 * ftrace_kill_atomic - kill ftrace from critical sections 1349 * ftrace_kill - kill ftrace
1751 * 1350 *
1752 * This function should be used by panic code. It stops ftrace 1351 * This function should be used by panic code. It stops ftrace
1753 * but in a not so nice way. If you need to simply kill ftrace 1352 * but in a not so nice way. If you need to simply kill ftrace
1754 * from a non-atomic section, use ftrace_kill. 1353 * from a non-atomic section, use ftrace_kill.
1755 */ 1354 */
1756void ftrace_kill_atomic(void)
1757{
1758 ftrace_disabled = 1;
1759 ftrace_enabled = 0;
1760#ifdef CONFIG_DYNAMIC_FTRACE
1761 ftraced_suspend = -1;
1762#endif
1763 clear_ftrace_function();
1764}
1765
1766/**
1767 * ftrace_kill - totally shutdown ftrace
1768 *
1769 * This is a safety measure. If something was detected that seems
1770 * wrong, calling this function will keep ftrace from doing
1771 * any more modifications, and updates.
1772 * used when something went wrong.
1773 */
1774void ftrace_kill(void) 1355void ftrace_kill(void)
1775{ 1356{
1776 mutex_lock(&ftrace_sysctl_lock);
1777 ftrace_disabled = 1; 1357 ftrace_disabled = 1;
1778 ftrace_enabled = 0; 1358 ftrace_enabled = 0;
1779
1780 clear_ftrace_function(); 1359 clear_ftrace_function();
1781 mutex_unlock(&ftrace_sysctl_lock);
1782
1783 /* Try to totally disable ftrace */
1784 ftrace_force_shutdown();
1785} 1360}
1786 1361
1787/** 1362/**
@@ -1870,3 +1445,4 @@ ftrace_enable_sysctl(struct ctl_table *table, int write,
1870 mutex_unlock(&ftrace_sysctl_lock); 1445 mutex_unlock(&ftrace_sysctl_lock);
1871 return ret; 1446 return ret;
1872} 1447}
1448