aboutsummaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/kprobes.c68
-rw-r--r--kernel/notifier.c2
-rw-r--r--kernel/trace/Kconfig12
-rw-r--r--kernel/trace/Makefile1
-rw-r--r--kernel/trace/trace.h23
-rw-r--r--kernel/trace/trace_event_profile.c4
-rw-r--r--kernel/trace/trace_events.c125
-rw-r--r--kernel/trace/trace_export.c14
-rw-r--r--kernel/trace/trace_kprobe.c1389
-rw-r--r--kernel/trace/trace_syscalls.c16
10 files changed, 1580 insertions, 74 deletions
diff --git a/kernel/kprobes.c b/kernel/kprobes.c
index cfadc1291d0b..b466afa4e148 100644
--- a/kernel/kprobes.c
+++ b/kernel/kprobes.c
@@ -90,6 +90,9 @@ static spinlock_t *kretprobe_table_lock_ptr(unsigned long hash)
90 */ 90 */
91static struct kprobe_blackpoint kprobe_blacklist[] = { 91static struct kprobe_blackpoint kprobe_blacklist[] = {
92 {"preempt_schedule",}, 92 {"preempt_schedule",},
93 {"native_get_debugreg",},
94 {"irq_entries_start",},
95 {"common_interrupt",},
93 {NULL} /* Terminator */ 96 {NULL} /* Terminator */
94}; 97};
95 98
@@ -673,6 +676,40 @@ static kprobe_opcode_t __kprobes *kprobe_addr(struct kprobe *p)
673 return (kprobe_opcode_t *)(((char *)addr) + p->offset); 676 return (kprobe_opcode_t *)(((char *)addr) + p->offset);
674} 677}
675 678
679/* Check passed kprobe is valid and return kprobe in kprobe_table. */
680static struct kprobe * __kprobes __get_valid_kprobe(struct kprobe *p)
681{
682 struct kprobe *old_p, *list_p;
683
684 old_p = get_kprobe(p->addr);
685 if (unlikely(!old_p))
686 return NULL;
687
688 if (p != old_p) {
689 list_for_each_entry_rcu(list_p, &old_p->list, list)
690 if (list_p == p)
691 /* kprobe p is a valid probe */
692 goto valid;
693 return NULL;
694 }
695valid:
696 return old_p;
697}
698
699/* Return error if the kprobe is being re-registered */
700static inline int check_kprobe_rereg(struct kprobe *p)
701{
702 int ret = 0;
703 struct kprobe *old_p;
704
705 mutex_lock(&kprobe_mutex);
706 old_p = __get_valid_kprobe(p);
707 if (old_p)
708 ret = -EINVAL;
709 mutex_unlock(&kprobe_mutex);
710 return ret;
711}
712
676int __kprobes register_kprobe(struct kprobe *p) 713int __kprobes register_kprobe(struct kprobe *p)
677{ 714{
678 int ret = 0; 715 int ret = 0;
@@ -685,6 +722,10 @@ int __kprobes register_kprobe(struct kprobe *p)
685 return -EINVAL; 722 return -EINVAL;
686 p->addr = addr; 723 p->addr = addr;
687 724
725 ret = check_kprobe_rereg(p);
726 if (ret)
727 return ret;
728
688 preempt_disable(); 729 preempt_disable();
689 if (!kernel_text_address((unsigned long) p->addr) || 730 if (!kernel_text_address((unsigned long) p->addr) ||
690 in_kprobes_functions((unsigned long) p->addr)) { 731 in_kprobes_functions((unsigned long) p->addr)) {
@@ -754,26 +795,6 @@ out:
754} 795}
755EXPORT_SYMBOL_GPL(register_kprobe); 796EXPORT_SYMBOL_GPL(register_kprobe);
756 797
757/* Check passed kprobe is valid and return kprobe in kprobe_table. */
758static struct kprobe * __kprobes __get_valid_kprobe(struct kprobe *p)
759{
760 struct kprobe *old_p, *list_p;
761
762 old_p = get_kprobe(p->addr);
763 if (unlikely(!old_p))
764 return NULL;
765
766 if (p != old_p) {
767 list_for_each_entry_rcu(list_p, &old_p->list, list)
768 if (list_p == p)
769 /* kprobe p is a valid probe */
770 goto valid;
771 return NULL;
772 }
773valid:
774 return old_p;
775}
776
777/* 798/*
778 * Unregister a kprobe without a scheduler synchronization. 799 * Unregister a kprobe without a scheduler synchronization.
779 */ 800 */
@@ -1141,6 +1162,13 @@ static void __kprobes kill_kprobe(struct kprobe *p)
1141 arch_remove_kprobe(p); 1162 arch_remove_kprobe(p);
1142} 1163}
1143 1164
1165void __kprobes dump_kprobe(struct kprobe *kp)
1166{
1167 printk(KERN_WARNING "Dumping kprobe:\n");
1168 printk(KERN_WARNING "Name: %s\nAddress: %p\nOffset: %x\n",
1169 kp->symbol_name, kp->addr, kp->offset);
1170}
1171
1144/* Module notifier call back, checking kprobes on the module */ 1172/* Module notifier call back, checking kprobes on the module */
1145static int __kprobes kprobes_module_callback(struct notifier_block *nb, 1173static int __kprobes kprobes_module_callback(struct notifier_block *nb,
1146 unsigned long val, void *data) 1174 unsigned long val, void *data)
diff --git a/kernel/notifier.c b/kernel/notifier.c
index 61d5aa5eced3..acd24e7643eb 100644
--- a/kernel/notifier.c
+++ b/kernel/notifier.c
@@ -558,7 +558,7 @@ EXPORT_SYMBOL(unregister_reboot_notifier);
558 558
559static ATOMIC_NOTIFIER_HEAD(die_chain); 559static ATOMIC_NOTIFIER_HEAD(die_chain);
560 560
561int notrace notify_die(enum die_val val, const char *str, 561int notrace __kprobes notify_die(enum die_val val, const char *str,
562 struct pt_regs *regs, long err, int trap, int sig) 562 struct pt_regs *regs, long err, int trap, int sig)
563{ 563{
564 struct die_args args = { 564 struct die_args args = {
diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig
index b416512ad17f..15372a9f2399 100644
--- a/kernel/trace/Kconfig
+++ b/kernel/trace/Kconfig
@@ -428,6 +428,18 @@ config BLK_DEV_IO_TRACE
428 428
429 If unsure, say N. 429 If unsure, say N.
430 430
431config KPROBE_TRACER
432 depends on KPROBES
433 depends on X86
434 bool "Trace kprobes"
435 select TRACING
436 select GENERIC_TRACER
437 help
438 This tracer probes everywhere where kprobes can probe it, and
439 records various registers and memories specified by user.
440 This also allows you to trace kprobe probe points as a dynamic
441 defined events. It provides per-probe event filtering interface.
442
431config DYNAMIC_FTRACE 443config DYNAMIC_FTRACE
432 bool "enable/disable ftrace tracepoints dynamically" 444 bool "enable/disable ftrace tracepoints dynamically"
433 depends on FUNCTION_TRACER 445 depends on FUNCTION_TRACER
diff --git a/kernel/trace/Makefile b/kernel/trace/Makefile
index 26f03ac07c2b..c8cb75d7f280 100644
--- a/kernel/trace/Makefile
+++ b/kernel/trace/Makefile
@@ -53,6 +53,7 @@ obj-$(CONFIG_EVENT_TRACING) += trace_export.o
53obj-$(CONFIG_FTRACE_SYSCALLS) += trace_syscalls.o 53obj-$(CONFIG_FTRACE_SYSCALLS) += trace_syscalls.o
54obj-$(CONFIG_EVENT_PROFILE) += trace_event_profile.o 54obj-$(CONFIG_EVENT_PROFILE) += trace_event_profile.o
55obj-$(CONFIG_EVENT_TRACING) += trace_events_filter.o 55obj-$(CONFIG_EVENT_TRACING) += trace_events_filter.o
56obj-$(CONFIG_KPROBE_TRACER) += trace_kprobe.o
56obj-$(CONFIG_EVENT_TRACING) += power-traces.o 57obj-$(CONFIG_EVENT_TRACING) += power-traces.o
57 58
58libftrace-y := ftrace.o 59libftrace-y := ftrace.o
diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h
index 405cb850b75d..104c1a72418f 100644
--- a/kernel/trace/trace.h
+++ b/kernel/trace/trace.h
@@ -101,6 +101,29 @@ struct syscall_trace_exit {
101 unsigned long ret; 101 unsigned long ret;
102}; 102};
103 103
104struct kprobe_trace_entry {
105 struct trace_entry ent;
106 unsigned long ip;
107 int nargs;
108 unsigned long args[];
109};
110
111#define SIZEOF_KPROBE_TRACE_ENTRY(n) \
112 (offsetof(struct kprobe_trace_entry, args) + \
113 (sizeof(unsigned long) * (n)))
114
115struct kretprobe_trace_entry {
116 struct trace_entry ent;
117 unsigned long func;
118 unsigned long ret_ip;
119 int nargs;
120 unsigned long args[];
121};
122
123#define SIZEOF_KRETPROBE_TRACE_ENTRY(n) \
124 (offsetof(struct kretprobe_trace_entry, args) + \
125 (sizeof(unsigned long) * (n)))
126
104/* 127/*
105 * trace_flag_type is an enumeration that holds different 128 * trace_flag_type is an enumeration that holds different
106 * states when a trace occurs. These are: 129 * states when a trace occurs. These are:
diff --git a/kernel/trace/trace_event_profile.c b/kernel/trace/trace_event_profile.c
index dd44b8768867..e812f1c1264c 100644
--- a/kernel/trace/trace_event_profile.c
+++ b/kernel/trace/trace_event_profile.c
@@ -45,7 +45,7 @@ static int ftrace_profile_enable_event(struct ftrace_event_call *event)
45 rcu_assign_pointer(trace_profile_buf_nmi, buf); 45 rcu_assign_pointer(trace_profile_buf_nmi, buf);
46 } 46 }
47 47
48 ret = event->profile_enable(); 48 ret = event->profile_enable(event);
49 if (!ret) 49 if (!ret)
50 return 0; 50 return 0;
51 51
@@ -84,7 +84,7 @@ static void ftrace_profile_disable_event(struct ftrace_event_call *event)
84 if (!atomic_add_negative(-1, &event->profile_count)) 84 if (!atomic_add_negative(-1, &event->profile_count))
85 return; 85 return;
86 86
87 event->profile_disable(); 87 event->profile_disable(event);
88 88
89 if (!--total_profile_count) { 89 if (!--total_profile_count) {
90 buf = trace_profile_buf; 90 buf = trace_profile_buf;
diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c
index 6f03c8a1105e..a4b7c9a9130c 100644
--- a/kernel/trace/trace_events.c
+++ b/kernel/trace/trace_events.c
@@ -93,9 +93,7 @@ int trace_define_common_fields(struct ftrace_event_call *call)
93} 93}
94EXPORT_SYMBOL_GPL(trace_define_common_fields); 94EXPORT_SYMBOL_GPL(trace_define_common_fields);
95 95
96#ifdef CONFIG_MODULES 96void trace_destroy_fields(struct ftrace_event_call *call)
97
98static void trace_destroy_fields(struct ftrace_event_call *call)
99{ 97{
100 struct ftrace_event_field *field, *next; 98 struct ftrace_event_field *field, *next;
101 99
@@ -107,8 +105,6 @@ static void trace_destroy_fields(struct ftrace_event_call *call)
107 } 105 }
108} 106}
109 107
110#endif /* CONFIG_MODULES */
111
112static void ftrace_event_enable_disable(struct ftrace_event_call *call, 108static void ftrace_event_enable_disable(struct ftrace_event_call *call,
113 int enable) 109 int enable)
114{ 110{
@@ -117,14 +113,14 @@ static void ftrace_event_enable_disable(struct ftrace_event_call *call,
117 if (call->enabled) { 113 if (call->enabled) {
118 call->enabled = 0; 114 call->enabled = 0;
119 tracing_stop_cmdline_record(); 115 tracing_stop_cmdline_record();
120 call->unregfunc(call->data); 116 call->unregfunc(call);
121 } 117 }
122 break; 118 break;
123 case 1: 119 case 1:
124 if (!call->enabled) { 120 if (!call->enabled) {
125 call->enabled = 1; 121 call->enabled = 1;
126 tracing_start_cmdline_record(); 122 tracing_start_cmdline_record();
127 call->regfunc(call->data); 123 call->regfunc(call);
128 } 124 }
129 break; 125 break;
130 } 126 }
@@ -939,27 +935,46 @@ event_create_dir(struct ftrace_event_call *call, struct dentry *d_events,
939 return 0; 935 return 0;
940} 936}
941 937
942#define for_each_event(event, start, end) \ 938static int __trace_add_event_call(struct ftrace_event_call *call)
943 for (event = start; \ 939{
944 (unsigned long)event < (unsigned long)end; \ 940 struct dentry *d_events;
945 event++) 941 int ret;
946 942
947#ifdef CONFIG_MODULES 943 if (!call->name)
944 return -EINVAL;
948 945
949static LIST_HEAD(ftrace_module_file_list); 946 if (call->raw_init) {
947 ret = call->raw_init(call);
948 if (ret < 0) {
949 if (ret != -ENOSYS)
950 pr_warning("Could not initialize trace "
951 "events/%s\n", call->name);
952 return ret;
953 }
954 }
950 955
951/* 956 d_events = event_trace_events_dir();
952 * Modules must own their file_operations to keep up with 957 if (!d_events)
953 * reference counting. 958 return -ENOENT;
954 */ 959
955struct ftrace_module_file_ops { 960 list_add(&call->list, &ftrace_events);
956 struct list_head list; 961 ret = event_create_dir(call, d_events, &ftrace_event_id_fops,
957 struct module *mod; 962 &ftrace_enable_fops, &ftrace_event_filter_fops,
958 struct file_operations id; 963 &ftrace_event_format_fops);
959 struct file_operations enable; 964 if (ret < 0)
960 struct file_operations format; 965 list_del(&call->list);
961 struct file_operations filter; 966 return ret;
962}; 967}
968
969/* Add an additional event_call dynamically */
970int trace_add_event_call(struct ftrace_event_call *call)
971{
972 int ret;
973 mutex_lock(&event_mutex);
974 ret = __trace_add_event_call(call);
975 mutex_unlock(&event_mutex);
976 return ret;
977}
963 978
964static void remove_subsystem_dir(const char *name) 979static void remove_subsystem_dir(const char *name)
965{ 980{
@@ -987,6 +1002,53 @@ static void remove_subsystem_dir(const char *name)
987 } 1002 }
988} 1003}
989 1004
1005/*
1006 * Must be called under locking both of event_mutex and trace_event_mutex.
1007 */
1008static void __trace_remove_event_call(struct ftrace_event_call *call)
1009{
1010 ftrace_event_enable_disable(call, 0);
1011 if (call->event)
1012 __unregister_ftrace_event(call->event);
1013 debugfs_remove_recursive(call->dir);
1014 list_del(&call->list);
1015 trace_destroy_fields(call);
1016 destroy_preds(call);
1017 remove_subsystem_dir(call->system);
1018}
1019
1020/* Remove an event_call */
1021void trace_remove_event_call(struct ftrace_event_call *call)
1022{
1023 mutex_lock(&event_mutex);
1024 down_write(&trace_event_mutex);
1025 __trace_remove_event_call(call);
1026 up_write(&trace_event_mutex);
1027 mutex_unlock(&event_mutex);
1028}
1029
1030#define for_each_event(event, start, end) \
1031 for (event = start; \
1032 (unsigned long)event < (unsigned long)end; \
1033 event++)
1034
1035#ifdef CONFIG_MODULES
1036
1037static LIST_HEAD(ftrace_module_file_list);
1038
1039/*
1040 * Modules must own their file_operations to keep up with
1041 * reference counting.
1042 */
1043struct ftrace_module_file_ops {
1044 struct list_head list;
1045 struct module *mod;
1046 struct file_operations id;
1047 struct file_operations enable;
1048 struct file_operations format;
1049 struct file_operations filter;
1050};
1051
990static struct ftrace_module_file_ops * 1052static struct ftrace_module_file_ops *
991trace_create_file_ops(struct module *mod) 1053trace_create_file_ops(struct module *mod)
992{ 1054{
@@ -1044,7 +1106,7 @@ static void trace_module_add_events(struct module *mod)
1044 if (!call->name) 1106 if (!call->name)
1045 continue; 1107 continue;
1046 if (call->raw_init) { 1108 if (call->raw_init) {
1047 ret = call->raw_init(); 1109 ret = call->raw_init(call);
1048 if (ret < 0) { 1110 if (ret < 0) {
1049 if (ret != -ENOSYS) 1111 if (ret != -ENOSYS)
1050 pr_warning("Could not initialize trace " 1112 pr_warning("Could not initialize trace "
@@ -1079,14 +1141,7 @@ static void trace_module_remove_events(struct module *mod)
1079 list_for_each_entry_safe(call, p, &ftrace_events, list) { 1141 list_for_each_entry_safe(call, p, &ftrace_events, list) {
1080 if (call->mod == mod) { 1142 if (call->mod == mod) {
1081 found = true; 1143 found = true;
1082 ftrace_event_enable_disable(call, 0); 1144 __trace_remove_event_call(call);
1083 if (call->event)
1084 __unregister_ftrace_event(call->event);
1085 debugfs_remove_recursive(call->dir);
1086 list_del(&call->list);
1087 trace_destroy_fields(call);
1088 destroy_preds(call);
1089 remove_subsystem_dir(call->system);
1090 } 1145 }
1091 } 1146 }
1092 1147
@@ -1204,7 +1259,7 @@ static __init int event_trace_init(void)
1204 if (!call->name) 1259 if (!call->name)
1205 continue; 1260 continue;
1206 if (call->raw_init) { 1261 if (call->raw_init) {
1207 ret = call->raw_init(); 1262 ret = call->raw_init(call);
1208 if (ret < 0) { 1263 if (ret < 0) {
1209 if (ret != -ENOSYS) 1264 if (ret != -ENOSYS)
1210 pr_warning("Could not initialize trace " 1265 pr_warning("Could not initialize trace "
diff --git a/kernel/trace/trace_export.c b/kernel/trace/trace_export.c
index 9753fcc61bc5..ed7d48083520 100644
--- a/kernel/trace/trace_export.c
+++ b/kernel/trace/trace_export.c
@@ -131,7 +131,6 @@ ftrace_format_##name(struct ftrace_event_call *unused, \
131 131
132#include "trace_entries.h" 132#include "trace_entries.h"
133 133
134
135#undef __field 134#undef __field
136#define __field(type, item) \ 135#define __field(type, item) \
137 ret = trace_define_field(event_call, #type, #item, \ 136 ret = trace_define_field(event_call, #type, #item, \
@@ -193,6 +192,11 @@ ftrace_define_fields_##name(struct ftrace_event_call *event_call) \
193 192
194#include "trace_entries.h" 193#include "trace_entries.h"
195 194
195static int ftrace_raw_init_event(struct ftrace_event_call *call)
196{
197 INIT_LIST_HEAD(&call->fields);
198 return 0;
199}
196 200
197#undef __field 201#undef __field
198#define __field(type, item) 202#define __field(type, item)
@@ -211,7 +215,6 @@ ftrace_define_fields_##name(struct ftrace_event_call *event_call) \
211 215
212#undef FTRACE_ENTRY 216#undef FTRACE_ENTRY
213#define FTRACE_ENTRY(call, struct_name, type, tstruct, print) \ 217#define FTRACE_ENTRY(call, struct_name, type, tstruct, print) \
214static int ftrace_raw_init_event_##call(void); \
215 \ 218 \
216struct ftrace_event_call __used \ 219struct ftrace_event_call __used \
217__attribute__((__aligned__(4))) \ 220__attribute__((__aligned__(4))) \
@@ -219,14 +222,9 @@ __attribute__((section("_ftrace_events"))) event_##call = { \
219 .name = #call, \ 222 .name = #call, \
220 .id = type, \ 223 .id = type, \
221 .system = __stringify(TRACE_SYSTEM), \ 224 .system = __stringify(TRACE_SYSTEM), \
222 .raw_init = ftrace_raw_init_event_##call, \ 225 .raw_init = ftrace_raw_init_event, \
223 .show_format = ftrace_format_##call, \ 226 .show_format = ftrace_format_##call, \
224 .define_fields = ftrace_define_fields_##call, \ 227 .define_fields = ftrace_define_fields_##call, \
225}; \ 228}; \
226static int ftrace_raw_init_event_##call(void) \
227{ \
228 INIT_LIST_HEAD(&event_##call.fields); \
229 return 0; \
230} \
231 229
232#include "trace_entries.h" 230#include "trace_entries.h"
diff --git a/kernel/trace/trace_kprobe.c b/kernel/trace/trace_kprobe.c
new file mode 100644
index 000000000000..09cba270392d
--- /dev/null
+++ b/kernel/trace/trace_kprobe.c
@@ -0,0 +1,1389 @@
1/*
2 * kprobe based kernel tracer
3 *
4 * Created by Masami Hiramatsu <mhiramat@redhat.com>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18 */
19
20#include <linux/module.h>
21#include <linux/uaccess.h>
22#include <linux/kprobes.h>
23#include <linux/seq_file.h>
24#include <linux/slab.h>
25#include <linux/smp.h>
26#include <linux/debugfs.h>
27#include <linux/types.h>
28#include <linux/string.h>
29#include <linux/ctype.h>
30#include <linux/ptrace.h>
31#include <linux/perf_event.h>
32
33#include "trace.h"
34#include "trace_output.h"
35
36#define MAX_TRACE_ARGS 128
37#define MAX_ARGSTR_LEN 63
38#define MAX_EVENT_NAME_LEN 64
39#define KPROBE_EVENT_SYSTEM "kprobes"
40
41/* currently, trace_kprobe only supports X86. */
42
43struct fetch_func {
44 unsigned long (*func)(struct pt_regs *, void *);
45 void *data;
46};
47
48static __kprobes unsigned long call_fetch(struct fetch_func *f,
49 struct pt_regs *regs)
50{
51 return f->func(regs, f->data);
52}
53
54/* fetch handlers */
55static __kprobes unsigned long fetch_register(struct pt_regs *regs,
56 void *offset)
57{
58 return regs_get_register(regs, (unsigned int)((unsigned long)offset));
59}
60
61static __kprobes unsigned long fetch_stack(struct pt_regs *regs,
62 void *num)
63{
64 return regs_get_kernel_stack_nth(regs,
65 (unsigned int)((unsigned long)num));
66}
67
68static __kprobes unsigned long fetch_memory(struct pt_regs *regs, void *addr)
69{
70 unsigned long retval;
71
72 if (probe_kernel_address(addr, retval))
73 return 0;
74 return retval;
75}
76
77static __kprobes unsigned long fetch_argument(struct pt_regs *regs, void *num)
78{
79 return regs_get_argument_nth(regs, (unsigned int)((unsigned long)num));
80}
81
82static __kprobes unsigned long fetch_retvalue(struct pt_regs *regs,
83 void *dummy)
84{
85 return regs_return_value(regs);
86}
87
88static __kprobes unsigned long fetch_ip(struct pt_regs *regs, void *dummy)
89{
90 return instruction_pointer(regs);
91}
92
93static __kprobes unsigned long fetch_stack_address(struct pt_regs *regs,
94 void *dummy)
95{
96 return kernel_stack_pointer(regs);
97}
98
99/* Memory fetching by symbol */
100struct symbol_cache {
101 char *symbol;
102 long offset;
103 unsigned long addr;
104};
105
106static unsigned long update_symbol_cache(struct symbol_cache *sc)
107{
108 sc->addr = (unsigned long)kallsyms_lookup_name(sc->symbol);
109 if (sc->addr)
110 sc->addr += sc->offset;
111 return sc->addr;
112}
113
114static void free_symbol_cache(struct symbol_cache *sc)
115{
116 kfree(sc->symbol);
117 kfree(sc);
118}
119
120static struct symbol_cache *alloc_symbol_cache(const char *sym, long offset)
121{
122 struct symbol_cache *sc;
123
124 if (!sym || strlen(sym) == 0)
125 return NULL;
126 sc = kzalloc(sizeof(struct symbol_cache), GFP_KERNEL);
127 if (!sc)
128 return NULL;
129
130 sc->symbol = kstrdup(sym, GFP_KERNEL);
131 if (!sc->symbol) {
132 kfree(sc);
133 return NULL;
134 }
135 sc->offset = offset;
136
137 update_symbol_cache(sc);
138 return sc;
139}
140
141static __kprobes unsigned long fetch_symbol(struct pt_regs *regs, void *data)
142{
143 struct symbol_cache *sc = data;
144
145 if (sc->addr)
146 return fetch_memory(regs, (void *)sc->addr);
147 else
148 return 0;
149}
150
151/* Special indirect memory access interface */
152struct indirect_fetch_data {
153 struct fetch_func orig;
154 long offset;
155};
156
157static __kprobes unsigned long fetch_indirect(struct pt_regs *regs, void *data)
158{
159 struct indirect_fetch_data *ind = data;
160 unsigned long addr;
161
162 addr = call_fetch(&ind->orig, regs);
163 if (addr) {
164 addr += ind->offset;
165 return fetch_memory(regs, (void *)addr);
166 } else
167 return 0;
168}
169
170static __kprobes void free_indirect_fetch_data(struct indirect_fetch_data *data)
171{
172 if (data->orig.func == fetch_indirect)
173 free_indirect_fetch_data(data->orig.data);
174 else if (data->orig.func == fetch_symbol)
175 free_symbol_cache(data->orig.data);
176 kfree(data);
177}
178
179/**
180 * Kprobe tracer core functions
181 */
182
183struct probe_arg {
184 struct fetch_func fetch;
185 const char *name;
186};
187
188/* Flags for trace_probe */
189#define TP_FLAG_TRACE 1
190#define TP_FLAG_PROFILE 2
191
192struct trace_probe {
193 struct list_head list;
194 struct kretprobe rp; /* Use rp.kp for kprobe use */
195 unsigned long nhit;
196 unsigned int flags; /* For TP_FLAG_* */
197 const char *symbol; /* symbol name */
198 struct ftrace_event_call call;
199 struct trace_event event;
200 unsigned int nr_args;
201 struct probe_arg args[];
202};
203
204#define SIZEOF_TRACE_PROBE(n) \
205 (offsetof(struct trace_probe, args) + \
206 (sizeof(struct probe_arg) * (n)))
207
208static __kprobes int probe_is_return(struct trace_probe *tp)
209{
210 return tp->rp.handler != NULL;
211}
212
213static __kprobes const char *probe_symbol(struct trace_probe *tp)
214{
215 return tp->symbol ? tp->symbol : "unknown";
216}
217
218static int probe_arg_string(char *buf, size_t n, struct fetch_func *ff)
219{
220 int ret = -EINVAL;
221
222 if (ff->func == fetch_argument)
223 ret = snprintf(buf, n, "a%lu", (unsigned long)ff->data);
224 else if (ff->func == fetch_register) {
225 const char *name;
226 name = regs_query_register_name((unsigned int)((long)ff->data));
227 ret = snprintf(buf, n, "%%%s", name);
228 } else if (ff->func == fetch_stack)
229 ret = snprintf(buf, n, "s%lu", (unsigned long)ff->data);
230 else if (ff->func == fetch_memory)
231 ret = snprintf(buf, n, "@0x%p", ff->data);
232 else if (ff->func == fetch_symbol) {
233 struct symbol_cache *sc = ff->data;
234 ret = snprintf(buf, n, "@%s%+ld", sc->symbol, sc->offset);
235 } else if (ff->func == fetch_retvalue)
236 ret = snprintf(buf, n, "rv");
237 else if (ff->func == fetch_ip)
238 ret = snprintf(buf, n, "ra");
239 else if (ff->func == fetch_stack_address)
240 ret = snprintf(buf, n, "sa");
241 else if (ff->func == fetch_indirect) {
242 struct indirect_fetch_data *id = ff->data;
243 size_t l = 0;
244 ret = snprintf(buf, n, "%+ld(", id->offset);
245 if (ret >= n)
246 goto end;
247 l += ret;
248 ret = probe_arg_string(buf + l, n - l, &id->orig);
249 if (ret < 0)
250 goto end;
251 l += ret;
252 ret = snprintf(buf + l, n - l, ")");
253 ret += l;
254 }
255end:
256 if (ret >= n)
257 return -ENOSPC;
258 return ret;
259}
260
261static int register_probe_event(struct trace_probe *tp);
262static void unregister_probe_event(struct trace_probe *tp);
263
264static DEFINE_MUTEX(probe_lock);
265static LIST_HEAD(probe_list);
266
267static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs);
268static int kretprobe_dispatcher(struct kretprobe_instance *ri,
269 struct pt_regs *regs);
270
271/*
272 * Allocate new trace_probe and initialize it (including kprobes).
273 */
274static struct trace_probe *alloc_trace_probe(const char *group,
275 const char *event,
276 void *addr,
277 const char *symbol,
278 unsigned long offs,
279 int nargs, int is_return)
280{
281 struct trace_probe *tp;
282
283 tp = kzalloc(SIZEOF_TRACE_PROBE(nargs), GFP_KERNEL);
284 if (!tp)
285 return ERR_PTR(-ENOMEM);
286
287 if (symbol) {
288 tp->symbol = kstrdup(symbol, GFP_KERNEL);
289 if (!tp->symbol)
290 goto error;
291 tp->rp.kp.symbol_name = tp->symbol;
292 tp->rp.kp.offset = offs;
293 } else
294 tp->rp.kp.addr = addr;
295
296 if (is_return)
297 tp->rp.handler = kretprobe_dispatcher;
298 else
299 tp->rp.kp.pre_handler = kprobe_dispatcher;
300
301 if (!event)
302 goto error;
303 tp->call.name = kstrdup(event, GFP_KERNEL);
304 if (!tp->call.name)
305 goto error;
306
307 if (!group)
308 goto error;
309 tp->call.system = kstrdup(group, GFP_KERNEL);
310 if (!tp->call.system)
311 goto error;
312
313 INIT_LIST_HEAD(&tp->list);
314 return tp;
315error:
316 kfree(tp->call.name);
317 kfree(tp->symbol);
318 kfree(tp);
319 return ERR_PTR(-ENOMEM);
320}
321
322static void free_probe_arg(struct probe_arg *arg)
323{
324 if (arg->fetch.func == fetch_symbol)
325 free_symbol_cache(arg->fetch.data);
326 else if (arg->fetch.func == fetch_indirect)
327 free_indirect_fetch_data(arg->fetch.data);
328 kfree(arg->name);
329}
330
331static void free_trace_probe(struct trace_probe *tp)
332{
333 int i;
334
335 for (i = 0; i < tp->nr_args; i++)
336 free_probe_arg(&tp->args[i]);
337
338 kfree(tp->call.system);
339 kfree(tp->call.name);
340 kfree(tp->symbol);
341 kfree(tp);
342}
343
344static struct trace_probe *find_probe_event(const char *event)
345{
346 struct trace_probe *tp;
347
348 list_for_each_entry(tp, &probe_list, list)
349 if (!strcmp(tp->call.name, event))
350 return tp;
351 return NULL;
352}
353
354/* Unregister a trace_probe and probe_event: call with locking probe_lock */
355static void unregister_trace_probe(struct trace_probe *tp)
356{
357 if (probe_is_return(tp))
358 unregister_kretprobe(&tp->rp);
359 else
360 unregister_kprobe(&tp->rp.kp);
361 list_del(&tp->list);
362 unregister_probe_event(tp);
363}
364
365/* Register a trace_probe and probe_event */
366static int register_trace_probe(struct trace_probe *tp)
367{
368 struct trace_probe *old_tp;
369 int ret;
370
371 mutex_lock(&probe_lock);
372
373 /* register as an event */
374 old_tp = find_probe_event(tp->call.name);
375 if (old_tp) {
376 /* delete old event */
377 unregister_trace_probe(old_tp);
378 free_trace_probe(old_tp);
379 }
380 ret = register_probe_event(tp);
381 if (ret) {
382 pr_warning("Faild to register probe event(%d)\n", ret);
383 goto end;
384 }
385
386 tp->rp.kp.flags |= KPROBE_FLAG_DISABLED;
387 if (probe_is_return(tp))
388 ret = register_kretprobe(&tp->rp);
389 else
390 ret = register_kprobe(&tp->rp.kp);
391
392 if (ret) {
393 pr_warning("Could not insert probe(%d)\n", ret);
394 if (ret == -EILSEQ) {
395 pr_warning("Probing address(0x%p) is not an "
396 "instruction boundary.\n",
397 tp->rp.kp.addr);
398 ret = -EINVAL;
399 }
400 unregister_probe_event(tp);
401 } else
402 list_add_tail(&tp->list, &probe_list);
403end:
404 mutex_unlock(&probe_lock);
405 return ret;
406}
407
408/* Split symbol and offset. */
409static int split_symbol_offset(char *symbol, unsigned long *offset)
410{
411 char *tmp;
412 int ret;
413
414 if (!offset)
415 return -EINVAL;
416
417 tmp = strchr(symbol, '+');
418 if (tmp) {
419 /* skip sign because strict_strtol doesn't accept '+' */
420 ret = strict_strtoul(tmp + 1, 0, offset);
421 if (ret)
422 return ret;
423 *tmp = '\0';
424 } else
425 *offset = 0;
426 return 0;
427}
428
429#define PARAM_MAX_ARGS 16
430#define PARAM_MAX_STACK (THREAD_SIZE / sizeof(unsigned long))
431
432static int parse_probe_arg(char *arg, struct fetch_func *ff, int is_return)
433{
434 int ret = 0;
435 unsigned long param;
436 long offset;
437 char *tmp;
438
439 switch (arg[0]) {
440 case 'a': /* argument */
441 ret = strict_strtoul(arg + 1, 10, &param);
442 if (ret || param > PARAM_MAX_ARGS)
443 ret = -EINVAL;
444 else {
445 ff->func = fetch_argument;
446 ff->data = (void *)param;
447 }
448 break;
449 case 'r': /* retval or retaddr */
450 if (is_return && arg[1] == 'v') {
451 ff->func = fetch_retvalue;
452 ff->data = NULL;
453 } else if (is_return && arg[1] == 'a') {
454 ff->func = fetch_ip;
455 ff->data = NULL;
456 } else
457 ret = -EINVAL;
458 break;
459 case '%': /* named register */
460 ret = regs_query_register_offset(arg + 1);
461 if (ret >= 0) {
462 ff->func = fetch_register;
463 ff->data = (void *)(unsigned long)ret;
464 ret = 0;
465 }
466 break;
467 case 's': /* stack */
468 if (arg[1] == 'a') {
469 ff->func = fetch_stack_address;
470 ff->data = NULL;
471 } else {
472 ret = strict_strtoul(arg + 1, 10, &param);
473 if (ret || param > PARAM_MAX_STACK)
474 ret = -EINVAL;
475 else {
476 ff->func = fetch_stack;
477 ff->data = (void *)param;
478 }
479 }
480 break;
481 case '@': /* memory or symbol */
482 if (isdigit(arg[1])) {
483 ret = strict_strtoul(arg + 1, 0, &param);
484 if (ret)
485 break;
486 ff->func = fetch_memory;
487 ff->data = (void *)param;
488 } else {
489 ret = split_symbol_offset(arg + 1, &offset);
490 if (ret)
491 break;
492 ff->data = alloc_symbol_cache(arg + 1,
493 offset);
494 if (ff->data)
495 ff->func = fetch_symbol;
496 else
497 ret = -EINVAL;
498 }
499 break;
500 case '+': /* indirect memory */
501 case '-':
502 tmp = strchr(arg, '(');
503 if (!tmp) {
504 ret = -EINVAL;
505 break;
506 }
507 *tmp = '\0';
508 ret = strict_strtol(arg + 1, 0, &offset);
509 if (ret)
510 break;
511 if (arg[0] == '-')
512 offset = -offset;
513 arg = tmp + 1;
514 tmp = strrchr(arg, ')');
515 if (tmp) {
516 struct indirect_fetch_data *id;
517 *tmp = '\0';
518 id = kzalloc(sizeof(struct indirect_fetch_data),
519 GFP_KERNEL);
520 if (!id)
521 return -ENOMEM;
522 id->offset = offset;
523 ret = parse_probe_arg(arg, &id->orig, is_return);
524 if (ret)
525 kfree(id);
526 else {
527 ff->func = fetch_indirect;
528 ff->data = (void *)id;
529 }
530 } else
531 ret = -EINVAL;
532 break;
533 default:
534 /* TODO: support custom handler */
535 ret = -EINVAL;
536 }
537 return ret;
538}
539
540static int create_trace_probe(int argc, char **argv)
541{
542 /*
543 * Argument syntax:
544 * - Add kprobe: p[:[GRP/]EVENT] KSYM[+OFFS]|KADDR [FETCHARGS]
545 * - Add kretprobe: r[:[GRP/]EVENT] KSYM[+0] [FETCHARGS]
546 * Fetch args:
547 * aN : fetch Nth of function argument. (N:0-)
548 * rv : fetch return value
549 * ra : fetch return address
550 * sa : fetch stack address
551 * sN : fetch Nth of stack (N:0-)
552 * @ADDR : fetch memory at ADDR (ADDR should be in kernel)
553 * @SYM[+|-offs] : fetch memory at SYM +|- offs (SYM is a data symbol)
554 * %REG : fetch register REG
555 * Indirect memory fetch:
556 * +|-offs(ARG) : fetch memory at ARG +|- offs address.
557 * Alias name of args:
558 * NAME=FETCHARG : set NAME as alias of FETCHARG.
559 */
560 struct trace_probe *tp;
561 int i, ret = 0;
562 int is_return = 0;
563 char *symbol = NULL, *event = NULL, *arg = NULL, *group = NULL;
564 unsigned long offset = 0;
565 void *addr = NULL;
566 char buf[MAX_EVENT_NAME_LEN];
567
568 if (argc < 2)
569 return -EINVAL;
570
571 if (argv[0][0] == 'p')
572 is_return = 0;
573 else if (argv[0][0] == 'r')
574 is_return = 1;
575 else
576 return -EINVAL;
577
578 if (argv[0][1] == ':') {
579 event = &argv[0][2];
580 if (strchr(event, '/')) {
581 group = event;
582 event = strchr(group, '/') + 1;
583 event[-1] = '\0';
584 if (strlen(group) == 0) {
585 pr_info("Group name is not specifiled\n");
586 return -EINVAL;
587 }
588 }
589 if (strlen(event) == 0) {
590 pr_info("Event name is not specifiled\n");
591 return -EINVAL;
592 }
593 }
594
595 if (isdigit(argv[1][0])) {
596 if (is_return)
597 return -EINVAL;
598 /* an address specified */
599 ret = strict_strtoul(&argv[0][2], 0, (unsigned long *)&addr);
600 if (ret)
601 return ret;
602 } else {
603 /* a symbol specified */
604 symbol = argv[1];
605 /* TODO: support .init module functions */
606 ret = split_symbol_offset(symbol, &offset);
607 if (ret)
608 return ret;
609 if (offset && is_return)
610 return -EINVAL;
611 }
612 argc -= 2; argv += 2;
613
614 /* setup a probe */
615 if (!group)
616 group = KPROBE_EVENT_SYSTEM;
617 if (!event) {
618 /* Make a new event name */
619 if (symbol)
620 snprintf(buf, MAX_EVENT_NAME_LEN, "%c@%s%+ld",
621 is_return ? 'r' : 'p', symbol, offset);
622 else
623 snprintf(buf, MAX_EVENT_NAME_LEN, "%c@0x%p",
624 is_return ? 'r' : 'p', addr);
625 event = buf;
626 }
627 tp = alloc_trace_probe(group, event, addr, symbol, offset, argc,
628 is_return);
629 if (IS_ERR(tp))
630 return PTR_ERR(tp);
631
632 /* parse arguments */
633 ret = 0;
634 for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) {
635 /* Parse argument name */
636 arg = strchr(argv[i], '=');
637 if (arg)
638 *arg++ = '\0';
639 else
640 arg = argv[i];
641 tp->args[i].name = kstrdup(argv[i], GFP_KERNEL);
642
643 /* Parse fetch argument */
644 if (strlen(arg) > MAX_ARGSTR_LEN) {
645 pr_info("Argument%d(%s) is too long.\n", i, arg);
646 ret = -ENOSPC;
647 goto error;
648 }
649 ret = parse_probe_arg(arg, &tp->args[i].fetch, is_return);
650 if (ret)
651 goto error;
652 }
653 tp->nr_args = i;
654
655 ret = register_trace_probe(tp);
656 if (ret)
657 goto error;
658 return 0;
659
660error:
661 free_trace_probe(tp);
662 return ret;
663}
664
665static void cleanup_all_probes(void)
666{
667 struct trace_probe *tp;
668
669 mutex_lock(&probe_lock);
670 /* TODO: Use batch unregistration */
671 while (!list_empty(&probe_list)) {
672 tp = list_entry(probe_list.next, struct trace_probe, list);
673 unregister_trace_probe(tp);
674 free_trace_probe(tp);
675 }
676 mutex_unlock(&probe_lock);
677}
678
679
680/* Probes listing interfaces */
681static void *probes_seq_start(struct seq_file *m, loff_t *pos)
682{
683 mutex_lock(&probe_lock);
684 return seq_list_start(&probe_list, *pos);
685}
686
687static void *probes_seq_next(struct seq_file *m, void *v, loff_t *pos)
688{
689 return seq_list_next(v, &probe_list, pos);
690}
691
692static void probes_seq_stop(struct seq_file *m, void *v)
693{
694 mutex_unlock(&probe_lock);
695}
696
697static int probes_seq_show(struct seq_file *m, void *v)
698{
699 struct trace_probe *tp = v;
700 int i, ret;
701 char buf[MAX_ARGSTR_LEN + 1];
702
703 seq_printf(m, "%c", probe_is_return(tp) ? 'r' : 'p');
704 seq_printf(m, ":%s", tp->call.name);
705
706 if (tp->symbol)
707 seq_printf(m, " %s+%u", probe_symbol(tp), tp->rp.kp.offset);
708 else
709 seq_printf(m, " 0x%p", tp->rp.kp.addr);
710
711 for (i = 0; i < tp->nr_args; i++) {
712 ret = probe_arg_string(buf, MAX_ARGSTR_LEN, &tp->args[i].fetch);
713 if (ret < 0) {
714 pr_warning("Argument%d decoding error(%d).\n", i, ret);
715 return ret;
716 }
717 seq_printf(m, " %s=%s", tp->args[i].name, buf);
718 }
719 seq_printf(m, "\n");
720 return 0;
721}
722
723static const struct seq_operations probes_seq_op = {
724 .start = probes_seq_start,
725 .next = probes_seq_next,
726 .stop = probes_seq_stop,
727 .show = probes_seq_show
728};
729
730static int probes_open(struct inode *inode, struct file *file)
731{
732 if ((file->f_mode & FMODE_WRITE) &&
733 (file->f_flags & O_TRUNC))
734 cleanup_all_probes();
735
736 return seq_open(file, &probes_seq_op);
737}
738
739static int command_trace_probe(const char *buf)
740{
741 char **argv;
742 int argc = 0, ret = 0;
743
744 argv = argv_split(GFP_KERNEL, buf, &argc);
745 if (!argv)
746 return -ENOMEM;
747
748 if (argc)
749 ret = create_trace_probe(argc, argv);
750
751 argv_free(argv);
752 return ret;
753}
754
755#define WRITE_BUFSIZE 128
756
757static ssize_t probes_write(struct file *file, const char __user *buffer,
758 size_t count, loff_t *ppos)
759{
760 char *kbuf, *tmp;
761 int ret;
762 size_t done;
763 size_t size;
764
765 kbuf = kmalloc(WRITE_BUFSIZE, GFP_KERNEL);
766 if (!kbuf)
767 return -ENOMEM;
768
769 ret = done = 0;
770 while (done < count) {
771 size = count - done;
772 if (size >= WRITE_BUFSIZE)
773 size = WRITE_BUFSIZE - 1;
774 if (copy_from_user(kbuf, buffer + done, size)) {
775 ret = -EFAULT;
776 goto out;
777 }
778 kbuf[size] = '\0';
779 tmp = strchr(kbuf, '\n');
780 if (tmp) {
781 *tmp = '\0';
782 size = tmp - kbuf + 1;
783 } else if (done + size < count) {
784 pr_warning("Line length is too long: "
785 "Should be less than %d.", WRITE_BUFSIZE);
786 ret = -EINVAL;
787 goto out;
788 }
789 done += size;
790 /* Remove comments */
791 tmp = strchr(kbuf, '#');
792 if (tmp)
793 *tmp = '\0';
794
795 ret = command_trace_probe(kbuf);
796 if (ret)
797 goto out;
798 }
799 ret = done;
800out:
801 kfree(kbuf);
802 return ret;
803}
804
805static const struct file_operations kprobe_events_ops = {
806 .owner = THIS_MODULE,
807 .open = probes_open,
808 .read = seq_read,
809 .llseek = seq_lseek,
810 .release = seq_release,
811 .write = probes_write,
812};
813
814/* Probes profiling interfaces */
815static int probes_profile_seq_show(struct seq_file *m, void *v)
816{
817 struct trace_probe *tp = v;
818
819 seq_printf(m, " %-44s %15lu %15lu\n", tp->call.name, tp->nhit,
820 tp->rp.kp.nmissed);
821
822 return 0;
823}
824
825static const struct seq_operations profile_seq_op = {
826 .start = probes_seq_start,
827 .next = probes_seq_next,
828 .stop = probes_seq_stop,
829 .show = probes_profile_seq_show
830};
831
832static int profile_open(struct inode *inode, struct file *file)
833{
834 return seq_open(file, &profile_seq_op);
835}
836
837static const struct file_operations kprobe_profile_ops = {
838 .owner = THIS_MODULE,
839 .open = profile_open,
840 .read = seq_read,
841 .llseek = seq_lseek,
842 .release = seq_release,
843};
844
845/* Kprobe handler */
846static __kprobes int kprobe_trace_func(struct kprobe *kp, struct pt_regs *regs)
847{
848 struct trace_probe *tp = container_of(kp, struct trace_probe, rp.kp);
849 struct kprobe_trace_entry *entry;
850 struct ring_buffer_event *event;
851 struct ring_buffer *buffer;
852 int size, i, pc;
853 unsigned long irq_flags;
854 struct ftrace_event_call *call = &tp->call;
855
856 tp->nhit++;
857
858 local_save_flags(irq_flags);
859 pc = preempt_count();
860
861 size = SIZEOF_KPROBE_TRACE_ENTRY(tp->nr_args);
862
863 event = trace_current_buffer_lock_reserve(&buffer, call->id, size,
864 irq_flags, pc);
865 if (!event)
866 return 0;
867
868 entry = ring_buffer_event_data(event);
869 entry->nargs = tp->nr_args;
870 entry->ip = (unsigned long)kp->addr;
871 for (i = 0; i < tp->nr_args; i++)
872 entry->args[i] = call_fetch(&tp->args[i].fetch, regs);
873
874 if (!filter_current_check_discard(buffer, call, entry, event))
875 trace_nowake_buffer_unlock_commit(buffer, event, irq_flags, pc);
876 return 0;
877}
878
879/* Kretprobe handler */
880static __kprobes int kretprobe_trace_func(struct kretprobe_instance *ri,
881 struct pt_regs *regs)
882{
883 struct trace_probe *tp = container_of(ri->rp, struct trace_probe, rp);
884 struct kretprobe_trace_entry *entry;
885 struct ring_buffer_event *event;
886 struct ring_buffer *buffer;
887 int size, i, pc;
888 unsigned long irq_flags;
889 struct ftrace_event_call *call = &tp->call;
890
891 local_save_flags(irq_flags);
892 pc = preempt_count();
893
894 size = SIZEOF_KRETPROBE_TRACE_ENTRY(tp->nr_args);
895
896 event = trace_current_buffer_lock_reserve(&buffer, call->id, size,
897 irq_flags, pc);
898 if (!event)
899 return 0;
900
901 entry = ring_buffer_event_data(event);
902 entry->nargs = tp->nr_args;
903 entry->func = (unsigned long)tp->rp.kp.addr;
904 entry->ret_ip = (unsigned long)ri->ret_addr;
905 for (i = 0; i < tp->nr_args; i++)
906 entry->args[i] = call_fetch(&tp->args[i].fetch, regs);
907
908 if (!filter_current_check_discard(buffer, call, entry, event))
909 trace_nowake_buffer_unlock_commit(buffer, event, irq_flags, pc);
910
911 return 0;
912}
913
914/* Event entry printers */
915enum print_line_t
916print_kprobe_event(struct trace_iterator *iter, int flags)
917{
918 struct kprobe_trace_entry *field;
919 struct trace_seq *s = &iter->seq;
920 struct trace_event *event;
921 struct trace_probe *tp;
922 int i;
923
924 field = (struct kprobe_trace_entry *)iter->ent;
925 event = ftrace_find_event(field->ent.type);
926 tp = container_of(event, struct trace_probe, event);
927
928 if (!trace_seq_printf(s, "%s: (", tp->call.name))
929 goto partial;
930
931 if (!seq_print_ip_sym(s, field->ip, flags | TRACE_ITER_SYM_OFFSET))
932 goto partial;
933
934 if (!trace_seq_puts(s, ")"))
935 goto partial;
936
937 for (i = 0; i < field->nargs; i++)
938 if (!trace_seq_printf(s, " %s=%lx",
939 tp->args[i].name, field->args[i]))
940 goto partial;
941
942 if (!trace_seq_puts(s, "\n"))
943 goto partial;
944
945 return TRACE_TYPE_HANDLED;
946partial:
947 return TRACE_TYPE_PARTIAL_LINE;
948}
949
950enum print_line_t
951print_kretprobe_event(struct trace_iterator *iter, int flags)
952{
953 struct kretprobe_trace_entry *field;
954 struct trace_seq *s = &iter->seq;
955 struct trace_event *event;
956 struct trace_probe *tp;
957 int i;
958
959 field = (struct kretprobe_trace_entry *)iter->ent;
960 event = ftrace_find_event(field->ent.type);
961 tp = container_of(event, struct trace_probe, event);
962
963 if (!trace_seq_printf(s, "%s: (", tp->call.name))
964 goto partial;
965
966 if (!seq_print_ip_sym(s, field->ret_ip, flags | TRACE_ITER_SYM_OFFSET))
967 goto partial;
968
969 if (!trace_seq_puts(s, " <- "))
970 goto partial;
971
972 if (!seq_print_ip_sym(s, field->func, flags & ~TRACE_ITER_SYM_OFFSET))
973 goto partial;
974
975 if (!trace_seq_puts(s, ")"))
976 goto partial;
977
978 for (i = 0; i < field->nargs; i++)
979 if (!trace_seq_printf(s, " %s=%lx",
980 tp->args[i].name, field->args[i]))
981 goto partial;
982
983 if (!trace_seq_puts(s, "\n"))
984 goto partial;
985
986 return TRACE_TYPE_HANDLED;
987partial:
988 return TRACE_TYPE_PARTIAL_LINE;
989}
990
991static int probe_event_enable(struct ftrace_event_call *call)
992{
993 struct trace_probe *tp = (struct trace_probe *)call->data;
994
995 tp->flags |= TP_FLAG_TRACE;
996 if (probe_is_return(tp))
997 return enable_kretprobe(&tp->rp);
998 else
999 return enable_kprobe(&tp->rp.kp);
1000}
1001
1002static void probe_event_disable(struct ftrace_event_call *call)
1003{
1004 struct trace_probe *tp = (struct trace_probe *)call->data;
1005
1006 tp->flags &= ~TP_FLAG_TRACE;
1007 if (!(tp->flags & (TP_FLAG_TRACE | TP_FLAG_PROFILE))) {
1008 if (probe_is_return(tp))
1009 disable_kretprobe(&tp->rp);
1010 else
1011 disable_kprobe(&tp->rp.kp);
1012 }
1013}
1014
1015static int probe_event_raw_init(struct ftrace_event_call *event_call)
1016{
1017 INIT_LIST_HEAD(&event_call->fields);
1018
1019 return 0;
1020}
1021
1022#undef DEFINE_FIELD
1023#define DEFINE_FIELD(type, item, name, is_signed) \
1024 do { \
1025 ret = trace_define_field(event_call, #type, name, \
1026 offsetof(typeof(field), item), \
1027 sizeof(field.item), is_signed, \
1028 FILTER_OTHER); \
1029 if (ret) \
1030 return ret; \
1031 } while (0)
1032
1033static int kprobe_event_define_fields(struct ftrace_event_call *event_call)
1034{
1035 int ret, i;
1036 struct kprobe_trace_entry field;
1037 struct trace_probe *tp = (struct trace_probe *)event_call->data;
1038
1039 ret = trace_define_common_fields(event_call);
1040 if (!ret)
1041 return ret;
1042
1043 DEFINE_FIELD(unsigned long, ip, "ip", 0);
1044 DEFINE_FIELD(int, nargs, "nargs", 1);
1045 /* Set argument names as fields */
1046 for (i = 0; i < tp->nr_args; i++)
1047 DEFINE_FIELD(unsigned long, args[i], tp->args[i].name, 0);
1048 return 0;
1049}
1050
1051static int kretprobe_event_define_fields(struct ftrace_event_call *event_call)
1052{
1053 int ret, i;
1054 struct kretprobe_trace_entry field;
1055 struct trace_probe *tp = (struct trace_probe *)event_call->data;
1056
1057 ret = trace_define_common_fields(event_call);
1058 if (!ret)
1059 return ret;
1060
1061 DEFINE_FIELD(unsigned long, func, "func", 0);
1062 DEFINE_FIELD(unsigned long, ret_ip, "ret_ip", 0);
1063 DEFINE_FIELD(int, nargs, "nargs", 1);
1064 /* Set argument names as fields */
1065 for (i = 0; i < tp->nr_args; i++)
1066 DEFINE_FIELD(unsigned long, args[i], tp->args[i].name, 0);
1067 return 0;
1068}
1069
1070static int __probe_event_show_format(struct trace_seq *s,
1071 struct trace_probe *tp, const char *fmt,
1072 const char *arg)
1073{
1074 int i;
1075
1076 /* Show format */
1077 if (!trace_seq_printf(s, "\nprint fmt: \"%s", fmt))
1078 return 0;
1079
1080 for (i = 0; i < tp->nr_args; i++)
1081 if (!trace_seq_printf(s, " %s=%%lx", tp->args[i].name))
1082 return 0;
1083
1084 if (!trace_seq_printf(s, "\", %s", arg))
1085 return 0;
1086
1087 for (i = 0; i < tp->nr_args; i++)
1088 if (!trace_seq_printf(s, ", REC->%s", tp->args[i].name))
1089 return 0;
1090
1091 return trace_seq_puts(s, "\n");
1092}
1093
1094#undef SHOW_FIELD
1095#define SHOW_FIELD(type, item, name) \
1096 do { \
1097 ret = trace_seq_printf(s, "\tfield: " #type " %s;\t" \
1098 "offset:%u;\tsize:%u;\n", name, \
1099 (unsigned int)offsetof(typeof(field), item),\
1100 (unsigned int)sizeof(type)); \
1101 if (!ret) \
1102 return 0; \
1103 } while (0)
1104
1105static int kprobe_event_show_format(struct ftrace_event_call *call,
1106 struct trace_seq *s)
1107{
1108 struct kprobe_trace_entry field __attribute__((unused));
1109 int ret, i;
1110 struct trace_probe *tp = (struct trace_probe *)call->data;
1111
1112 SHOW_FIELD(unsigned long, ip, "ip");
1113 SHOW_FIELD(int, nargs, "nargs");
1114
1115 /* Show fields */
1116 for (i = 0; i < tp->nr_args; i++)
1117 SHOW_FIELD(unsigned long, args[i], tp->args[i].name);
1118 trace_seq_puts(s, "\n");
1119
1120 return __probe_event_show_format(s, tp, "(%lx)", "REC->ip");
1121}
1122
1123static int kretprobe_event_show_format(struct ftrace_event_call *call,
1124 struct trace_seq *s)
1125{
1126 struct kretprobe_trace_entry field __attribute__((unused));
1127 int ret, i;
1128 struct trace_probe *tp = (struct trace_probe *)call->data;
1129
1130 SHOW_FIELD(unsigned long, func, "func");
1131 SHOW_FIELD(unsigned long, ret_ip, "ret_ip");
1132 SHOW_FIELD(int, nargs, "nargs");
1133
1134 /* Show fields */
1135 for (i = 0; i < tp->nr_args; i++)
1136 SHOW_FIELD(unsigned long, args[i], tp->args[i].name);
1137 trace_seq_puts(s, "\n");
1138
1139 return __probe_event_show_format(s, tp, "(%lx <- %lx)",
1140 "REC->func, REC->ret_ip");
1141}
1142
1143#ifdef CONFIG_EVENT_PROFILE
1144
1145/* Kprobe profile handler */
1146static __kprobes int kprobe_profile_func(struct kprobe *kp,
1147 struct pt_regs *regs)
1148{
1149 struct trace_probe *tp = container_of(kp, struct trace_probe, rp.kp);
1150 struct ftrace_event_call *call = &tp->call;
1151 struct kprobe_trace_entry *entry;
1152 int size, __size, i, pc;
1153 unsigned long irq_flags;
1154
1155 local_save_flags(irq_flags);
1156 pc = preempt_count();
1157
1158 __size = SIZEOF_KPROBE_TRACE_ENTRY(tp->nr_args);
1159 size = ALIGN(__size + sizeof(u32), sizeof(u64));
1160 size -= sizeof(u32);
1161
1162 do {
1163 char raw_data[size];
1164 struct trace_entry *ent;
1165 /*
1166 * Zero dead bytes from alignment to avoid stack leak
1167 * to userspace
1168 */
1169 *(u64 *)(&raw_data[size - sizeof(u64)]) = 0ULL;
1170 entry = (struct kprobe_trace_entry *)raw_data;
1171 ent = &entry->ent;
1172
1173 tracing_generic_entry_update(ent, irq_flags, pc);
1174 ent->type = call->id;
1175 entry->nargs = tp->nr_args;
1176 entry->ip = (unsigned long)kp->addr;
1177 for (i = 0; i < tp->nr_args; i++)
1178 entry->args[i] = call_fetch(&tp->args[i].fetch, regs);
1179 perf_tp_event(call->id, entry->ip, 1, entry, size);
1180 } while (0);
1181 return 0;
1182}
1183
1184/* Kretprobe profile handler */
1185static __kprobes int kretprobe_profile_func(struct kretprobe_instance *ri,
1186 struct pt_regs *regs)
1187{
1188 struct trace_probe *tp = container_of(ri->rp, struct trace_probe, rp);
1189 struct ftrace_event_call *call = &tp->call;
1190 struct kretprobe_trace_entry *entry;
1191 int size, __size, i, pc;
1192 unsigned long irq_flags;
1193
1194 local_save_flags(irq_flags);
1195 pc = preempt_count();
1196
1197 __size = SIZEOF_KRETPROBE_TRACE_ENTRY(tp->nr_args);
1198 size = ALIGN(__size + sizeof(u32), sizeof(u64));
1199 size -= sizeof(u32);
1200
1201 do {
1202 char raw_data[size];
1203 struct trace_entry *ent;
1204
1205 *(u64 *)(&raw_data[size - sizeof(u64)]) = 0ULL;
1206 entry = (struct kretprobe_trace_entry *)raw_data;
1207 ent = &entry->ent;
1208
1209 tracing_generic_entry_update(ent, irq_flags, pc);
1210 ent->type = call->id;
1211 entry->nargs = tp->nr_args;
1212 entry->func = (unsigned long)tp->rp.kp.addr;
1213 entry->ret_ip = (unsigned long)ri->ret_addr;
1214 for (i = 0; i < tp->nr_args; i++)
1215 entry->args[i] = call_fetch(&tp->args[i].fetch, regs);
1216 perf_tp_event(call->id, entry->ret_ip, 1, entry, size);
1217 } while (0);
1218 return 0;
1219}
1220
1221static int probe_profile_enable(struct ftrace_event_call *call)
1222{
1223 struct trace_probe *tp = (struct trace_probe *)call->data;
1224
1225 tp->flags |= TP_FLAG_PROFILE;
1226
1227 if (probe_is_return(tp))
1228 return enable_kretprobe(&tp->rp);
1229 else
1230 return enable_kprobe(&tp->rp.kp);
1231}
1232
1233static void probe_profile_disable(struct ftrace_event_call *call)
1234{
1235 struct trace_probe *tp = (struct trace_probe *)call->data;
1236
1237 tp->flags &= ~TP_FLAG_PROFILE;
1238
1239 if (!(tp->flags & TP_FLAG_TRACE)) {
1240 if (probe_is_return(tp))
1241 disable_kretprobe(&tp->rp);
1242 else
1243 disable_kprobe(&tp->rp.kp);
1244 }
1245}
1246#endif /* CONFIG_EVENT_PROFILE */
1247
1248
1249static __kprobes
1250int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs)
1251{
1252 struct trace_probe *tp = container_of(kp, struct trace_probe, rp.kp);
1253
1254 if (tp->flags & TP_FLAG_TRACE)
1255 kprobe_trace_func(kp, regs);
1256#ifdef CONFIG_EVENT_PROFILE
1257 if (tp->flags & TP_FLAG_PROFILE)
1258 kprobe_profile_func(kp, regs);
1259#endif /* CONFIG_EVENT_PROFILE */
1260 return 0; /* We don't tweek kernel, so just return 0 */
1261}
1262
1263static __kprobes
1264int kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs)
1265{
1266 struct trace_probe *tp = container_of(ri->rp, struct trace_probe, rp);
1267
1268 if (tp->flags & TP_FLAG_TRACE)
1269 kretprobe_trace_func(ri, regs);
1270#ifdef CONFIG_EVENT_PROFILE
1271 if (tp->flags & TP_FLAG_PROFILE)
1272 kretprobe_profile_func(ri, regs);
1273#endif /* CONFIG_EVENT_PROFILE */
1274 return 0; /* We don't tweek kernel, so just return 0 */
1275}
1276
1277static int register_probe_event(struct trace_probe *tp)
1278{
1279 struct ftrace_event_call *call = &tp->call;
1280 int ret;
1281
1282 /* Initialize ftrace_event_call */
1283 if (probe_is_return(tp)) {
1284 tp->event.trace = print_kretprobe_event;
1285 call->raw_init = probe_event_raw_init;
1286 call->show_format = kretprobe_event_show_format;
1287 call->define_fields = kretprobe_event_define_fields;
1288 } else {
1289 tp->event.trace = print_kprobe_event;
1290 call->raw_init = probe_event_raw_init;
1291 call->show_format = kprobe_event_show_format;
1292 call->define_fields = kprobe_event_define_fields;
1293 }
1294 call->event = &tp->event;
1295 call->id = register_ftrace_event(&tp->event);
1296 if (!call->id)
1297 return -ENODEV;
1298 call->enabled = 0;
1299 call->regfunc = probe_event_enable;
1300 call->unregfunc = probe_event_disable;
1301
1302#ifdef CONFIG_EVENT_PROFILE
1303 atomic_set(&call->profile_count, -1);
1304 call->profile_enable = probe_profile_enable;
1305 call->profile_disable = probe_profile_disable;
1306#endif
1307 call->data = tp;
1308 ret = trace_add_event_call(call);
1309 if (ret) {
1310 pr_info("Failed to register kprobe event: %s\n", call->name);
1311 unregister_ftrace_event(&tp->event);
1312 }
1313 return ret;
1314}
1315
1316static void unregister_probe_event(struct trace_probe *tp)
1317{
1318 /* tp->event is unregistered in trace_remove_event_call() */
1319 trace_remove_event_call(&tp->call);
1320}
1321
1322/* Make a debugfs interface for controling probe points */
1323static __init int init_kprobe_trace(void)
1324{
1325 struct dentry *d_tracer;
1326 struct dentry *entry;
1327
1328 d_tracer = tracing_init_dentry();
1329 if (!d_tracer)
1330 return 0;
1331
1332 entry = debugfs_create_file("kprobe_events", 0644, d_tracer,
1333 NULL, &kprobe_events_ops);
1334
1335 /* Event list interface */
1336 if (!entry)
1337 pr_warning("Could not create debugfs "
1338 "'kprobe_events' entry\n");
1339
1340 /* Profile interface */
1341 entry = debugfs_create_file("kprobe_profile", 0444, d_tracer,
1342 NULL, &kprobe_profile_ops);
1343
1344 if (!entry)
1345 pr_warning("Could not create debugfs "
1346 "'kprobe_profile' entry\n");
1347 return 0;
1348}
1349fs_initcall(init_kprobe_trace);
1350
1351
1352#ifdef CONFIG_FTRACE_STARTUP_TEST
1353
1354static int kprobe_trace_selftest_target(int a1, int a2, int a3,
1355 int a4, int a5, int a6)
1356{
1357 return a1 + a2 + a3 + a4 + a5 + a6;
1358}
1359
1360static __init int kprobe_trace_self_tests_init(void)
1361{
1362 int ret;
1363 int (*target)(int, int, int, int, int, int);
1364
1365 target = kprobe_trace_selftest_target;
1366
1367 pr_info("Testing kprobe tracing: ");
1368
1369 ret = command_trace_probe("p:testprobe kprobe_trace_selftest_target "
1370 "a1 a2 a3 a4 a5 a6");
1371 if (WARN_ON_ONCE(ret))
1372 pr_warning("error enabling function entry\n");
1373
1374 ret = command_trace_probe("r:testprobe2 kprobe_trace_selftest_target "
1375 "ra rv");
1376 if (WARN_ON_ONCE(ret))
1377 pr_warning("error enabling function return\n");
1378
1379 ret = target(1, 2, 3, 4, 5, 6);
1380
1381 cleanup_all_probes();
1382
1383 pr_cont("OK\n");
1384 return 0;
1385}
1386
1387late_initcall(kprobe_trace_self_tests_init);
1388
1389#endif
diff --git a/kernel/trace/trace_syscalls.c b/kernel/trace/trace_syscalls.c
index 9fbce6c9d2e1..1b050ab47120 100644
--- a/kernel/trace/trace_syscalls.c
+++ b/kernel/trace/trace_syscalls.c
@@ -285,13 +285,13 @@ void ftrace_syscall_exit(struct pt_regs *regs, long ret)
285 trace_current_buffer_unlock_commit(buffer, event, 0, 0); 285 trace_current_buffer_unlock_commit(buffer, event, 0, 0);
286} 286}
287 287
288int reg_event_syscall_enter(void *ptr) 288int reg_event_syscall_enter(struct ftrace_event_call *call)
289{ 289{
290 int ret = 0; 290 int ret = 0;
291 int num; 291 int num;
292 char *name; 292 char *name;
293 293
294 name = (char *)ptr; 294 name = (char *)call->data;
295 num = syscall_name_to_nr(name); 295 num = syscall_name_to_nr(name);
296 if (num < 0 || num >= NR_syscalls) 296 if (num < 0 || num >= NR_syscalls)
297 return -ENOSYS; 297 return -ENOSYS;
@@ -309,12 +309,12 @@ int reg_event_syscall_enter(void *ptr)
309 return ret; 309 return ret;
310} 310}
311 311
312void unreg_event_syscall_enter(void *ptr) 312void unreg_event_syscall_enter(struct ftrace_event_call *call)
313{ 313{
314 int num; 314 int num;
315 char *name; 315 char *name;
316 316
317 name = (char *)ptr; 317 name = (char *)call->data;
318 num = syscall_name_to_nr(name); 318 num = syscall_name_to_nr(name);
319 if (num < 0 || num >= NR_syscalls) 319 if (num < 0 || num >= NR_syscalls)
320 return; 320 return;
@@ -326,13 +326,13 @@ void unreg_event_syscall_enter(void *ptr)
326 mutex_unlock(&syscall_trace_lock); 326 mutex_unlock(&syscall_trace_lock);
327} 327}
328 328
329int reg_event_syscall_exit(void *ptr) 329int reg_event_syscall_exit(struct ftrace_event_call *call)
330{ 330{
331 int ret = 0; 331 int ret = 0;
332 int num; 332 int num;
333 char *name; 333 char *name;
334 334
335 name = (char *)ptr; 335 name = call->data;
336 num = syscall_name_to_nr(name); 336 num = syscall_name_to_nr(name);
337 if (num < 0 || num >= NR_syscalls) 337 if (num < 0 || num >= NR_syscalls)
338 return -ENOSYS; 338 return -ENOSYS;
@@ -350,12 +350,12 @@ int reg_event_syscall_exit(void *ptr)
350 return ret; 350 return ret;
351} 351}
352 352
353void unreg_event_syscall_exit(void *ptr) 353void unreg_event_syscall_exit(struct ftrace_event_call *call)
354{ 354{
355 int num; 355 int num;
356 char *name; 356 char *name;
357 357
358 name = (char *)ptr; 358 name = call->data;
359 num = syscall_name_to_nr(name); 359 num = syscall_name_to_nr(name);
360 if (num < 0 || num >= NR_syscalls) 360 if (num < 0 || num >= NR_syscalls)
361 return; 361 return;
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-26 18:51:04 -0500