aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMasami Hiramatsu <mhiramat@kernel.org>2018-04-25 08:18:03 -0400
committerSteven Rostedt (VMware) <rostedt@goodmis.org>2018-10-10 22:19:07 -0400
commit533059281ee594f9fbb9e58042aaec77083ef251 (patch)
tree92019727bc72f3253f5e9de7152ced000cc4f27b
parent7bfbc63eda08b8158c040d6882c807f62b0750bb (diff)
tracing: probeevent: Introduce new argument fetching code
Replace {k,u}probe event argument fetching framework with switch-case based. Currently that is implemented with structures, macros and chain of function-pointers, which is more complicated than necessary and may get a performance penalty by retpoline. This simplify that with an array of "fetch_insn" (opcode and oprands), and make process_fetch_insn() just interprets it. No function pointers are used. Link: http://lkml.kernel.org/r/152465868340.26224.2551120475197839464.stgit@devbox Signed-off-by: Masami Hiramatsu <mhiramat@kernel.org> Signed-off-by: Steven Rostedt (VMware) <rostedt@goodmis.org>
Diffstat
-rw-r--r--kernel/trace/trace_kprobe.c291
-rw-r--r--kernel/trace/trace_probe.c401
-rw-r--r--kernel/trace/trace_probe.h230
-rw-r--r--kernel/trace/trace_probe_tmpl.h120
-rw-r--r--kernel/trace/trace_uprobe.c127
5 files changed, 491 insertions, 678 deletions
diff --git a/kernel/trace/trace_kprobe.c b/kernel/trace/trace_kprobe.c
index 1356927e32d0..c024cc40d509 100644
--- a/kernel/trace/trace_kprobe.c
+++ b/kernel/trace/trace_kprobe.c
@@ -14,6 +14,7 @@
14 14
15#include "trace_kprobe_selftest.h" 15#include "trace_kprobe_selftest.h"
16#include "trace_probe.h" 16#include "trace_probe.h"
17#include "trace_probe_tmpl.h"
17 18
18#define KPROBE_EVENT_SYSTEM "kprobes" 19#define KPROBE_EVENT_SYSTEM "kprobes"
19#define KRETPROBE_MAXACTIVE_MAX 4096 20#define KRETPROBE_MAXACTIVE_MAX 4096
@@ -120,160 +121,6 @@ static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs);
120static int kretprobe_dispatcher(struct kretprobe_instance *ri, 121static int kretprobe_dispatcher(struct kretprobe_instance *ri,
121 struct pt_regs *regs); 122 struct pt_regs *regs);
122 123
123/* Memory fetching by symbol */
124struct symbol_cache {
125 char *symbol;
126 long offset;
127 unsigned long addr;
128};
129
130unsigned long update_symbol_cache(struct symbol_cache *sc)
131{
132 sc->addr = (unsigned long)kallsyms_lookup_name(sc->symbol);
133
134 if (sc->addr)
135 sc->addr += sc->offset;
136
137 return sc->addr;
138}
139
140void free_symbol_cache(struct symbol_cache *sc)
141{
142 kfree(sc->symbol);
143 kfree(sc);
144}
145
146struct symbol_cache *alloc_symbol_cache(const char *sym, long offset)
147{
148 struct symbol_cache *sc;
149
150 if (!sym || strlen(sym) == 0)
151 return NULL;
152
153 sc = kzalloc(sizeof(struct symbol_cache), GFP_KERNEL);
154 if (!sc)
155 return NULL;
156
157 sc->symbol = kstrdup(sym, GFP_KERNEL);
158 if (!sc->symbol) {
159 kfree(sc);
160 return NULL;
161 }
162 sc->offset = offset;
163 update_symbol_cache(sc);
164
165 return sc;
166}
167
168/*
169 * Kprobes-specific fetch functions
170 */
171#define DEFINE_FETCH_stack(type) \
172static void FETCH_FUNC_NAME(stack, type)(struct pt_regs *regs, \
173 void *offset, void *dest) \
174{ \
175 *(type *)dest = (type)regs_get_kernel_stack_nth(regs, \
176 (unsigned int)((unsigned long)offset)); \
177} \
178NOKPROBE_SYMBOL(FETCH_FUNC_NAME(stack, type));
179
180DEFINE_BASIC_FETCH_FUNCS(stack)
181/* No string on the stack entry */
182#define fetch_stack_string NULL
183#define fetch_stack_string_size NULL
184
185#define DEFINE_FETCH_memory(type) \
186static void FETCH_FUNC_NAME(memory, type)(struct pt_regs *regs, \
187 void *addr, void *dest) \
188{ \
189 type retval; \
190 if (probe_kernel_address(addr, retval)) \
191 *(type *)dest = 0; \
192 else \
193 *(type *)dest = retval; \
194} \
195NOKPROBE_SYMBOL(FETCH_FUNC_NAME(memory, type));
196
197DEFINE_BASIC_FETCH_FUNCS(memory)
198/*
199 * Fetch a null-terminated string. Caller MUST set *(u32 *)dest with max
200 * length and relative data location.
201 */
202static void FETCH_FUNC_NAME(memory, string)(struct pt_regs *regs,
203 void *addr, void *dest)
204{
205 int maxlen = get_rloc_len(*(u32 *)dest);
206 u8 *dst = get_rloc_data(dest);
207 long ret;
208
209 if (!maxlen)
210 return;
211
212 /*
213 * Try to get string again, since the string can be changed while
214 * probing.
215 */
216 ret = strncpy_from_unsafe(dst, addr, maxlen);
217
218 if (ret < 0) { /* Failed to fetch string */
219 dst[0] = '\0';
220 *(u32 *)dest = make_data_rloc(0, get_rloc_offs(*(u32 *)dest));
221 } else {
222 *(u32 *)dest = make_data_rloc(ret, get_rloc_offs(*(u32 *)dest));
223 }
224}
225NOKPROBE_SYMBOL(FETCH_FUNC_NAME(memory, string));
226
227/* Return the length of string -- including null terminal byte */
228static void FETCH_FUNC_NAME(memory, string_size)(struct pt_regs *regs,
229 void *addr, void *dest)
230{
231 mm_segment_t old_fs;
232 int ret, len = 0;
233 u8 c;
234
235 old_fs = get_fs();
236 set_fs(KERNEL_DS);
237 pagefault_disable();
238
239 do {
240 ret = __copy_from_user_inatomic(&c, (u8 *)addr + len, 1);
241 len++;
242 } while (c && ret == 0 && len < MAX_STRING_SIZE);
243
244 pagefault_enable();
245 set_fs(old_fs);
246
247 if (ret < 0) /* Failed to check the length */
248 *(u32 *)dest = 0;
249 else
250 *(u32 *)dest = len;
251}
252NOKPROBE_SYMBOL(FETCH_FUNC_NAME(memory, string_size));
253
254#define DEFINE_FETCH_symbol(type) \
255void FETCH_FUNC_NAME(symbol, type)(struct pt_regs *regs, void *data, void *dest)\
256{ \
257 struct symbol_cache *sc = data; \
258 if (sc->addr) \
259 fetch_memory_##type(regs, (void *)sc->addr, dest); \
260 else \
261 *(type *)dest = 0; \
262} \
263NOKPROBE_SYMBOL(FETCH_FUNC_NAME(symbol, type));
264
265DEFINE_BASIC_FETCH_FUNCS(symbol)
266DEFINE_FETCH_symbol(string)
267DEFINE_FETCH_symbol(string_size)
268
269/* kprobes don't support file_offset fetch methods */
270#define fetch_file_offset_u8 NULL
271#define fetch_file_offset_u16 NULL
272#define fetch_file_offset_u32 NULL
273#define fetch_file_offset_u64 NULL
274#define fetch_file_offset_string NULL
275#define fetch_file_offset_string_size NULL
276
277/* Fetch type information table */ 124/* Fetch type information table */
278static const struct fetch_type kprobes_fetch_type_table[] = { 125static const struct fetch_type kprobes_fetch_type_table[] = {
279 /* Special types */ 126 /* Special types */
@@ -529,7 +376,7 @@ static bool within_notrace_func(struct trace_kprobe *tk)
529/* Internal register function - just handle k*probes and flags */ 376/* Internal register function - just handle k*probes and flags */
530static int __register_trace_kprobe(struct trace_kprobe *tk) 377static int __register_trace_kprobe(struct trace_kprobe *tk)
531{ 378{
532 int i, ret; 379 int ret;
533 380
534 if (trace_probe_is_registered(&tk->tp)) 381 if (trace_probe_is_registered(&tk->tp))
535 return -EINVAL; 382 return -EINVAL;
@@ -540,9 +387,6 @@ static int __register_trace_kprobe(struct trace_kprobe *tk)
540 return -EINVAL; 387 return -EINVAL;
541 } 388 }
542 389
543 for (i = 0; i < tk->tp.nr_args; i++)
544 traceprobe_update_arg(&tk->tp.args[i]);
545
546 /* Set/clear disabled flag according to tp->flag */ 390 /* Set/clear disabled flag according to tp->flag */
547 if (trace_probe_is_enabled(&tk->tp)) 391 if (trace_probe_is_enabled(&tk->tp))
548 tk->rp.kp.flags &= ~KPROBE_FLAG_DISABLED; 392 tk->rp.kp.flags &= ~KPROBE_FLAG_DISABLED;
@@ -876,8 +720,8 @@ static int create_trace_kprobe(int argc, char **argv)
876 720
877 /* Parse fetch argument */ 721 /* Parse fetch argument */
878 ret = traceprobe_parse_probe_arg(arg, &tk->tp.size, parg, 722 ret = traceprobe_parse_probe_arg(arg, &tk->tp.size, parg,
879 is_return, true, 723 is_return, true,
880 kprobes_fetch_type_table); 724 kprobes_fetch_type_table);
881 if (ret) { 725 if (ret) {
882 pr_info("Parse error at argument[%d]. (%d)\n", i, ret); 726 pr_info("Parse error at argument[%d]. (%d)\n", i, ret);
883 goto error; 727 goto error;
@@ -1031,6 +875,133 @@ static const struct file_operations kprobe_profile_ops = {
1031 .release = seq_release, 875 .release = seq_release,
1032}; 876};
1033 877
878/* Kprobe specific fetch functions */
879
880/* Return the length of string -- including null terminal byte */
881static nokprobe_inline void
882fetch_store_strlen(unsigned long addr, void *dest)
883{
884 mm_segment_t old_fs;
885 int ret, len = 0;
886 u8 c;
887
888 old_fs = get_fs();
889 set_fs(KERNEL_DS);
890 pagefault_disable();
891
892 do {
893 ret = __copy_from_user_inatomic(&c, (u8 *)addr + len, 1);
894 len++;
895 } while (c && ret == 0 && len < MAX_STRING_SIZE);
896
897 pagefault_enable();
898 set_fs(old_fs);
899
900 if (ret < 0) /* Failed to check the length */
901 *(u32 *)dest = 0;
902 else
903 *(u32 *)dest = len;
904}
905
906/*
907 * Fetch a null-terminated string. Caller MUST set *(u32 *)buf with max
908 * length and relative data location.
909 */
910static nokprobe_inline void
911fetch_store_string(unsigned long addr, void *dest)
912{
913 int maxlen = get_rloc_len(*(u32 *)dest);
914 u8 *dst = get_rloc_data(dest);
915 long ret;
916
917 if (!maxlen)
918 return;
919
920 /*
921 * Try to get string again, since the string can be changed while
922 * probing.
923 */
924 ret = strncpy_from_unsafe(dst, (void *)addr, maxlen);
925
926 if (ret < 0) { /* Failed to fetch string */
927 dst[0] = '\0';
928 *(u32 *)dest = make_data_rloc(0, get_rloc_offs(*(u32 *)dest));
929 } else {
930 *(u32 *)dest = make_data_rloc(ret, get_rloc_offs(*(u32 *)dest));
931 }
932}
933
934/* Note that we don't verify it, since the code does not come from user space */
935static int
936process_fetch_insn(struct fetch_insn *code, struct pt_regs *regs, void *dest,
937 bool pre)
938{
939 unsigned long val;
940 int ret;
941
942 /* 1st stage: get value from context */
943 switch (code->op) {
944 case FETCH_OP_REG:
945 val = regs_get_register(regs, code->param);
946 break;
947 case FETCH_OP_STACK:
948 val = regs_get_kernel_stack_nth(regs, code->param);
949 break;
950 case FETCH_OP_STACKP:
951 val = kernel_stack_pointer(regs);
952 break;
953 case FETCH_OP_RETVAL:
954 val = regs_return_value(regs);
955 break;
956 case FETCH_OP_IMM:
957 val = code->immediate;
958 break;
959 case FETCH_OP_COMM:
960 val = (unsigned long)current->comm;
961 break;
962 default:
963 return -EILSEQ;
964 }
965 code++;
966
967 /* 2nd stage: dereference memory if needed */
968 while (code->op == FETCH_OP_DEREF) {
969 ret = probe_kernel_read(&val, (void *)val + code->offset,
970 sizeof(val));
971 if (ret)
972 return ret;
973 code++;
974 }
975
976 /* 3rd stage: store value to buffer */
977 switch (code->op) {
978 case FETCH_OP_ST_RAW:
979 fetch_store_raw(val, code, dest);
980 break;
981 case FETCH_OP_ST_MEM:
982 probe_kernel_read(dest, (void *)val + code->offset, code->size);
983 break;
984 case FETCH_OP_ST_STRING:
985 if (pre)
986 fetch_store_strlen(val + code->offset, dest);
987 else
988 fetch_store_string(val + code->offset, dest);
989 break;
990 default:
991 return -EILSEQ;
992 }
993 code++;
994
995 /* 4th stage: modify stored value if needed */
996 if (code->op == FETCH_OP_MOD_BF) {
997 fetch_apply_bitfield(code, dest);
998 code++;
999 }
1000
1001 return code->op == FETCH_OP_END ? 0 : -EILSEQ;
1002}
1003NOKPROBE_SYMBOL(process_fetch_insn)
1004
1034/* Kprobe handler */ 1005/* Kprobe handler */
1035static nokprobe_inline void 1006static nokprobe_inline void
1036__kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs, 1007__kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs,
diff --git a/kernel/trace/trace_probe.c b/kernel/trace/trace_probe.c
index 5f3b5b3fd2cd..c59c69cb2f2e 100644
--- a/kernel/trace/trace_probe.c
+++ b/kernel/trace/trace_probe.c
@@ -61,174 +61,6 @@ int PRINT_TYPE_FUNC_NAME(string)(struct trace_seq *s, void *data, void *ent)
61 61
62const char PRINT_TYPE_FMT_NAME(string)[] = "\\\"%s\\\""; 62const char PRINT_TYPE_FMT_NAME(string)[] = "\\\"%s\\\"";
63 63
64#define CHECK_FETCH_FUNCS(method, fn) \
65 (((FETCH_FUNC_NAME(method, u8) == fn) || \
66 (FETCH_FUNC_NAME(method, u16) == fn) || \
67 (FETCH_FUNC_NAME(method, u32) == fn) || \
68 (FETCH_FUNC_NAME(method, u64) == fn) || \
69 (FETCH_FUNC_NAME(method, string) == fn) || \
70 (FETCH_FUNC_NAME(method, string_size) == fn)) \
71 && (fn != NULL))
72
73/* Data fetch function templates */
74#define DEFINE_FETCH_reg(type) \
75void FETCH_FUNC_NAME(reg, type)(struct pt_regs *regs, void *offset, void *dest) \
76{ \
77 *(type *)dest = (type)regs_get_register(regs, \
78 (unsigned int)((unsigned long)offset)); \
79} \
80NOKPROBE_SYMBOL(FETCH_FUNC_NAME(reg, type));
81DEFINE_BASIC_FETCH_FUNCS(reg)
82/* No string on the register */
83#define fetch_reg_string NULL
84#define fetch_reg_string_size NULL
85
86#define DEFINE_FETCH_retval(type) \
87void FETCH_FUNC_NAME(retval, type)(struct pt_regs *regs, \
88 void *dummy, void *dest) \
89{ \
90 *(type *)dest = (type)regs_return_value(regs); \
91} \
92NOKPROBE_SYMBOL(FETCH_FUNC_NAME(retval, type));
93DEFINE_BASIC_FETCH_FUNCS(retval)
94/* No string on the retval */
95#define fetch_retval_string NULL
96#define fetch_retval_string_size NULL
97
98/* Dereference memory access function */
99struct deref_fetch_param {
100 struct fetch_param orig;
101 long offset;
102 fetch_func_t fetch;
103 fetch_func_t fetch_size;
104};
105
106#define DEFINE_FETCH_deref(type) \
107void FETCH_FUNC_NAME(deref, type)(struct pt_regs *regs, \
108 void *data, void *dest) \
109{ \
110 struct deref_fetch_param *dprm = data; \
111 unsigned long addr; \
112 call_fetch(&dprm->orig, regs, &addr); \
113 if (addr) { \
114 addr += dprm->offset; \
115 dprm->fetch(regs, (void *)addr, dest); \
116 } else \
117 *(type *)dest = 0; \
118} \
119NOKPROBE_SYMBOL(FETCH_FUNC_NAME(deref, type));
120DEFINE_BASIC_FETCH_FUNCS(deref)
121DEFINE_FETCH_deref(string)
122
123void FETCH_FUNC_NAME(deref, string_size)(struct pt_regs *regs,
124 void *data, void *dest)
125{
126 struct deref_fetch_param *dprm = data;
127 unsigned long addr;
128
129 call_fetch(&dprm->orig, regs, &addr);
130 if (addr && dprm->fetch_size) {
131 addr += dprm->offset;
132 dprm->fetch_size(regs, (void *)addr, dest);
133 } else
134 *(string_size *)dest = 0;
135}
136NOKPROBE_SYMBOL(FETCH_FUNC_NAME(deref, string_size));
137
138static void update_deref_fetch_param(struct deref_fetch_param *data)
139{
140 if (CHECK_FETCH_FUNCS(deref, data->orig.fn))
141 update_deref_fetch_param(data->orig.data);
142 else if (CHECK_FETCH_FUNCS(symbol, data->orig.fn))
143 update_symbol_cache(data->orig.data);
144}
145NOKPROBE_SYMBOL(update_deref_fetch_param);
146
147static void free_deref_fetch_param(struct deref_fetch_param *data)
148{
149 if (CHECK_FETCH_FUNCS(deref, data->orig.fn))
150 free_deref_fetch_param(data->orig.data);
151 else if (CHECK_FETCH_FUNCS(symbol, data->orig.fn))
152 free_symbol_cache(data->orig.data);
153 kfree(data);
154}
155NOKPROBE_SYMBOL(free_deref_fetch_param);
156
157/* Bitfield fetch function */
158struct bitfield_fetch_param {
159 struct fetch_param orig;
160 unsigned char hi_shift;
161 unsigned char low_shift;
162};
163
164#define DEFINE_FETCH_bitfield(type) \
165void FETCH_FUNC_NAME(bitfield, type)(struct pt_regs *regs, \
166 void *data, void *dest) \
167{ \
168 struct bitfield_fetch_param *bprm = data; \
169 type buf = 0; \
170 call_fetch(&bprm->orig, regs, &buf); \
171 if (buf) { \
172 buf <<= bprm->hi_shift; \
173 buf >>= bprm->low_shift; \
174 } \
175 *(type *)dest = buf; \
176} \
177NOKPROBE_SYMBOL(FETCH_FUNC_NAME(bitfield, type));
178DEFINE_BASIC_FETCH_FUNCS(bitfield)
179#define fetch_bitfield_string NULL
180#define fetch_bitfield_string_size NULL
181
182static void
183update_bitfield_fetch_param(struct bitfield_fetch_param *data)
184{
185 /*
186 * Don't check the bitfield itself, because this must be the
187 * last fetch function.
188 */
189 if (CHECK_FETCH_FUNCS(deref, data->orig.fn))
190 update_deref_fetch_param(data->orig.data);
191 else if (CHECK_FETCH_FUNCS(symbol, data->orig.fn))
192 update_symbol_cache(data->orig.data);
193}
194
195static void
196free_bitfield_fetch_param(struct bitfield_fetch_param *data)
197{
198 /*
199 * Don't check the bitfield itself, because this must be the
200 * last fetch function.
201 */
202 if (CHECK_FETCH_FUNCS(deref, data->orig.fn))
203 free_deref_fetch_param(data->orig.data);
204 else if (CHECK_FETCH_FUNCS(symbol, data->orig.fn))
205 free_symbol_cache(data->orig.data);
206
207 kfree(data);
208}
209
210void FETCH_FUNC_NAME(comm, string)(struct pt_regs *regs,
211 void *data, void *dest)
212{
213 int maxlen = get_rloc_len(*(u32 *)dest);
214 u8 *dst = get_rloc_data(dest);
215 long ret;
216
217 if (!maxlen)
218 return;
219
220 ret = strlcpy(dst, current->comm, maxlen);
221 *(u32 *)dest = make_data_rloc(ret, get_rloc_offs(*(u32 *)dest));
222}
223NOKPROBE_SYMBOL(FETCH_FUNC_NAME(comm, string));
224
225void FETCH_FUNC_NAME(comm, string_size)(struct pt_regs *regs,
226 void *data, void *dest)
227{
228 *(u32 *)dest = strlen(current->comm) + 1;
229}
230NOKPROBE_SYMBOL(FETCH_FUNC_NAME(comm, string_size));
231
232static const struct fetch_type *find_fetch_type(const char *type, 64static const struct fetch_type *find_fetch_type(const char *type,
233 const struct fetch_type *ftbl) 65 const struct fetch_type *ftbl)
234{ 66{
@@ -272,37 +104,6 @@ fail:
272 return NULL; 104 return NULL;
273} 105}
274 106
275/* Special function : only accept unsigned long */
276static void fetch_kernel_stack_address(struct pt_regs *regs, void *dummy, void *dest)
277{
278 *(unsigned long *)dest = kernel_stack_pointer(regs);
279}
280NOKPROBE_SYMBOL(fetch_kernel_stack_address);
281
282static void fetch_user_stack_address(struct pt_regs *regs, void *dummy, void *dest)
283{
284 *(unsigned long *)dest = user_stack_pointer(regs);
285}
286NOKPROBE_SYMBOL(fetch_user_stack_address);
287
288static fetch_func_t get_fetch_size_function(const struct fetch_type *type,
289 fetch_func_t orig_fn,
290 const struct fetch_type *ftbl)
291{
292 int i;
293
294 if (type != &ftbl[FETCH_TYPE_STRING])
295 return NULL; /* Only string type needs size function */
296
297 for (i = 0; i < FETCH_MTD_END; i++)
298 if (type->fetch[i] == orig_fn)
299 return ftbl[FETCH_TYPE_STRSIZE].fetch[i];
300
301 WARN_ON(1); /* This should not happen */
302
303 return NULL;
304}
305
306/* Split symbol and offset. */ 107/* Split symbol and offset. */
307int traceprobe_split_symbol_offset(char *symbol, long *offset) 108int traceprobe_split_symbol_offset(char *symbol, long *offset)
308{ 109{
@@ -327,7 +128,7 @@ int traceprobe_split_symbol_offset(char *symbol, long *offset)
327#define PARAM_MAX_STACK (THREAD_SIZE / sizeof(unsigned long)) 128#define PARAM_MAX_STACK (THREAD_SIZE / sizeof(unsigned long))
328 129
329static int parse_probe_vars(char *arg, const struct fetch_type *t, 130static int parse_probe_vars(char *arg, const struct fetch_type *t,
330 struct fetch_param *f, bool is_return, 131 struct fetch_insn *code, bool is_return,
331 bool is_kprobe) 132 bool is_kprobe)
332{ 133{
333 int ret = 0; 134 int ret = 0;
@@ -335,33 +136,24 @@ static int parse_probe_vars(char *arg, const struct fetch_type *t,
335 136
336 if (strcmp(arg, "retval") == 0) { 137 if (strcmp(arg, "retval") == 0) {
337 if (is_return) 138 if (is_return)
338 f->fn = t->fetch[FETCH_MTD_retval]; 139 code->op = FETCH_OP_RETVAL;
339 else 140 else
340 ret = -EINVAL; 141 ret = -EINVAL;
341 } else if (strncmp(arg, "stack", 5) == 0) { 142 } else if (strncmp(arg, "stack", 5) == 0) {
342 if (arg[5] == '\0') { 143 if (arg[5] == '\0') {
343 if (strcmp(t->name, DEFAULT_FETCH_TYPE_STR)) 144 code->op = FETCH_OP_STACKP;
344 return -EINVAL;
345
346 if (is_kprobe)
347 f->fn = fetch_kernel_stack_address;
348 else
349 f->fn = fetch_user_stack_address;
350 } else if (isdigit(arg[5])) { 145 } else if (isdigit(arg[5])) {
351 ret = kstrtoul(arg + 5, 10, &param); 146 ret = kstrtoul(arg + 5, 10, &param);
352 if (ret || (is_kprobe && param > PARAM_MAX_STACK)) 147 if (ret || (is_kprobe && param > PARAM_MAX_STACK))
353 ret = -EINVAL; 148 ret = -EINVAL;
354 else { 149 else {
355 f->fn = t->fetch[FETCH_MTD_stack]; 150 code->op = FETCH_OP_STACK;
356 f->data = (void *)param; 151 code->param = (unsigned int)param;
357 } 152 }
358 } else 153 } else
359 ret = -EINVAL; 154 ret = -EINVAL;
360 } else if (strcmp(arg, "comm") == 0) { 155 } else if (strcmp(arg, "comm") == 0) {
361 if (strcmp(t->name, "string") != 0 && 156 code->op = FETCH_OP_COMM;
362 strcmp(t->name, "string_size") != 0)
363 return -EINVAL;
364 f->fn = t->fetch[FETCH_MTD_comm];
365 } else 157 } else
366 ret = -EINVAL; 158 ret = -EINVAL;
367 159
@@ -369,10 +161,13 @@ static int parse_probe_vars(char *arg, const struct fetch_type *t,
369} 161}
370 162
371/* Recursive argument parser */ 163/* Recursive argument parser */
372static int parse_probe_arg(char *arg, const struct fetch_type *t, 164static int
373 struct fetch_param *f, bool is_return, bool is_kprobe, 165parse_probe_arg(char *arg, const struct fetch_type *type,
374 const struct fetch_type *ftbl) 166 struct fetch_insn **pcode, struct fetch_insn *end,
167 bool is_return, bool is_kprobe,
168 const struct fetch_type *ftbl)
375{ 169{
170 struct fetch_insn *code = *pcode;
376 unsigned long param; 171 unsigned long param;
377 long offset; 172 long offset;
378 char *tmp; 173 char *tmp;
@@ -380,14 +175,15 @@ static int parse_probe_arg(char *arg, const struct fetch_type *t,
380 175
381 switch (arg[0]) { 176 switch (arg[0]) {
382 case '$': 177 case '$':
383 ret = parse_probe_vars(arg + 1, t, f, is_return, is_kprobe); 178 ret = parse_probe_vars(arg + 1, type, code,
179 is_return, is_kprobe);
384 break; 180 break;
385 181
386 case '%': /* named register */ 182 case '%': /* named register */
387 ret = regs_query_register_offset(arg + 1); 183 ret = regs_query_register_offset(arg + 1);
388 if (ret >= 0) { 184 if (ret >= 0) {
389 f->fn = t->fetch[FETCH_MTD_reg]; 185 code->op = FETCH_OP_REG;
390 f->data = (void *)(unsigned long)ret; 186 code->param = (unsigned int)ret;
391 ret = 0; 187 ret = 0;
392 } 188 }
393 break; 189 break;
@@ -397,9 +193,9 @@ static int parse_probe_arg(char *arg, const struct fetch_type *t,
397 ret = kstrtoul(arg + 1, 0, &param); 193 ret = kstrtoul(arg + 1, 0, &param);
398 if (ret) 194 if (ret)
399 break; 195 break;
400 196 /* load address */
401 f->fn = t->fetch[FETCH_MTD_memory]; 197 code->op = FETCH_OP_IMM;
402 f->data = (void *)param; 198 code->immediate = param;
403 } else if (arg[1] == '+') { 199 } else if (arg[1] == '+') {
404 /* kprobes don't support file offsets */ 200 /* kprobes don't support file offsets */
405 if (is_kprobe) 201 if (is_kprobe)
@@ -409,8 +205,8 @@ static int parse_probe_arg(char *arg, const struct fetch_type *t,
409 if (ret) 205 if (ret)
410 break; 206 break;
411 207
412 f->fn = t->fetch[FETCH_MTD_file_offset]; 208 code->op = FETCH_OP_FOFFS;
413 f->data = (void *)offset; 209 code->immediate = (unsigned long)offset; // imm64?
414 } else { 210 } else {
415 /* uprobes don't support symbols */ 211 /* uprobes don't support symbols */
416 if (!is_kprobe) 212 if (!is_kprobe)
@@ -420,10 +216,19 @@ static int parse_probe_arg(char *arg, const struct fetch_type *t,
420 if (ret) 216 if (ret)
421 break; 217 break;
422 218
423 f->data = alloc_symbol_cache(arg + 1, offset); 219 code->op = FETCH_OP_IMM;
424 if (f->data) 220 code->immediate =
425 f->fn = t->fetch[FETCH_MTD_symbol]; 221 (unsigned long)kallsyms_lookup_name(arg + 1);
222 if (!code->immediate)
223 return -ENOENT;
224 code->immediate += offset;
426 } 225 }
226 /* These are fetching from memory */
227 if (++code == end)
228 return -E2BIG;
229 *pcode = code;
230 code->op = FETCH_OP_DEREF;
231 code->offset = offset;
427 break; 232 break;
428 233
429 case '+': /* deref memory */ 234 case '+': /* deref memory */
@@ -431,11 +236,10 @@ static int parse_probe_arg(char *arg, const struct fetch_type *t,
431 case '-': 236 case '-':
432 tmp = strchr(arg, '('); 237 tmp = strchr(arg, '(');
433 if (!tmp) 238 if (!tmp)
434 break; 239 return -EINVAL;
435 240
436 *tmp = '\0'; 241 *tmp = '\0';
437 ret = kstrtol(arg, 0, &offset); 242 ret = kstrtol(arg, 0, &offset);
438
439 if (ret) 243 if (ret)
440 break; 244 break;
441 245
@@ -443,36 +247,29 @@ static int parse_probe_arg(char *arg, const struct fetch_type *t,
443 tmp = strrchr(arg, ')'); 247 tmp = strrchr(arg, ')');
444 248
445 if (tmp) { 249 if (tmp) {
446 struct deref_fetch_param *dprm; 250 const struct fetch_type *t2;
447 const struct fetch_type *t2;
448 251
449 t2 = find_fetch_type(NULL, ftbl); 252 t2 = find_fetch_type(NULL, ftbl);
450 *tmp = '\0'; 253 *tmp = '\0';
451 dprm = kzalloc(sizeof(struct deref_fetch_param), GFP_KERNEL); 254 ret = parse_probe_arg(arg, t2, &code, end, is_return,
452 255 is_kprobe, ftbl);
453 if (!dprm)
454 return -ENOMEM;
455
456 dprm->offset = offset;
457 dprm->fetch = t->fetch[FETCH_MTD_memory];
458 dprm->fetch_size = get_fetch_size_function(t,
459 dprm->fetch, ftbl);
460 ret = parse_probe_arg(arg, t2, &dprm->orig, is_return,
461 is_kprobe, ftbl);
462 if (ret) 256 if (ret)
463 kfree(dprm); 257 break;
464 else { 258 if (code->op == FETCH_OP_COMM)
465 f->fn = t->fetch[FETCH_MTD_deref]; 259 return -EINVAL;
466 f->data = (void *)dprm; 260 if (++code == end)
467 } 261 return -E2BIG;
262 *pcode = code;
263
264 code->op = FETCH_OP_DEREF;
265 code->offset = offset;
468 } 266 }
469 break; 267 break;
470 } 268 }
471 if (!ret && !f->fn) { /* Parsed, but do not find fetch method */ 269 if (!ret && code->op == FETCH_OP_NOP) {
472 pr_info("%s type has no corresponding fetch method.\n", t->name); 270 /* Parsed, but do not find fetch method */
473 ret = -EINVAL; 271 ret = -EINVAL;
474 } 272 }
475
476 return ret; 273 return ret;
477} 274}
478 275
@@ -481,22 +278,15 @@ static int parse_probe_arg(char *arg, const struct fetch_type *t,
481/* Bitfield type needs to be parsed into a fetch function */ 278/* Bitfield type needs to be parsed into a fetch function */
482static int __parse_bitfield_probe_arg(const char *bf, 279static int __parse_bitfield_probe_arg(const char *bf,
483 const struct fetch_type *t, 280 const struct fetch_type *t,
484 struct fetch_param *f) 281 struct fetch_insn **pcode)
485{ 282{
486 struct bitfield_fetch_param *bprm; 283 struct fetch_insn *code = *pcode;
487 unsigned long bw, bo; 284 unsigned long bw, bo;
488 char *tail; 285 char *tail;
489 286
490 if (*bf != 'b') 287 if (*bf != 'b')
491 return 0; 288 return 0;
492 289
493 bprm = kzalloc(sizeof(*bprm), GFP_KERNEL);
494 if (!bprm)
495 return -ENOMEM;
496
497 bprm->orig = *f;
498 f->fn = t->fetch[FETCH_MTD_bitfield];
499 f->data = (void *)bprm;
500 bw = simple_strtoul(bf + 1, &tail, 0); /* Use simple one */ 290 bw = simple_strtoul(bf + 1, &tail, 0); /* Use simple one */
501 291
502 if (bw == 0 || *tail != '@') 292 if (bw == 0 || *tail != '@')
@@ -507,9 +297,15 @@ static int __parse_bitfield_probe_arg(const char *bf,
507 297
508 if (tail == bf || *tail != '/') 298 if (tail == bf || *tail != '/')
509 return -EINVAL; 299 return -EINVAL;
300 code++;
301 if (code->op != FETCH_OP_NOP)
302 return -E2BIG;
303 *pcode = code;
510 304
511 bprm->hi_shift = BYTES_TO_BITS(t->size) - (bw + bo); 305 code->op = FETCH_OP_MOD_BF;
512 bprm->low_shift = bprm->hi_shift + bo; 306 code->lshift = BYTES_TO_BITS(t->size) - (bw + bo);
307 code->rshift = BYTES_TO_BITS(t->size) - bw;
308 code->basesize = t->size;
513 309
514 return (BYTES_TO_BITS(t->size) < (bw + bo)) ? -EINVAL : 0; 310 return (BYTES_TO_BITS(t->size) < (bw + bo)) ? -EINVAL : 0;
515} 311}
@@ -519,6 +315,7 @@ int traceprobe_parse_probe_arg(char *arg, ssize_t *size,
519 struct probe_arg *parg, bool is_return, bool is_kprobe, 315 struct probe_arg *parg, bool is_return, bool is_kprobe,
520 const struct fetch_type *ftbl) 316 const struct fetch_type *ftbl)
521{ 317{
318 struct fetch_insn *code, *tmp = NULL;
522 const char *t; 319 const char *t;
523 int ret; 320 int ret;
524 321
@@ -549,18 +346,60 @@ int traceprobe_parse_probe_arg(char *arg, ssize_t *size,
549 } 346 }
550 parg->offset = *size; 347 parg->offset = *size;
551 *size += parg->type->size; 348 *size += parg->type->size;
552 ret = parse_probe_arg(arg, parg->type, &parg->fetch, is_return,
553 is_kprobe, ftbl);
554
555 if (ret >= 0 && t != NULL)
556 ret = __parse_bitfield_probe_arg(t, parg->type, &parg->fetch);
557 349
558 if (ret >= 0) { 350 code = tmp = kzalloc(sizeof(*code) * FETCH_INSN_MAX, GFP_KERNEL);
559 parg->fetch_size.fn = get_fetch_size_function(parg->type, 351 if (!code)
560 parg->fetch.fn, 352 return -ENOMEM;
561 ftbl); 353 code[FETCH_INSN_MAX - 1].op = FETCH_OP_END;
562 parg->fetch_size.data = parg->fetch.data; 354
355 ret = parse_probe_arg(arg, parg->type, &code, &code[FETCH_INSN_MAX - 1],
356 is_return, is_kprobe, ftbl);
357 if (ret)
358 goto fail;
359
360 /* Store operation */
361 if (!strcmp(parg->type->name, "string")) {
362 if (code->op != FETCH_OP_DEREF && code->op != FETCH_OP_IMM &&
363 code->op != FETCH_OP_COMM) {
364 pr_info("string only accepts memory or address.\n");
365 ret = -EINVAL;
366 goto fail;
367 }
368 /* Since IMM or COMM must be the 1st insn, this is safe */
369 if (code->op == FETCH_OP_IMM || code->op == FETCH_OP_COMM)
370 code++;
371 code->op = FETCH_OP_ST_STRING; /* In DEREF case, replace it */
372 parg->dynamic = true;
373 } else if (code->op == FETCH_OP_DEREF) {
374 code->op = FETCH_OP_ST_MEM;
375 code->size = parg->type->size;
376 } else {
377 code++;
378 if (code->op != FETCH_OP_NOP) {
379 ret = -E2BIG;
380 goto fail;
381 }
382 code->op = FETCH_OP_ST_RAW;
383 code->size = parg->type->size;
384 }
385 /* Modify operation */
386 if (t != NULL) {
387 ret = __parse_bitfield_probe_arg(t, parg->type, &code);
388 if (ret)
389 goto fail;
563 } 390 }
391 code++;
392 code->op = FETCH_OP_END;
393
394 /* Shrink down the code buffer */
395 parg->code = kzalloc(sizeof(*code) * (code - tmp + 1), GFP_KERNEL);
396 if (!parg->code)
397 ret = -ENOMEM;
398 else
399 memcpy(parg->code, tmp, sizeof(*code) * (code - tmp + 1));
400
401fail:
402 kfree(tmp);
564 403
565 return ret; 404 return ret;
566} 405}
@@ -582,25 +421,9 @@ int traceprobe_conflict_field_name(const char *name,
582 return 0; 421 return 0;
583} 422}
584 423
585void traceprobe_update_arg(struct probe_arg *arg)
586{
587 if (CHECK_FETCH_FUNCS(bitfield, arg->fetch.fn))
588 update_bitfield_fetch_param(arg->fetch.data);
589 else if (CHECK_FETCH_FUNCS(deref, arg->fetch.fn))
590 update_deref_fetch_param(arg->fetch.data);
591 else if (CHECK_FETCH_FUNCS(symbol, arg->fetch.fn))
592 update_symbol_cache(arg->fetch.data);
593}
594
595void traceprobe_free_probe_arg(struct probe_arg *arg) 424void traceprobe_free_probe_arg(struct probe_arg *arg)
596{ 425{
597 if (CHECK_FETCH_FUNCS(bitfield, arg->fetch.fn)) 426 kfree(arg->code);
598 free_bitfield_fetch_param(arg->fetch.data);
599 else if (CHECK_FETCH_FUNCS(deref, arg->fetch.fn))
600 free_deref_fetch_param(arg->fetch.data);
601 else if (CHECK_FETCH_FUNCS(symbol, arg->fetch.fn))
602 free_symbol_cache(arg->fetch.data);
603
604 kfree(arg->name); 427 kfree(arg->name);
605 kfree(arg->comm); 428 kfree(arg->comm);
606} 429}
diff --git a/kernel/trace/trace_probe.h b/kernel/trace/trace_probe.h
index a1df7763b797..42c724a7ad11 100644
--- a/kernel/trace/trace_probe.h
+++ b/kernel/trace/trace_probe.h
@@ -79,25 +79,50 @@ static nokprobe_inline void *get_loc_data(u32 *dl, void *ent)
79 return (u8 *)ent + get_rloc_offs(*dl); 79 return (u8 *)ent + get_rloc_offs(*dl);
80} 80}
81 81
82/* Data fetch function type */
83typedef void (*fetch_func_t)(struct pt_regs *, void *, void *);
84/* Printing function type */ 82/* Printing function type */
85typedef int (*print_type_func_t)(struct trace_seq *, void *, void *); 83typedef int (*print_type_func_t)(struct trace_seq *, void *, void *);
86 84
87/* Fetch types */ 85enum fetch_op {
88enum { 86 FETCH_OP_NOP = 0,
89 FETCH_MTD_reg = 0, 87 // Stage 1 (load) ops
90 FETCH_MTD_stack, 88 FETCH_OP_REG, /* Register : .param = offset */
91 FETCH_MTD_retval, 89 FETCH_OP_STACK, /* Stack : .param = index */
92 FETCH_MTD_comm, 90 FETCH_OP_STACKP, /* Stack pointer */
93 FETCH_MTD_memory, 91 FETCH_OP_RETVAL, /* Return value */
94 FETCH_MTD_symbol, 92 FETCH_OP_IMM, /* Immediate : .immediate */
95 FETCH_MTD_deref, 93 FETCH_OP_COMM, /* Current comm */
96 FETCH_MTD_bitfield, 94 FETCH_OP_FOFFS, /* File offset: .immediate */
97 FETCH_MTD_file_offset, 95 // Stage 2 (dereference) op
98 FETCH_MTD_END, 96 FETCH_OP_DEREF, /* Dereference: .offset */
97 // Stage 3 (store) ops
98 FETCH_OP_ST_RAW, /* Raw: .size */
99 FETCH_OP_ST_MEM, /* Mem: .offset, .size */
100 FETCH_OP_ST_STRING, /* String: .offset, .size */
101 // Stage 4 (modify) op
102 FETCH_OP_MOD_BF, /* Bitfield: .basesize, .lshift, .rshift */
103 FETCH_OP_END,
99}; 104};
100 105
106struct fetch_insn {
107 enum fetch_op op;
108 union {
109 unsigned int param;
110 struct {
111 unsigned int size;
112 int offset;
113 };
114 struct {
115 unsigned char basesize;
116 unsigned char lshift;
117 unsigned char rshift;
118 };
119 unsigned long immediate;
120 };
121};
122
123/* fetch + deref*N + store + mod + end <= 16, this allows N=12, enough */
124#define FETCH_INSN_MAX 16
125
101/* Fetch type information table */ 126/* Fetch type information table */
102struct fetch_type { 127struct fetch_type {
103 const char *name; /* Name of type */ 128 const char *name; /* Name of type */
@@ -106,13 +131,6 @@ struct fetch_type {
106 print_type_func_t print; /* Print functions */ 131 print_type_func_t print; /* Print functions */
107 const char *fmt; /* Fromat string */ 132 const char *fmt; /* Fromat string */
108 const char *fmttype; /* Name in format file */ 133 const char *fmttype; /* Name in format file */
109 /* Fetch functions */
110 fetch_func_t fetch[FETCH_MTD_END];
111};
112
113struct fetch_param {
114 fetch_func_t fn;
115 void *data;
116}; 134};
117 135
118/* For defining macros, define string/string_size types */ 136/* For defining macros, define string/string_size types */
@@ -142,66 +160,12 @@ DECLARE_BASIC_PRINT_TYPE_FUNC(x64);
142 160
143DECLARE_BASIC_PRINT_TYPE_FUNC(string); 161DECLARE_BASIC_PRINT_TYPE_FUNC(string);
144 162
145#define FETCH_FUNC_NAME(method, type) fetch_##method##_##type
146
147/* Declare macro for basic types */
148#define DECLARE_FETCH_FUNC(method, type) \
149extern void FETCH_FUNC_NAME(method, type)(struct pt_regs *regs, \
150 void *data, void *dest)
151
152#define DECLARE_BASIC_FETCH_FUNCS(method) \
153DECLARE_FETCH_FUNC(method, u8); \
154DECLARE_FETCH_FUNC(method, u16); \
155DECLARE_FETCH_FUNC(method, u32); \
156DECLARE_FETCH_FUNC(method, u64)
157
158DECLARE_BASIC_FETCH_FUNCS(reg);
159#define fetch_reg_string NULL
160#define fetch_reg_string_size NULL
161
162DECLARE_BASIC_FETCH_FUNCS(retval);
163#define fetch_retval_string NULL
164#define fetch_retval_string_size NULL
165
166DECLARE_BASIC_FETCH_FUNCS(symbol);
167DECLARE_FETCH_FUNC(symbol, string);
168DECLARE_FETCH_FUNC(symbol, string_size);
169
170DECLARE_BASIC_FETCH_FUNCS(deref);
171DECLARE_FETCH_FUNC(deref, string);
172DECLARE_FETCH_FUNC(deref, string_size);
173
174DECLARE_BASIC_FETCH_FUNCS(bitfield);
175#define fetch_bitfield_string NULL
176#define fetch_bitfield_string_size NULL
177
178/* comm only makes sense as a string */
179#define fetch_comm_u8 NULL
180#define fetch_comm_u16 NULL
181#define fetch_comm_u32 NULL
182#define fetch_comm_u64 NULL
183DECLARE_FETCH_FUNC(comm, string);
184DECLARE_FETCH_FUNC(comm, string_size);
185
186/*
187 * Define macro for basic types - we don't need to define s* types, because
188 * we have to care only about bitwidth at recording time.
189 */
190#define DEFINE_BASIC_FETCH_FUNCS(method) \
191DEFINE_FETCH_##method(u8) \
192DEFINE_FETCH_##method(u16) \
193DEFINE_FETCH_##method(u32) \
194DEFINE_FETCH_##method(u64)
195
196/* Default (unsigned long) fetch type */ 163/* Default (unsigned long) fetch type */
197#define __DEFAULT_FETCH_TYPE(t) x##t 164#define __DEFAULT_FETCH_TYPE(t) x##t
198#define _DEFAULT_FETCH_TYPE(t) __DEFAULT_FETCH_TYPE(t) 165#define _DEFAULT_FETCH_TYPE(t) __DEFAULT_FETCH_TYPE(t)
199#define DEFAULT_FETCH_TYPE _DEFAULT_FETCH_TYPE(BITS_PER_LONG) 166#define DEFAULT_FETCH_TYPE _DEFAULT_FETCH_TYPE(BITS_PER_LONG)
200#define DEFAULT_FETCH_TYPE_STR __stringify(DEFAULT_FETCH_TYPE) 167#define DEFAULT_FETCH_TYPE_STR __stringify(DEFAULT_FETCH_TYPE)
201 168
202#define ASSIGN_FETCH_FUNC(method, type) \
203 [FETCH_MTD_##method] = FETCH_FUNC_NAME(method, type)
204
205#define __ASSIGN_FETCH_TYPE(_name, ptype, ftype, _size, sign, _fmttype) \ 169#define __ASSIGN_FETCH_TYPE(_name, ptype, ftype, _size, sign, _fmttype) \
206 {.name = _name, \ 170 {.name = _name, \
207 .size = _size, \ 171 .size = _size, \
@@ -209,17 +173,6 @@ DEFINE_FETCH_##method(u64)
209 .print = PRINT_TYPE_FUNC_NAME(ptype), \ 173 .print = PRINT_TYPE_FUNC_NAME(ptype), \
210 .fmt = PRINT_TYPE_FMT_NAME(ptype), \ 174 .fmt = PRINT_TYPE_FMT_NAME(ptype), \
211 .fmttype = _fmttype, \ 175 .fmttype = _fmttype, \
212 .fetch = { \
213ASSIGN_FETCH_FUNC(reg, ftype), \
214ASSIGN_FETCH_FUNC(stack, ftype), \
215ASSIGN_FETCH_FUNC(retval, ftype), \
216ASSIGN_FETCH_FUNC(comm, ftype), \
217ASSIGN_FETCH_FUNC(memory, ftype), \
218ASSIGN_FETCH_FUNC(symbol, ftype), \
219ASSIGN_FETCH_FUNC(deref, ftype), \
220ASSIGN_FETCH_FUNC(bitfield, ftype), \
221ASSIGN_FETCH_FUNC(file_offset, ftype), \
222 } \
223 } 176 }
224 177
225#define ASSIGN_FETCH_TYPE(ptype, ftype, sign) \ 178#define ASSIGN_FETCH_TYPE(ptype, ftype, sign) \
@@ -231,42 +184,13 @@ ASSIGN_FETCH_FUNC(file_offset, ftype), \
231 184
232#define ASSIGN_FETCH_TYPE_END {} 185#define ASSIGN_FETCH_TYPE_END {}
233 186
234#define FETCH_TYPE_STRING 0 187#define FETCH_TYPE_STRING 0
235#define FETCH_TYPE_STRSIZE 1 188#define FETCH_TYPE_STRSIZE 1
236 189
237#ifdef CONFIG_KPROBE_EVENTS 190#ifdef CONFIG_KPROBE_EVENTS
238struct symbol_cache;
239unsigned long update_symbol_cache(struct symbol_cache *sc);
240void free_symbol_cache(struct symbol_cache *sc);
241struct symbol_cache *alloc_symbol_cache(const char *sym, long offset);
242bool trace_kprobe_on_func_entry(struct trace_event_call *call); 191bool trace_kprobe_on_func_entry(struct trace_event_call *call);
243bool trace_kprobe_error_injectable(struct trace_event_call *call); 192bool trace_kprobe_error_injectable(struct trace_event_call *call);
244#else 193#else
245/* uprobes do not support symbol fetch methods */
246#define fetch_symbol_u8 NULL
247#define fetch_symbol_u16 NULL
248#define fetch_symbol_u32 NULL
249#define fetch_symbol_u64 NULL
250#define fetch_symbol_string NULL
251#define fetch_symbol_string_size NULL
252
253struct symbol_cache {
254};
255static inline unsigned long __used update_symbol_cache(struct symbol_cache *sc)
256{
257 return 0;
258}
259
260static inline void __used free_symbol_cache(struct symbol_cache *sc)
261{
262}
263
264static inline struct symbol_cache * __used
265alloc_symbol_cache(const char *sym, long offset)
266{
267 return NULL;
268}
269
270static inline bool trace_kprobe_on_func_entry(struct trace_event_call *call) 194static inline bool trace_kprobe_on_func_entry(struct trace_event_call *call)
271{ 195{
272 return false; 196 return false;
@@ -279,8 +203,8 @@ static inline bool trace_kprobe_error_injectable(struct trace_event_call *call)
279#endif /* CONFIG_KPROBE_EVENTS */ 203#endif /* CONFIG_KPROBE_EVENTS */
280 204
281struct probe_arg { 205struct probe_arg {
282 struct fetch_param fetch; 206 struct fetch_insn *code;
283 struct fetch_param fetch_size; 207 bool dynamic;/* Dynamic array (string) is used */
284 unsigned int offset; /* Offset from argument entry */ 208 unsigned int offset; /* Offset from argument entry */
285 const char *name; /* Name of this argument */ 209 const char *name; /* Name of this argument */
286 const char *comm; /* Command of this argument */ 210 const char *comm; /* Command of this argument */
@@ -312,12 +236,6 @@ static inline bool trace_probe_is_registered(struct trace_probe *tp)
312 return !!(tp->flags & TP_FLAG_REGISTERED); 236 return !!(tp->flags & TP_FLAG_REGISTERED);
313} 237}
314 238
315static nokprobe_inline void call_fetch(struct fetch_param *fprm,
316 struct pt_regs *regs, void *dest)
317{
318 return fprm->fn(regs, fprm->data, dest);
319}
320
321/* Check the name is good for event/group/fields */ 239/* Check the name is good for event/group/fields */
322static inline bool is_good_name(const char *name) 240static inline bool is_good_name(const char *name)
323{ 241{
@@ -354,68 +272,6 @@ extern void traceprobe_free_probe_arg(struct probe_arg *arg);
354 272
355extern int traceprobe_split_symbol_offset(char *symbol, long *offset); 273extern int traceprobe_split_symbol_offset(char *symbol, long *offset);
356 274
357/* Sum up total data length for dynamic arraies (strings) */
358static nokprobe_inline int
359__get_data_size(struct trace_probe *tp, struct pt_regs *regs)
360{
361 int i, ret = 0;
362 u32 len;
363
364 for (i = 0; i < tp->nr_args; i++)
365 if (unlikely(tp->args[i].fetch_size.fn)) {
366 call_fetch(&tp->args[i].fetch_size, regs, &len);
367 ret += len;
368 }
369
370 return ret;
371}
372
373/* Store the value of each argument */
374static nokprobe_inline void
375store_trace_args(int ent_size, struct trace_probe *tp, struct pt_regs *regs,
376 u8 *data, int maxlen)
377{
378 int i;
379 u32 end = tp->size;
380 u32 *dl; /* Data (relative) location */
381
382 for (i = 0; i < tp->nr_args; i++) {
383 if (unlikely(tp->args[i].fetch_size.fn)) {
384 /*
385 * First, we set the relative location and
386 * maximum data length to *dl
387 */
388 dl = (u32 *)(data + tp->args[i].offset);
389 *dl = make_data_rloc(maxlen, end - tp->args[i].offset);
390 /* Then try to fetch string or dynamic array data */
391 call_fetch(&tp->args[i].fetch, regs, dl);
392 /* Reduce maximum length */
393 end += get_rloc_len(*dl);
394 maxlen -= get_rloc_len(*dl);
395 /* Trick here, convert data_rloc to data_loc */
396 *dl = convert_rloc_to_loc(*dl,
397 ent_size + tp->args[i].offset);
398 } else
399 /* Just fetching data normally */
400 call_fetch(&tp->args[i].fetch, regs,
401 data + tp->args[i].offset);
402 }
403}
404
405static inline int
406print_probe_args(struct trace_seq *s, struct probe_arg *args, int nr_args,
407 u8 *data, void *field)
408{
409 int i;
410
411 for (i = 0; i < nr_args; i++) {
412 trace_seq_printf(s, " %s=", args[i].name);
413 if (!args[i].type->print(s, data + args[i].offset, field))
414 return -ENOMEM;
415 }
416 return 0;
417}
418
419extern int set_print_fmt(struct trace_probe *tp, bool is_return); 275extern int set_print_fmt(struct trace_probe *tp, bool is_return);
420 276
421#ifdef CONFIG_PERF_EVENTS 277#ifdef CONFIG_PERF_EVENTS
diff --git a/kernel/trace/trace_probe_tmpl.h b/kernel/trace/trace_probe_tmpl.h
new file mode 100644
index 000000000000..c8a5272abf01
--- /dev/null
+++ b/kernel/trace/trace_probe_tmpl.h
@@ -0,0 +1,120 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Traceprobe fetch helper inlines
4 */
5
6static nokprobe_inline void
7fetch_store_raw(unsigned long val, struct fetch_insn *code, void *buf)
8{
9 switch (code->size) {
10 case 1:
11 *(u8 *)buf = (u8)val;
12 break;
13 case 2:
14 *(u16 *)buf = (u16)val;
15 break;
16 case 4:
17 *(u32 *)buf = (u32)val;
18 break;
19 case 8:
20 //TBD: 32bit signed
21 *(u64 *)buf = (u64)val;
22 break;
23 default:
24 *(unsigned long *)buf = val;
25 }
26}
27
28static nokprobe_inline void
29fetch_apply_bitfield(struct fetch_insn *code, void *buf)
30{
31 switch (code->basesize) {
32 case 1:
33 *(u8 *)buf <<= code->lshift;
34 *(u8 *)buf >>= code->rshift;
35 break;
36 case 2:
37 *(u16 *)buf <<= code->lshift;
38 *(u16 *)buf >>= code->rshift;
39 break;
40 case 4:
41 *(u32 *)buf <<= code->lshift;
42 *(u32 *)buf >>= code->rshift;
43 break;
44 case 8:
45 *(u64 *)buf <<= code->lshift;
46 *(u64 *)buf >>= code->rshift;
47 break;
48 }
49}
50
51/* Define this for each callsite */
52static int
53process_fetch_insn(struct fetch_insn *code, struct pt_regs *regs,
54 void *dest, bool pre);
55
56/* Sum up total data length for dynamic arraies (strings) */
57static nokprobe_inline int
58__get_data_size(struct trace_probe *tp, struct pt_regs *regs)
59{
60 struct probe_arg *arg;
61 int i, ret = 0;
62 u32 len;
63
64 for (i = 0; i < tp->nr_args; i++) {
65 arg = tp->args + i;
66 if (unlikely(arg->dynamic)) {
67 process_fetch_insn(arg->code, regs, &len, true);
68 ret += len;
69 }
70 }
71
72 return ret;
73}
74
75/* Store the value of each argument */
76static nokprobe_inline void
77store_trace_args(int ent_size, struct trace_probe *tp, struct pt_regs *regs,
78 u8 *data, int maxlen)
79{
80 struct probe_arg *arg;
81 u32 end = tp->size;
82 u32 *dl; /* Data (relative) location */
83 int i;
84
85 for (i = 0; i < tp->nr_args; i++) {
86 arg = tp->args + i;
87 if (unlikely(arg->dynamic)) {
88 /*
89 * First, we set the relative location and
90 * maximum data length to *dl
91 */
92 dl = (u32 *)(data + arg->offset);
93 *dl = make_data_rloc(maxlen, end - arg->offset);
94 /* Then try to fetch string or dynamic array data */
95 process_fetch_insn(arg->code, regs, dl, false);
96 /* Reduce maximum length */
97 end += get_rloc_len(*dl);
98 maxlen -= get_rloc_len(*dl);
99 /* Trick here, convert data_rloc to data_loc */
100 *dl = convert_rloc_to_loc(*dl, ent_size + arg->offset);
101 } else
102 /* Just fetching data normally */
103 process_fetch_insn(arg->code, regs, data + arg->offset,
104 false);
105 }
106}
107
108static inline int
109print_probe_args(struct trace_seq *s, struct probe_arg *args, int nr_args,
110 u8 *data, void *field)
111{
112 int i;
113
114 for (i = 0; i < nr_args; i++) {
115 trace_seq_printf(s, " %s=", args[i].name);
116 if (!args[i].type->print(s, data + args[i].offset, field))
117 return -ENOMEM;
118 }
119 return 0;
120}
diff --git a/kernel/trace/trace_uprobe.c b/kernel/trace/trace_uprobe.c
index 28a8f69cec89..e076f89ab33a 100644
--- a/kernel/trace/trace_uprobe.c
+++ b/kernel/trace/trace_uprobe.c
@@ -15,6 +15,7 @@
15#include <linux/rculist.h> 15#include <linux/rculist.h>
16 16
17#include "trace_probe.h" 17#include "trace_probe.h"
18#include "trace_probe_tmpl.h"
18 19
19#define UPROBE_EVENT_SYSTEM "uprobes" 20#define UPROBE_EVENT_SYSTEM "uprobes"
20 21
@@ -99,37 +100,19 @@ static unsigned long get_user_stack_nth(struct pt_regs *regs, unsigned int n)
99/* 100/*
100 * Uprobes-specific fetch functions 101 * Uprobes-specific fetch functions
101 */ 102 */
102#define DEFINE_FETCH_stack(type) \ 103static nokprobe_inline int
103static void FETCH_FUNC_NAME(stack, type)(struct pt_regs *regs, \ 104probe_user_read(void *dest, void *src, size_t size)
104 void *offset, void *dest) \ 105{
105{ \ 106 void __user *vaddr = (void __force __user *)src;
106 *(type *)dest = (type)get_user_stack_nth(regs, \ 107
107 ((unsigned long)offset)); \ 108 return copy_from_user(dest, vaddr, size);
108}
109DEFINE_BASIC_FETCH_FUNCS(stack)
110/* No string on the stack entry */
111#define fetch_stack_string NULL
112#define fetch_stack_string_size NULL
113
114#define DEFINE_FETCH_memory(type) \
115static void FETCH_FUNC_NAME(memory, type)(struct pt_regs *regs, \
116 void *addr, void *dest) \
117{ \
118 type retval; \
119 void __user *vaddr = (void __force __user *) addr; \
120 \
121 if (copy_from_user(&retval, vaddr, sizeof(type))) \
122 *(type *)dest = 0; \
123 else \
124 *(type *) dest = retval; \
125} 109}
126DEFINE_BASIC_FETCH_FUNCS(memory)
127/* 110/*
128 * Fetch a null-terminated string. Caller MUST set *(u32 *)dest with max 111 * Fetch a null-terminated string. Caller MUST set *(u32 *)dest with max
129 * length and relative data location. 112 * length and relative data location.
130 */ 113 */
131static void FETCH_FUNC_NAME(memory, string)(struct pt_regs *regs, 114static nokprobe_inline void
132 void *addr, void *dest) 115fetch_store_string(unsigned long addr, void *dest)
133{ 116{
134 long ret; 117 long ret;
135 u32 rloc = *(u32 *)dest; 118 u32 rloc = *(u32 *)dest;
@@ -152,8 +135,9 @@ static void FETCH_FUNC_NAME(memory, string)(struct pt_regs *regs,
152 } 135 }
153} 136}
154 137
155static void FETCH_FUNC_NAME(memory, string_size)(struct pt_regs *regs, 138/* Return the length of string -- including null terminal byte */
156 void *addr, void *dest) 139static nokprobe_inline void
140fetch_store_strlen(unsigned long addr, void *dest)
157{ 141{
158 int len; 142 int len;
159 void __user *vaddr = (void __force __user *) addr; 143 void __user *vaddr = (void __force __user *) addr;
@@ -166,7 +150,7 @@ static void FETCH_FUNC_NAME(memory, string_size)(struct pt_regs *regs,
166 *(u32 *)dest = len; 150 *(u32 *)dest = len;
167} 151}
168 152
169static unsigned long translate_user_vaddr(void *file_offset) 153static unsigned long translate_user_vaddr(unsigned long file_offset)
170{ 154{
171 unsigned long base_addr; 155 unsigned long base_addr;
172 struct uprobe_dispatch_data *udd; 156 struct uprobe_dispatch_data *udd;
@@ -174,21 +158,9 @@ static unsigned long translate_user_vaddr(void *file_offset)
174 udd = (void *) current->utask->vaddr; 158 udd = (void *) current->utask->vaddr;
175 159
176 base_addr = udd->bp_addr - udd->tu->offset; 160 base_addr = udd->bp_addr - udd->tu->offset;
177 return base_addr + (unsigned long)file_offset; 161 return base_addr + file_offset;
178} 162}
179 163
180#define DEFINE_FETCH_file_offset(type) \
181static void FETCH_FUNC_NAME(file_offset, type)(struct pt_regs *regs, \
182 void *offset, void *dest)\
183{ \
184 void *vaddr = (void *)translate_user_vaddr(offset); \
185 \
186 FETCH_FUNC_NAME(memory, type)(regs, vaddr, dest); \
187}
188DEFINE_BASIC_FETCH_FUNCS(file_offset)
189DEFINE_FETCH_file_offset(string)
190DEFINE_FETCH_file_offset(string_size)
191
192/* Fetch type information table */ 164/* Fetch type information table */
193static const struct fetch_type uprobes_fetch_type_table[] = { 165static const struct fetch_type uprobes_fetch_type_table[] = {
194 /* Special types */ 166 /* Special types */
@@ -213,6 +185,77 @@ static const struct fetch_type uprobes_fetch_type_table[] = {
213 ASSIGN_FETCH_TYPE_END 185 ASSIGN_FETCH_TYPE_END
214}; 186};
215 187
188/* Note that we don't verify it, since the code does not come from user space */
189static int
190process_fetch_insn(struct fetch_insn *code, struct pt_regs *regs, void *dest,
191 bool pre)
192{
193 unsigned long val;
194 int ret;
195
196 /* 1st stage: get value from context */
197 switch (code->op) {
198 case FETCH_OP_REG:
199 val = regs_get_register(regs, code->param);
200 break;
201 case FETCH_OP_STACK:
202 val = get_user_stack_nth(regs, code->param);
203 break;
204 case FETCH_OP_STACKP:
205 val = user_stack_pointer(regs);
206 break;
207 case FETCH_OP_RETVAL:
208 val = regs_return_value(regs);
209 break;
210 case FETCH_OP_IMM:
211 val = code->immediate;
212 break;
213 case FETCH_OP_FOFFS:
214 val = translate_user_vaddr(code->immediate);
215 break;
216 default:
217 return -EILSEQ;
218 }
219 code++;
220
221 /* 2nd stage: dereference memory if needed */
222 while (code->op == FETCH_OP_DEREF) {
223 ret = probe_user_read(&val, (void *)val + code->offset,
224 sizeof(val));
225 if (ret)
226 return ret;
227 code++;
228 }
229
230 /* 3rd stage: store value to buffer */
231 switch (code->op) {
232 case FETCH_OP_ST_RAW:
233 fetch_store_raw(val, code, dest);
234 break;
235 case FETCH_OP_ST_MEM:
236 probe_user_read(dest, (void *)val + code->offset, code->size);
237 break;
238 case FETCH_OP_ST_STRING:
239 if (pre)
240 fetch_store_strlen(val + code->offset, dest);
241 else
242 fetch_store_string(val + code->offset, dest);
243 break;
244 default:
245 return -EILSEQ;
246 }
247 code++;
248
249 /* 4th stage: modify stored value if needed */
250 if (code->op == FETCH_OP_MOD_BF) {
251 fetch_apply_bitfield(code, dest);
252 code++;
253 }
254
255 return code->op == FETCH_OP_END ? 0 : -EILSEQ;
256}
257NOKPROBE_SYMBOL(process_fetch_insn)
258
216static inline void init_trace_uprobe_filter(struct trace_uprobe_filter *filter) 259static inline void init_trace_uprobe_filter(struct trace_uprobe_filter *filter)
217{ 260{
218 rwlock_init(&filter->rwlock); 261 rwlock_init(&filter->rwlock);
generated by cgit v1.2.2 (git 2.25.0) at 2025-08-24 01:33:31 -0400