summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Documentation/trace/kprobetrace.rst23
-rw-r--r--arch/Kconfig7
-rw-r--r--arch/x86/Kconfig1
-rw-r--r--arch/x86/include/asm/ptrace.h38
-rw-r--r--include/linux/trace_events.h3
-rw-r--r--include/linux/uprobes.h5
-rw-r--r--kernel/events/core.c49
-rw-r--r--kernel/events/uprobes.c278
-rw-r--r--kernel/trace/trace.c12
-rw-r--r--kernel/trace/trace_event_perf.c7
-rw-r--r--kernel/trace/trace_kprobe.c412
-rw-r--r--kernel/trace/trace_probe.c672
-rw-r--r--kernel/trace/trace_probe.h289
-rw-r--r--kernel/trace/trace_probe_tmpl.h216
-rw-r--r--kernel/trace/trace_stack.c2
-rw-r--r--kernel/trace/trace_uprobe.c255
-rw-r--r--tools/perf/util/probe-event.c39
-rw-r--r--tools/perf/util/probe-event.h1
-rw-r--r--tools/perf/util/probe-file.c34
-rw-r--r--tools/perf/util/probe-file.h1
-rw-r--r--tools/perf/util/symbol-elf.c46
-rw-r--r--tools/perf/util/symbol.h7
22 files changed, 1439 insertions, 958 deletions
diff --git a/Documentation/trace/kprobetrace.rst b/Documentation/trace/kprobetrace.rst
index 8bfc75c90806..47e765c2f2c3 100644
--- a/Documentation/trace/kprobetrace.rst
+++ b/Documentation/trace/kprobetrace.rst
@@ -45,16 +45,18 @@ Synopsis of kprobe_events
45 @SYM[+|-offs] : Fetch memory at SYM +|- offs (SYM should be a data symbol) 45 @SYM[+|-offs] : Fetch memory at SYM +|- offs (SYM should be a data symbol)
46 $stackN : Fetch Nth entry of stack (N >= 0) 46 $stackN : Fetch Nth entry of stack (N >= 0)
47 $stack : Fetch stack address. 47 $stack : Fetch stack address.
48 $retval : Fetch return value.(*) 48 $argN : Fetch the Nth function argument. (N >= 1) (\*1)
49 $retval : Fetch return value.(\*2)
49 $comm : Fetch current task comm. 50 $comm : Fetch current task comm.
50 +|-offs(FETCHARG) : Fetch memory at FETCHARG +|- offs address.(**) 51 +|-offs(FETCHARG) : Fetch memory at FETCHARG +|- offs address.(\*3)
51 NAME=FETCHARG : Set NAME as the argument name of FETCHARG. 52 NAME=FETCHARG : Set NAME as the argument name of FETCHARG.
52 FETCHARG:TYPE : Set TYPE as the type of FETCHARG. Currently, basic types 53 FETCHARG:TYPE : Set TYPE as the type of FETCHARG. Currently, basic types
53 (u8/u16/u32/u64/s8/s16/s32/s64), hexadecimal types 54 (u8/u16/u32/u64/s8/s16/s32/s64), hexadecimal types
54 (x8/x16/x32/x64), "string" and bitfield are supported. 55 (x8/x16/x32/x64), "string" and bitfield are supported.
55 56
56 (*) only for return probe. 57 (\*1) only for the probe on function entry (offs == 0).
57 (**) this is useful for fetching a field of data structures. 58 (\*2) only for return probe.
59 (\*3) this is useful for fetching a field of data structures.
58 60
59Types 61Types
60----- 62-----
@@ -64,14 +66,27 @@ respectively. 'x' prefix implies it is unsigned. Traced arguments are shown
64in decimal ('s' and 'u') or hexadecimal ('x'). Without type casting, 'x32' 66in decimal ('s' and 'u') or hexadecimal ('x'). Without type casting, 'x32'
65or 'x64' is used depends on the architecture (e.g. x86-32 uses x32, and 67or 'x64' is used depends on the architecture (e.g. x86-32 uses x32, and
66x86-64 uses x64). 68x86-64 uses x64).
69These value types can be an array. To record array data, you can add '[N]'
70(where N is a fixed number, less than 64) to the base type.
71E.g. 'x16[4]' means an array of x16 (2bytes hex) with 4 elements.
72Note that the array can be applied to memory type fetchargs, you can not
73apply it to registers/stack-entries etc. (for example, '$stack1:x8[8]' is
74wrong, but '+8($stack):x8[8]' is OK.)
67String type is a special type, which fetches a "null-terminated" string from 75String type is a special type, which fetches a "null-terminated" string from
68kernel space. This means it will fail and store NULL if the string container 76kernel space. This means it will fail and store NULL if the string container
69has been paged out. 77has been paged out.
78The string array type is a bit different from other types. For other base
79types, <base-type>[1] is equal to <base-type> (e.g. +0(%di):x32[1] is same
80as +0(%di):x32.) But string[1] is not equal to string. The string type itself
81represents "char array", but string array type represents "char * array".
82So, for example, +0(%di):string[1] is equal to +0(+0(%di)):string.
70Bitfield is another special type, which takes 3 parameters, bit-width, bit- 83Bitfield is another special type, which takes 3 parameters, bit-width, bit-
71offset, and container-size (usually 32). The syntax is:: 84offset, and container-size (usually 32). The syntax is::
72 85
73 b<bit-width>@<bit-offset>/<container-size> 86 b<bit-width>@<bit-offset>/<container-size>
74 87
88Symbol type('symbol') is an alias of u32 or u64 type (depends on BITS_PER_LONG)
89which shows given pointer in "symbol+offset" style.
75For $comm, the default type is "string"; any other type is invalid. 90For $comm, the default type is "string"; any other type is invalid.
76 91
77 92
diff --git a/arch/Kconfig b/arch/Kconfig
index 9d329608913e..ed27fd262627 100644
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -290,6 +290,13 @@ config HAVE_RSEQ
290 This symbol should be selected by an architecture if it 290 This symbol should be selected by an architecture if it
291 supports an implementation of restartable sequences. 291 supports an implementation of restartable sequences.
292 292
293config HAVE_FUNCTION_ARG_ACCESS_API
294 bool
295 help
296 This symbol should be selected by an architecure if it supports
297 the API needed to access function arguments from pt_regs,
298 declared in asm/ptrace.h
299
293config HAVE_CLK 300config HAVE_CLK
294 bool 301 bool
295 help 302 help
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index cbd5f28ea8e2..ffebfc3f43c1 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -186,6 +186,7 @@ config X86
186 select HAVE_RCU_TABLE_INVALIDATE if HAVE_RCU_TABLE_FREE 186 select HAVE_RCU_TABLE_INVALIDATE if HAVE_RCU_TABLE_FREE
187 select HAVE_REGS_AND_STACK_ACCESS_API 187 select HAVE_REGS_AND_STACK_ACCESS_API
188 select HAVE_RELIABLE_STACKTRACE if X86_64 && (UNWINDER_FRAME_POINTER || UNWINDER_ORC) && STACK_VALIDATION 188 select HAVE_RELIABLE_STACKTRACE if X86_64 && (UNWINDER_FRAME_POINTER || UNWINDER_ORC) && STACK_VALIDATION
189 select HAVE_FUNCTION_ARG_ACCESS_API
189 select HAVE_STACKPROTECTOR if CC_HAS_SANE_STACKPROTECTOR 190 select HAVE_STACKPROTECTOR if CC_HAS_SANE_STACKPROTECTOR
190 select HAVE_STACK_VALIDATION if X86_64 191 select HAVE_STACK_VALIDATION if X86_64
191 select HAVE_RSEQ 192 select HAVE_RSEQ
diff --git a/arch/x86/include/asm/ptrace.h b/arch/x86/include/asm/ptrace.h
index 143c99499531..8a7fc0cca2d1 100644
--- a/arch/x86/include/asm/ptrace.h
+++ b/arch/x86/include/asm/ptrace.h
@@ -286,6 +286,44 @@ static inline unsigned long regs_get_kernel_stack_nth(struct pt_regs *regs,
286 return 0; 286 return 0;
287} 287}
288 288
289/**
290 * regs_get_kernel_argument() - get Nth function argument in kernel
291 * @regs: pt_regs of that context
292 * @n: function argument number (start from 0)
293 *
294 * regs_get_argument() returns @n th argument of the function call.
295 * Note that this chooses most probably assignment, in some case
296 * it can be incorrect.
297 * This is expected to be called from kprobes or ftrace with regs
298 * where the top of stack is the return address.
299 */
300static inline unsigned long regs_get_kernel_argument(struct pt_regs *regs,
301 unsigned int n)
302{
303 static const unsigned int argument_offs[] = {
304#ifdef __i386__
305 offsetof(struct pt_regs, ax),
306 offsetof(struct pt_regs, cx),
307 offsetof(struct pt_regs, dx),
308#define NR_REG_ARGUMENTS 3
309#else
310 offsetof(struct pt_regs, di),
311 offsetof(struct pt_regs, si),
312 offsetof(struct pt_regs, dx),
313 offsetof(struct pt_regs, cx),
314 offsetof(struct pt_regs, r8),
315 offsetof(struct pt_regs, r9),
316#define NR_REG_ARGUMENTS 6
317#endif
318 };
319
320 if (n >= NR_REG_ARGUMENTS) {
321 n -= NR_REG_ARGUMENTS - 1;
322 return regs_get_kernel_stack_nth(regs, n);
323 } else
324 return regs_get_register(regs, argument_offs[n]);
325}
326
289#define arch_has_single_step() (1) 327#define arch_has_single_step() (1)
290#ifdef CONFIG_X86_DEBUGCTLMSR 328#ifdef CONFIG_X86_DEBUGCTLMSR
291#define arch_has_block_step() (1) 329#define arch_has_block_step() (1)
diff --git a/include/linux/trace_events.h b/include/linux/trace_events.h
index 78a010e19ed4..4130a5497d40 100644
--- a/include/linux/trace_events.h
+++ b/include/linux/trace_events.h
@@ -575,7 +575,8 @@ extern int bpf_get_kprobe_info(const struct perf_event *event,
575 bool perf_type_tracepoint); 575 bool perf_type_tracepoint);
576#endif 576#endif
577#ifdef CONFIG_UPROBE_EVENTS 577#ifdef CONFIG_UPROBE_EVENTS
578extern int perf_uprobe_init(struct perf_event *event, bool is_retprobe); 578extern int perf_uprobe_init(struct perf_event *event,
579 unsigned long ref_ctr_offset, bool is_retprobe);
579extern void perf_uprobe_destroy(struct perf_event *event); 580extern void perf_uprobe_destroy(struct perf_event *event);
580extern int bpf_get_uprobe_info(const struct perf_event *event, 581extern int bpf_get_uprobe_info(const struct perf_event *event,
581 u32 *fd_type, const char **filename, 582 u32 *fd_type, const char **filename,
diff --git a/include/linux/uprobes.h b/include/linux/uprobes.h
index bb9d2084af03..103a48a48872 100644
--- a/include/linux/uprobes.h
+++ b/include/linux/uprobes.h
@@ -123,6 +123,7 @@ extern unsigned long uprobe_get_swbp_addr(struct pt_regs *regs);
123extern unsigned long uprobe_get_trap_addr(struct pt_regs *regs); 123extern unsigned long uprobe_get_trap_addr(struct pt_regs *regs);
124extern int uprobe_write_opcode(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr, uprobe_opcode_t); 124extern int uprobe_write_opcode(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr, uprobe_opcode_t);
125extern int uprobe_register(struct inode *inode, loff_t offset, struct uprobe_consumer *uc); 125extern int uprobe_register(struct inode *inode, loff_t offset, struct uprobe_consumer *uc);
126extern int uprobe_register_refctr(struct inode *inode, loff_t offset, loff_t ref_ctr_offset, struct uprobe_consumer *uc);
126extern int uprobe_apply(struct inode *inode, loff_t offset, struct uprobe_consumer *uc, bool); 127extern int uprobe_apply(struct inode *inode, loff_t offset, struct uprobe_consumer *uc, bool);
127extern void uprobe_unregister(struct inode *inode, loff_t offset, struct uprobe_consumer *uc); 128extern void uprobe_unregister(struct inode *inode, loff_t offset, struct uprobe_consumer *uc);
128extern int uprobe_mmap(struct vm_area_struct *vma); 129extern int uprobe_mmap(struct vm_area_struct *vma);
@@ -160,6 +161,10 @@ uprobe_register(struct inode *inode, loff_t offset, struct uprobe_consumer *uc)
160{ 161{
161 return -ENOSYS; 162 return -ENOSYS;
162} 163}
164static inline int uprobe_register_refctr(struct inode *inode, loff_t offset, loff_t ref_ctr_offset, struct uprobe_consumer *uc)
165{
166 return -ENOSYS;
167}
163static inline int 168static inline int
164uprobe_apply(struct inode *inode, loff_t offset, struct uprobe_consumer *uc, bool add) 169uprobe_apply(struct inode *inode, loff_t offset, struct uprobe_consumer *uc, bool add)
165{ 170{
diff --git a/kernel/events/core.c b/kernel/events/core.c
index 5a97f34bc14c..8c490130c4fb 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -8376,30 +8376,39 @@ static struct pmu perf_tracepoint = {
8376 * 8376 *
8377 * PERF_PROBE_CONFIG_IS_RETPROBE if set, create kretprobe/uretprobe 8377 * PERF_PROBE_CONFIG_IS_RETPROBE if set, create kretprobe/uretprobe
8378 * if not set, create kprobe/uprobe 8378 * if not set, create kprobe/uprobe
8379 *
8380 * The following values specify a reference counter (or semaphore in the
8381 * terminology of tools like dtrace, systemtap, etc.) Userspace Statically
8382 * Defined Tracepoints (USDT). Currently, we use 40 bit for the offset.
8383 *
8384 * PERF_UPROBE_REF_CTR_OFFSET_BITS # of bits in config as th offset
8385 * PERF_UPROBE_REF_CTR_OFFSET_SHIFT # of bits to shift left
8379 */ 8386 */
8380enum perf_probe_config { 8387enum perf_probe_config {
8381 PERF_PROBE_CONFIG_IS_RETPROBE = 1U << 0, /* [k,u]retprobe */ 8388 PERF_PROBE_CONFIG_IS_RETPROBE = 1U << 0, /* [k,u]retprobe */
8389 PERF_UPROBE_REF_CTR_OFFSET_BITS = 32,
8390 PERF_UPROBE_REF_CTR_OFFSET_SHIFT = 64 - PERF_UPROBE_REF_CTR_OFFSET_BITS,
8382}; 8391};
8383 8392
8384PMU_FORMAT_ATTR(retprobe, "config:0"); 8393PMU_FORMAT_ATTR(retprobe, "config:0");
8394#endif
8385 8395
8386static struct attribute *probe_attrs[] = { 8396#ifdef CONFIG_KPROBE_EVENTS
8397static struct attribute *kprobe_attrs[] = {
8387 &format_attr_retprobe.attr, 8398 &format_attr_retprobe.attr,
8388 NULL, 8399 NULL,
8389}; 8400};
8390 8401
8391static struct attribute_group probe_format_group = { 8402static struct attribute_group kprobe_format_group = {
8392 .name = "format", 8403 .name = "format",
8393 .attrs = probe_attrs, 8404 .attrs = kprobe_attrs,
8394}; 8405};
8395 8406
8396static const struct attribute_group *probe_attr_groups[] = { 8407static const struct attribute_group *kprobe_attr_groups[] = {
8397 &probe_format_group, 8408 &kprobe_format_group,
8398 NULL, 8409 NULL,
8399}; 8410};
8400#endif
8401 8411
8402#ifdef CONFIG_KPROBE_EVENTS
8403static int perf_kprobe_event_init(struct perf_event *event); 8412static int perf_kprobe_event_init(struct perf_event *event);
8404static struct pmu perf_kprobe = { 8413static struct pmu perf_kprobe = {
8405 .task_ctx_nr = perf_sw_context, 8414 .task_ctx_nr = perf_sw_context,
@@ -8409,7 +8418,7 @@ static struct pmu perf_kprobe = {
8409 .start = perf_swevent_start, 8418 .start = perf_swevent_start,
8410 .stop = perf_swevent_stop, 8419 .stop = perf_swevent_stop,
8411 .read = perf_swevent_read, 8420 .read = perf_swevent_read,
8412 .attr_groups = probe_attr_groups, 8421 .attr_groups = kprobe_attr_groups,
8413}; 8422};
8414 8423
8415static int perf_kprobe_event_init(struct perf_event *event) 8424static int perf_kprobe_event_init(struct perf_event *event)
@@ -8441,6 +8450,24 @@ static int perf_kprobe_event_init(struct perf_event *event)
8441#endif /* CONFIG_KPROBE_EVENTS */ 8450#endif /* CONFIG_KPROBE_EVENTS */
8442 8451
8443#ifdef CONFIG_UPROBE_EVENTS 8452#ifdef CONFIG_UPROBE_EVENTS
8453PMU_FORMAT_ATTR(ref_ctr_offset, "config:32-63");
8454
8455static struct attribute *uprobe_attrs[] = {
8456 &format_attr_retprobe.attr,
8457 &format_attr_ref_ctr_offset.attr,
8458 NULL,
8459};
8460
8461static struct attribute_group uprobe_format_group = {
8462 .name = "format",
8463 .attrs = uprobe_attrs,
8464};
8465
8466static const struct attribute_group *uprobe_attr_groups[] = {
8467 &uprobe_format_group,
8468 NULL,
8469};
8470
8444static int perf_uprobe_event_init(struct perf_event *event); 8471static int perf_uprobe_event_init(struct perf_event *event);
8445static struct pmu perf_uprobe = { 8472static struct pmu perf_uprobe = {
8446 .task_ctx_nr = perf_sw_context, 8473 .task_ctx_nr = perf_sw_context,
@@ -8450,12 +8477,13 @@ static struct pmu perf_uprobe = {
8450 .start = perf_swevent_start, 8477 .start = perf_swevent_start,
8451 .stop = perf_swevent_stop, 8478 .stop = perf_swevent_stop,
8452 .read = perf_swevent_read, 8479 .read = perf_swevent_read,
8453 .attr_groups = probe_attr_groups, 8480 .attr_groups = uprobe_attr_groups,
8454}; 8481};
8455 8482
8456static int perf_uprobe_event_init(struct perf_event *event) 8483static int perf_uprobe_event_init(struct perf_event *event)
8457{ 8484{
8458 int err; 8485 int err;
8486 unsigned long ref_ctr_offset;
8459 bool is_retprobe; 8487 bool is_retprobe;
8460 8488
8461 if (event->attr.type != perf_uprobe.type) 8489 if (event->attr.type != perf_uprobe.type)
@@ -8471,7 +8499,8 @@ static int perf_uprobe_event_init(struct perf_event *event)
8471 return -EOPNOTSUPP; 8499 return -EOPNOTSUPP;
8472 8500
8473 is_retprobe = event->attr.config & PERF_PROBE_CONFIG_IS_RETPROBE; 8501 is_retprobe = event->attr.config & PERF_PROBE_CONFIG_IS_RETPROBE;
8474 err = perf_uprobe_init(event, is_retprobe); 8502 ref_ctr_offset = event->attr.config >> PERF_UPROBE_REF_CTR_OFFSET_SHIFT;
8503 err = perf_uprobe_init(event, ref_ctr_offset, is_retprobe);
8475 if (err) 8504 if (err)
8476 return err; 8505 return err;
8477 8506
diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c
index 2bf792d22087..96d4bee83489 100644
--- a/kernel/events/uprobes.c
+++ b/kernel/events/uprobes.c
@@ -73,6 +73,7 @@ struct uprobe {
73 struct uprobe_consumer *consumers; 73 struct uprobe_consumer *consumers;
74 struct inode *inode; /* Also hold a ref to inode */ 74 struct inode *inode; /* Also hold a ref to inode */
75 loff_t offset; 75 loff_t offset;
76 loff_t ref_ctr_offset;
76 unsigned long flags; 77 unsigned long flags;
77 78
78 /* 79 /*
@@ -88,6 +89,15 @@ struct uprobe {
88 struct arch_uprobe arch; 89 struct arch_uprobe arch;
89}; 90};
90 91
92struct delayed_uprobe {
93 struct list_head list;
94 struct uprobe *uprobe;
95 struct mm_struct *mm;
96};
97
98static DEFINE_MUTEX(delayed_uprobe_lock);
99static LIST_HEAD(delayed_uprobe_list);
100
91/* 101/*
92 * Execute out of line area: anonymous executable mapping installed 102 * Execute out of line area: anonymous executable mapping installed
93 * by the probed task to execute the copy of the original instruction 103 * by the probed task to execute the copy of the original instruction
@@ -282,6 +292,166 @@ static int verify_opcode(struct page *page, unsigned long vaddr, uprobe_opcode_t
282 return 1; 292 return 1;
283} 293}
284 294
295static struct delayed_uprobe *
296delayed_uprobe_check(struct uprobe *uprobe, struct mm_struct *mm)
297{
298 struct delayed_uprobe *du;
299
300 list_for_each_entry(du, &delayed_uprobe_list, list)
301 if (du->uprobe == uprobe && du->mm == mm)
302 return du;
303 return NULL;
304}
305
306static int delayed_uprobe_add(struct uprobe *uprobe, struct mm_struct *mm)
307{
308 struct delayed_uprobe *du;
309
310 if (delayed_uprobe_check(uprobe, mm))
311 return 0;
312
313 du = kzalloc(sizeof(*du), GFP_KERNEL);
314 if (!du)
315 return -ENOMEM;
316
317 du->uprobe = uprobe;
318 du->mm = mm;
319 list_add(&du->list, &delayed_uprobe_list);
320 return 0;
321}
322
323static void delayed_uprobe_delete(struct delayed_uprobe *du)
324{
325 if (WARN_ON(!du))
326 return;
327 list_del(&du->list);
328 kfree(du);
329}
330
331static void delayed_uprobe_remove(struct uprobe *uprobe, struct mm_struct *mm)
332{
333 struct list_head *pos, *q;
334 struct delayed_uprobe *du;
335
336 if (!uprobe && !mm)
337 return;
338
339 list_for_each_safe(pos, q, &delayed_uprobe_list) {
340 du = list_entry(pos, struct delayed_uprobe, list);
341
342 if (uprobe && du->uprobe != uprobe)
343 continue;
344 if (mm && du->mm != mm)
345 continue;
346
347 delayed_uprobe_delete(du);
348 }
349}
350
351static bool valid_ref_ctr_vma(struct uprobe *uprobe,
352 struct vm_area_struct *vma)
353{
354 unsigned long vaddr = offset_to_vaddr(vma, uprobe->ref_ctr_offset);
355
356 return uprobe->ref_ctr_offset &&
357 vma->vm_file &&
358 file_inode(vma->vm_file) == uprobe->inode &&
359 (vma->vm_flags & (VM_WRITE|VM_SHARED)) == VM_WRITE &&
360 vma->vm_start <= vaddr &&
361 vma->vm_end > vaddr;
362}
363
364static struct vm_area_struct *
365find_ref_ctr_vma(struct uprobe *uprobe, struct mm_struct *mm)
366{
367 struct vm_area_struct *tmp;
368
369 for (tmp = mm->mmap; tmp; tmp = tmp->vm_next)
370 if (valid_ref_ctr_vma(uprobe, tmp))
371 return tmp;
372
373 return NULL;
374}
375
376static int
377__update_ref_ctr(struct mm_struct *mm, unsigned long vaddr, short d)
378{
379 void *kaddr;
380 struct page *page;
381 struct vm_area_struct *vma;
382 int ret;
383 short *ptr;
384
385 if (!vaddr || !d)
386 return -EINVAL;
387
388 ret = get_user_pages_remote(NULL, mm, vaddr, 1,
389 FOLL_WRITE, &page, &vma, NULL);
390 if (unlikely(ret <= 0)) {
391 /*
392 * We are asking for 1 page. If get_user_pages_remote() fails,
393 * it may return 0, in that case we have to return error.
394 */
395 return ret == 0 ? -EBUSY : ret;
396 }
397
398 kaddr = kmap_atomic(page);
399 ptr = kaddr + (vaddr & ~PAGE_MASK);
400
401 if (unlikely(*ptr + d < 0)) {
402 pr_warn("ref_ctr going negative. vaddr: 0x%lx, "
403 "curr val: %d, delta: %d\n", vaddr, *ptr, d);
404 ret = -EINVAL;
405 goto out;
406 }
407
408 *ptr += d;
409 ret = 0;
410out:
411 kunmap_atomic(kaddr);
412 put_page(page);
413 return ret;
414}
415
416static void update_ref_ctr_warn(struct uprobe *uprobe,
417 struct mm_struct *mm, short d)
418{
419 pr_warn("ref_ctr %s failed for inode: 0x%lx offset: "
420 "0x%llx ref_ctr_offset: 0x%llx of mm: 0x%pK\n",
421 d > 0 ? "increment" : "decrement", uprobe->inode->i_ino,
422 (unsigned long long) uprobe->offset,
423 (unsigned long long) uprobe->ref_ctr_offset, mm);
424}
425
426static int update_ref_ctr(struct uprobe *uprobe, struct mm_struct *mm,
427 short d)
428{
429 struct vm_area_struct *rc_vma;
430 unsigned long rc_vaddr;
431 int ret = 0;
432
433 rc_vma = find_ref_ctr_vma(uprobe, mm);
434
435 if (rc_vma) {
436 rc_vaddr = offset_to_vaddr(rc_vma, uprobe->ref_ctr_offset);
437 ret = __update_ref_ctr(mm, rc_vaddr, d);
438 if (ret)
439 update_ref_ctr_warn(uprobe, mm, d);
440
441 if (d > 0)
442 return ret;
443 }
444
445 mutex_lock(&delayed_uprobe_lock);
446 if (d > 0)
447 ret = delayed_uprobe_add(uprobe, mm);
448 else
449 delayed_uprobe_remove(uprobe, mm);
450 mutex_unlock(&delayed_uprobe_lock);
451
452 return ret;
453}
454
285/* 455/*
286 * NOTE: 456 * NOTE:
287 * Expect the breakpoint instruction to be the smallest size instruction for 457 * Expect the breakpoint instruction to be the smallest size instruction for
@@ -302,9 +472,13 @@ static int verify_opcode(struct page *page, unsigned long vaddr, uprobe_opcode_t
302int uprobe_write_opcode(struct arch_uprobe *auprobe, struct mm_struct *mm, 472int uprobe_write_opcode(struct arch_uprobe *auprobe, struct mm_struct *mm,
303 unsigned long vaddr, uprobe_opcode_t opcode) 473 unsigned long vaddr, uprobe_opcode_t opcode)
304{ 474{
475 struct uprobe *uprobe;
305 struct page *old_page, *new_page; 476 struct page *old_page, *new_page;
306 struct vm_area_struct *vma; 477 struct vm_area_struct *vma;
307 int ret; 478 int ret, is_register, ref_ctr_updated = 0;
479
480 is_register = is_swbp_insn(&opcode);
481 uprobe = container_of(auprobe, struct uprobe, arch);
308 482
309retry: 483retry:
310 /* Read the page with vaddr into memory */ 484 /* Read the page with vaddr into memory */
@@ -317,6 +491,15 @@ retry:
317 if (ret <= 0) 491 if (ret <= 0)
318 goto put_old; 492 goto put_old;
319 493
494 /* We are going to replace instruction, update ref_ctr. */
495 if (!ref_ctr_updated && uprobe->ref_ctr_offset) {
496 ret = update_ref_ctr(uprobe, mm, is_register ? 1 : -1);
497 if (ret)
498 goto put_old;
499
500 ref_ctr_updated = 1;
501 }
502
320 ret = anon_vma_prepare(vma); 503 ret = anon_vma_prepare(vma);
321 if (ret) 504 if (ret)
322 goto put_old; 505 goto put_old;
@@ -337,6 +520,11 @@ put_old:
337 520
338 if (unlikely(ret == -EAGAIN)) 521 if (unlikely(ret == -EAGAIN))
339 goto retry; 522 goto retry;
523
524 /* Revert back reference counter if instruction update failed. */
525 if (ret && is_register && ref_ctr_updated)
526 update_ref_ctr(uprobe, mm, -1);
527
340 return ret; 528 return ret;
341} 529}
342 530
@@ -378,8 +566,15 @@ static struct uprobe *get_uprobe(struct uprobe *uprobe)
378 566
379static void put_uprobe(struct uprobe *uprobe) 567static void put_uprobe(struct uprobe *uprobe)
380{ 568{
381 if (atomic_dec_and_test(&uprobe->ref)) 569 if (atomic_dec_and_test(&uprobe->ref)) {
570 /*
571 * If application munmap(exec_vma) before uprobe_unregister()
572 * gets called, we don't get a chance to remove uprobe from
573 * delayed_uprobe_list from remove_breakpoint(). Do it here.
574 */
575 delayed_uprobe_remove(uprobe, NULL);
382 kfree(uprobe); 576 kfree(uprobe);
577 }
383} 578}
384 579
385static int match_uprobe(struct uprobe *l, struct uprobe *r) 580static int match_uprobe(struct uprobe *l, struct uprobe *r)
@@ -484,7 +679,18 @@ static struct uprobe *insert_uprobe(struct uprobe *uprobe)
484 return u; 679 return u;
485} 680}
486 681
487static struct uprobe *alloc_uprobe(struct inode *inode, loff_t offset) 682static void
683ref_ctr_mismatch_warn(struct uprobe *cur_uprobe, struct uprobe *uprobe)
684{
685 pr_warn("ref_ctr_offset mismatch. inode: 0x%lx offset: 0x%llx "
686 "ref_ctr_offset(old): 0x%llx ref_ctr_offset(new): 0x%llx\n",
687 uprobe->inode->i_ino, (unsigned long long) uprobe->offset,
688 (unsigned long long) cur_uprobe->ref_ctr_offset,
689 (unsigned long long) uprobe->ref_ctr_offset);
690}
691
692static struct uprobe *alloc_uprobe(struct inode *inode, loff_t offset,
693 loff_t ref_ctr_offset)
488{ 694{
489 struct uprobe *uprobe, *cur_uprobe; 695 struct uprobe *uprobe, *cur_uprobe;
490 696
@@ -494,6 +700,7 @@ static struct uprobe *alloc_uprobe(struct inode *inode, loff_t offset)
494 700
495 uprobe->inode = inode; 701 uprobe->inode = inode;
496 uprobe->offset = offset; 702 uprobe->offset = offset;
703 uprobe->ref_ctr_offset = ref_ctr_offset;
497 init_rwsem(&uprobe->register_rwsem); 704 init_rwsem(&uprobe->register_rwsem);
498 init_rwsem(&uprobe->consumer_rwsem); 705 init_rwsem(&uprobe->consumer_rwsem);
499 706
@@ -501,6 +708,12 @@ static struct uprobe *alloc_uprobe(struct inode *inode, loff_t offset)
501 cur_uprobe = insert_uprobe(uprobe); 708 cur_uprobe = insert_uprobe(uprobe);
502 /* a uprobe exists for this inode:offset combination */ 709 /* a uprobe exists for this inode:offset combination */
503 if (cur_uprobe) { 710 if (cur_uprobe) {
711 if (cur_uprobe->ref_ctr_offset != uprobe->ref_ctr_offset) {
712 ref_ctr_mismatch_warn(cur_uprobe, uprobe);
713 put_uprobe(cur_uprobe);
714 kfree(uprobe);
715 return ERR_PTR(-EINVAL);
716 }
504 kfree(uprobe); 717 kfree(uprobe);
505 uprobe = cur_uprobe; 718 uprobe = cur_uprobe;
506 } 719 }
@@ -895,7 +1108,7 @@ EXPORT_SYMBOL_GPL(uprobe_unregister);
895 * else return 0 (success) 1108 * else return 0 (success)
896 */ 1109 */
897static int __uprobe_register(struct inode *inode, loff_t offset, 1110static int __uprobe_register(struct inode *inode, loff_t offset,
898 struct uprobe_consumer *uc) 1111 loff_t ref_ctr_offset, struct uprobe_consumer *uc)
899{ 1112{
900 struct uprobe *uprobe; 1113 struct uprobe *uprobe;
901 int ret; 1114 int ret;
@@ -912,9 +1125,12 @@ static int __uprobe_register(struct inode *inode, loff_t offset,
912 return -EINVAL; 1125 return -EINVAL;
913 1126
914 retry: 1127 retry:
915 uprobe = alloc_uprobe(inode, offset); 1128 uprobe = alloc_uprobe(inode, offset, ref_ctr_offset);
916 if (!uprobe) 1129 if (!uprobe)
917 return -ENOMEM; 1130 return -ENOMEM;
1131 if (IS_ERR(uprobe))
1132 return PTR_ERR(uprobe);
1133
918 /* 1134 /*
919 * We can race with uprobe_unregister()->delete_uprobe(). 1135 * We can race with uprobe_unregister()->delete_uprobe().
920 * Check uprobe_is_active() and retry if it is false. 1136 * Check uprobe_is_active() and retry if it is false.
@@ -938,10 +1154,17 @@ static int __uprobe_register(struct inode *inode, loff_t offset,
938int uprobe_register(struct inode *inode, loff_t offset, 1154int uprobe_register(struct inode *inode, loff_t offset,
939 struct uprobe_consumer *uc) 1155 struct uprobe_consumer *uc)
940{ 1156{
941 return __uprobe_register(inode, offset, uc); 1157 return __uprobe_register(inode, offset, 0, uc);
942} 1158}
943EXPORT_SYMBOL_GPL(uprobe_register); 1159EXPORT_SYMBOL_GPL(uprobe_register);
944 1160
1161int uprobe_register_refctr(struct inode *inode, loff_t offset,
1162 loff_t ref_ctr_offset, struct uprobe_consumer *uc)
1163{
1164 return __uprobe_register(inode, offset, ref_ctr_offset, uc);
1165}
1166EXPORT_SYMBOL_GPL(uprobe_register_refctr);
1167
945/* 1168/*
946 * uprobe_apply - unregister an already registered probe. 1169 * uprobe_apply - unregister an already registered probe.
947 * @inode: the file in which the probe has to be removed. 1170 * @inode: the file in which the probe has to be removed.
@@ -1060,6 +1283,35 @@ static void build_probe_list(struct inode *inode,
1060 spin_unlock(&uprobes_treelock); 1283 spin_unlock(&uprobes_treelock);
1061} 1284}
1062 1285
1286/* @vma contains reference counter, not the probed instruction. */
1287static int delayed_ref_ctr_inc(struct vm_area_struct *vma)
1288{
1289 struct list_head *pos, *q;
1290 struct delayed_uprobe *du;
1291 unsigned long vaddr;
1292 int ret = 0, err = 0;
1293
1294 mutex_lock(&delayed_uprobe_lock);
1295 list_for_each_safe(pos, q, &delayed_uprobe_list) {
1296 du = list_entry(pos, struct delayed_uprobe, list);
1297
1298 if (du->mm != vma->vm_mm ||
1299 !valid_ref_ctr_vma(du->uprobe, vma))
1300 continue;
1301
1302 vaddr = offset_to_vaddr(vma, du->uprobe->ref_ctr_offset);
1303 ret = __update_ref_ctr(vma->vm_mm, vaddr, 1);
1304 if (ret) {
1305 update_ref_ctr_warn(du->uprobe, vma->vm_mm, 1);
1306 if (!err)
1307 err = ret;
1308 }
1309 delayed_uprobe_delete(du);
1310 }
1311 mutex_unlock(&delayed_uprobe_lock);
1312 return err;
1313}
1314
1063/* 1315/*
1064 * Called from mmap_region/vma_adjust with mm->mmap_sem acquired. 1316 * Called from mmap_region/vma_adjust with mm->mmap_sem acquired.
1065 * 1317 *
@@ -1072,7 +1324,15 @@ int uprobe_mmap(struct vm_area_struct *vma)
1072 struct uprobe *uprobe, *u; 1324 struct uprobe *uprobe, *u;
1073 struct inode *inode; 1325 struct inode *inode;
1074 1326
1075 if (no_uprobe_events() || !valid_vma(vma, true)) 1327 if (no_uprobe_events())
1328 return 0;
1329
1330 if (vma->vm_file &&
1331 (vma->vm_flags & (VM_WRITE|VM_SHARED)) == VM_WRITE &&
1332 test_bit(MMF_HAS_UPROBES, &vma->vm_mm->flags))
1333 delayed_ref_ctr_inc(vma);
1334
1335 if (!valid_vma(vma, true))
1076 return 0; 1336 return 0;
1077 1337
1078 inode = file_inode(vma->vm_file); 1338 inode = file_inode(vma->vm_file);
@@ -1246,6 +1506,10 @@ void uprobe_clear_state(struct mm_struct *mm)
1246{ 1506{
1247 struct xol_area *area = mm->uprobes_state.xol_area; 1507 struct xol_area *area = mm->uprobes_state.xol_area;
1248 1508
1509 mutex_lock(&delayed_uprobe_lock);
1510 delayed_uprobe_remove(NULL, mm);
1511 mutex_unlock(&delayed_uprobe_lock);
1512
1249 if (!area) 1513 if (!area)
1250 return; 1514 return;
1251 1515
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index bf6f1d70484d..ff1c4b20cd0a 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -2727,6 +2727,7 @@ void trace_dump_stack(int skip)
2727 __ftrace_trace_stack(global_trace.trace_buffer.buffer, 2727 __ftrace_trace_stack(global_trace.trace_buffer.buffer,
2728 flags, skip, preempt_count(), NULL); 2728 flags, skip, preempt_count(), NULL);
2729} 2729}
2730EXPORT_SYMBOL_GPL(trace_dump_stack);
2730 2731
2731static DEFINE_PER_CPU(int, user_stack_count); 2732static DEFINE_PER_CPU(int, user_stack_count);
2732 2733
@@ -4621,13 +4622,18 @@ static const char readme_msg[] =
4621 "place (kretprobe): [<module>:]<symbol>[+<offset>]|<memaddr>\n" 4622 "place (kretprobe): [<module>:]<symbol>[+<offset>]|<memaddr>\n"
4622#endif 4623#endif
4623#ifdef CONFIG_UPROBE_EVENTS 4624#ifdef CONFIG_UPROBE_EVENTS
4624 "\t place: <path>:<offset>\n" 4625 " place (uprobe): <path>:<offset>[(ref_ctr_offset)]\n"
4625#endif 4626#endif
4626 "\t args: <name>=fetcharg[:type]\n" 4627 "\t args: <name>=fetcharg[:type]\n"
4627 "\t fetcharg: %<register>, @<address>, @<symbol>[+|-<offset>],\n" 4628 "\t fetcharg: %<register>, @<address>, @<symbol>[+|-<offset>],\n"
4629#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
4630 "\t $stack<index>, $stack, $retval, $comm, $arg<N>\n"
4631#else
4628 "\t $stack<index>, $stack, $retval, $comm\n" 4632 "\t $stack<index>, $stack, $retval, $comm\n"
4629 "\t type: s8/16/32/64, u8/16/32/64, x8/16/32/64, string,\n" 4633#endif
4630 "\t b<bit-width>@<bit-offset>/<container-size>\n" 4634 "\t type: s8/16/32/64, u8/16/32/64, x8/16/32/64, string, symbol,\n"
4635 "\t b<bit-width>@<bit-offset>/<container-size>,\n"
4636 "\t <type>\\[<array-size>\\]\n"
4631#endif 4637#endif
4632 " events/\t\t- Directory containing all trace event subsystems:\n" 4638 " events/\t\t- Directory containing all trace event subsystems:\n"
4633 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n" 4639 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
diff --git a/kernel/trace/trace_event_perf.c b/kernel/trace/trace_event_perf.c
index 69a3fe926e8c..76217bbef815 100644
--- a/kernel/trace/trace_event_perf.c
+++ b/kernel/trace/trace_event_perf.c
@@ -290,7 +290,8 @@ void perf_kprobe_destroy(struct perf_event *p_event)
290#endif /* CONFIG_KPROBE_EVENTS */ 290#endif /* CONFIG_KPROBE_EVENTS */
291 291
292#ifdef CONFIG_UPROBE_EVENTS 292#ifdef CONFIG_UPROBE_EVENTS
293int perf_uprobe_init(struct perf_event *p_event, bool is_retprobe) 293int perf_uprobe_init(struct perf_event *p_event,
294 unsigned long ref_ctr_offset, bool is_retprobe)
294{ 295{
295 int ret; 296 int ret;
296 char *path = NULL; 297 char *path = NULL;
@@ -312,8 +313,8 @@ int perf_uprobe_init(struct perf_event *p_event, bool is_retprobe)
312 goto out; 313 goto out;
313 } 314 }
314 315
315 tp_event = create_local_trace_uprobe( 316 tp_event = create_local_trace_uprobe(path, p_event->attr.probe_offset,
316 path, p_event->attr.probe_offset, is_retprobe); 317 ref_ctr_offset, is_retprobe);
317 if (IS_ERR(tp_event)) { 318 if (IS_ERR(tp_event)) {
318 ret = PTR_ERR(tp_event); 319 ret = PTR_ERR(tp_event);
319 goto out; 320 goto out;
diff --git a/kernel/trace/trace_kprobe.c b/kernel/trace/trace_kprobe.c
index c30032367aab..fec67188c4d2 100644
--- a/kernel/trace/trace_kprobe.c
+++ b/kernel/trace/trace_kprobe.c
@@ -14,6 +14,7 @@
14 14
15#include "trace_kprobe_selftest.h" 15#include "trace_kprobe_selftest.h"
16#include "trace_probe.h" 16#include "trace_probe.h"
17#include "trace_probe_tmpl.h"
17 18
18#define KPROBE_EVENT_SYSTEM "kprobes" 19#define KPROBE_EVENT_SYSTEM "kprobes"
19#define KRETPROBE_MAXACTIVE_MAX 4096 20#define KRETPROBE_MAXACTIVE_MAX 4096
@@ -61,9 +62,23 @@ static nokprobe_inline bool trace_kprobe_within_module(struct trace_kprobe *tk,
61 return strncmp(mod->name, name, len) == 0 && name[len] == ':'; 62 return strncmp(mod->name, name, len) == 0 && name[len] == ':';
62} 63}
63 64
64static nokprobe_inline bool trace_kprobe_is_on_module(struct trace_kprobe *tk) 65static nokprobe_inline bool trace_kprobe_module_exist(struct trace_kprobe *tk)
65{ 66{
66 return !!strchr(trace_kprobe_symbol(tk), ':'); 67 char *p;
68 bool ret;
69
70 if (!tk->symbol)
71 return false;
72 p = strchr(tk->symbol, ':');
73 if (!p)
74 return true;
75 *p = '\0';
76 mutex_lock(&module_mutex);
77 ret = !!find_module(tk->symbol);
78 mutex_unlock(&module_mutex);
79 *p = ':';
80
81 return ret;
67} 82}
68 83
69static nokprobe_inline unsigned long trace_kprobe_nhit(struct trace_kprobe *tk) 84static nokprobe_inline unsigned long trace_kprobe_nhit(struct trace_kprobe *tk)
@@ -120,184 +135,6 @@ static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs);
120static int kretprobe_dispatcher(struct kretprobe_instance *ri, 135static int kretprobe_dispatcher(struct kretprobe_instance *ri,
121 struct pt_regs *regs); 136 struct pt_regs *regs);
122 137
123/* Memory fetching by symbol */
124struct symbol_cache {
125 char *symbol;
126 long offset;
127 unsigned long addr;
128};
129
130unsigned long update_symbol_cache(struct symbol_cache *sc)
131{
132 sc->addr = (unsigned long)kallsyms_lookup_name(sc->symbol);
133
134 if (sc->addr)
135 sc->addr += sc->offset;
136
137 return sc->addr;
138}
139
140void free_symbol_cache(struct symbol_cache *sc)
141{
142 kfree(sc->symbol);
143 kfree(sc);
144}
145
146struct symbol_cache *alloc_symbol_cache(const char *sym, long offset)
147{
148 struct symbol_cache *sc;
149
150 if (!sym || strlen(sym) == 0)
151 return NULL;
152
153 sc = kzalloc(sizeof(struct symbol_cache), GFP_KERNEL);
154 if (!sc)
155 return NULL;
156
157 sc->symbol = kstrdup(sym, GFP_KERNEL);
158 if (!sc->symbol) {
159 kfree(sc);
160 return NULL;
161 }
162 sc->offset = offset;
163 update_symbol_cache(sc);
164
165 return sc;
166}
167
168/*
169 * Kprobes-specific fetch functions
170 */
171#define DEFINE_FETCH_stack(type) \
172static void FETCH_FUNC_NAME(stack, type)(struct pt_regs *regs, \
173 void *offset, void *dest) \
174{ \
175 *(type *)dest = (type)regs_get_kernel_stack_nth(regs, \
176 (unsigned int)((unsigned long)offset)); \
177} \
178NOKPROBE_SYMBOL(FETCH_FUNC_NAME(stack, type));
179
180DEFINE_BASIC_FETCH_FUNCS(stack)
181/* No string on the stack entry */
182#define fetch_stack_string NULL
183#define fetch_stack_string_size NULL
184
185#define DEFINE_FETCH_memory(type) \
186static void FETCH_FUNC_NAME(memory, type)(struct pt_regs *regs, \
187 void *addr, void *dest) \
188{ \
189 type retval; \
190 if (probe_kernel_address(addr, retval)) \
191 *(type *)dest = 0; \
192 else \
193 *(type *)dest = retval; \
194} \
195NOKPROBE_SYMBOL(FETCH_FUNC_NAME(memory, type));
196
197DEFINE_BASIC_FETCH_FUNCS(memory)
198/*
199 * Fetch a null-terminated string. Caller MUST set *(u32 *)dest with max
200 * length and relative data location.
201 */
202static void FETCH_FUNC_NAME(memory, string)(struct pt_regs *regs,
203 void *addr, void *dest)
204{
205 int maxlen = get_rloc_len(*(u32 *)dest);
206 u8 *dst = get_rloc_data(dest);
207 long ret;
208
209 if (!maxlen)
210 return;
211
212 /*
213 * Try to get string again, since the string can be changed while
214 * probing.
215 */
216 ret = strncpy_from_unsafe(dst, addr, maxlen);
217
218 if (ret < 0) { /* Failed to fetch string */
219 dst[0] = '\0';
220 *(u32 *)dest = make_data_rloc(0, get_rloc_offs(*(u32 *)dest));
221 } else {
222 *(u32 *)dest = make_data_rloc(ret, get_rloc_offs(*(u32 *)dest));
223 }
224}
225NOKPROBE_SYMBOL(FETCH_FUNC_NAME(memory, string));
226
227/* Return the length of string -- including null terminal byte */
228static void FETCH_FUNC_NAME(memory, string_size)(struct pt_regs *regs,
229 void *addr, void *dest)
230{
231 mm_segment_t old_fs;
232 int ret, len = 0;
233 u8 c;
234
235 old_fs = get_fs();
236 set_fs(KERNEL_DS);
237 pagefault_disable();
238
239 do {
240 ret = __copy_from_user_inatomic(&c, (u8 *)addr + len, 1);
241 len++;
242 } while (c && ret == 0 && len < MAX_STRING_SIZE);
243
244 pagefault_enable();
245 set_fs(old_fs);
246
247 if (ret < 0) /* Failed to check the length */
248 *(u32 *)dest = 0;
249 else
250 *(u32 *)dest = len;
251}
252NOKPROBE_SYMBOL(FETCH_FUNC_NAME(memory, string_size));
253
254#define DEFINE_FETCH_symbol(type) \
255void FETCH_FUNC_NAME(symbol, type)(struct pt_regs *regs, void *data, void *dest)\
256{ \
257 struct symbol_cache *sc = data; \
258 if (sc->addr) \
259 fetch_memory_##type(regs, (void *)sc->addr, dest); \
260 else \
261 *(type *)dest = 0; \
262} \
263NOKPROBE_SYMBOL(FETCH_FUNC_NAME(symbol, type));
264
265DEFINE_BASIC_FETCH_FUNCS(symbol)
266DEFINE_FETCH_symbol(string)
267DEFINE_FETCH_symbol(string_size)
268
269/* kprobes don't support file_offset fetch methods */
270#define fetch_file_offset_u8 NULL
271#define fetch_file_offset_u16 NULL
272#define fetch_file_offset_u32 NULL
273#define fetch_file_offset_u64 NULL
274#define fetch_file_offset_string NULL
275#define fetch_file_offset_string_size NULL
276
277/* Fetch type information table */
278static const struct fetch_type kprobes_fetch_type_table[] = {
279 /* Special types */
280 [FETCH_TYPE_STRING] = __ASSIGN_FETCH_TYPE("string", string, string,
281 sizeof(u32), 1, "__data_loc char[]"),
282 [FETCH_TYPE_STRSIZE] = __ASSIGN_FETCH_TYPE("string_size", u32,
283 string_size, sizeof(u32), 0, "u32"),
284 /* Basic types */
285 ASSIGN_FETCH_TYPE(u8, u8, 0),
286 ASSIGN_FETCH_TYPE(u16, u16, 0),
287 ASSIGN_FETCH_TYPE(u32, u32, 0),
288 ASSIGN_FETCH_TYPE(u64, u64, 0),
289 ASSIGN_FETCH_TYPE(s8, u8, 1),
290 ASSIGN_FETCH_TYPE(s16, u16, 1),
291 ASSIGN_FETCH_TYPE(s32, u32, 1),
292 ASSIGN_FETCH_TYPE(s64, u64, 1),
293 ASSIGN_FETCH_TYPE_ALIAS(x8, u8, u8, 0),
294 ASSIGN_FETCH_TYPE_ALIAS(x16, u16, u16, 0),
295 ASSIGN_FETCH_TYPE_ALIAS(x32, u32, u32, 0),
296 ASSIGN_FETCH_TYPE_ALIAS(x64, u64, u64, 0),
297
298 ASSIGN_FETCH_TYPE_END
299};
300
301/* 138/*
302 * Allocate new trace_probe and initialize it (including kprobes). 139 * Allocate new trace_probe and initialize it (including kprobes).
303 */ 140 */
@@ -540,8 +377,11 @@ static int __register_trace_kprobe(struct trace_kprobe *tk)
540 return -EINVAL; 377 return -EINVAL;
541 } 378 }
542 379
543 for (i = 0; i < tk->tp.nr_args; i++) 380 for (i = 0; i < tk->tp.nr_args; i++) {
544 traceprobe_update_arg(&tk->tp.args[i]); 381 ret = traceprobe_update_arg(&tk->tp.args[i]);
382 if (ret)
383 return ret;
384 }
545 385
546 /* Set/clear disabled flag according to tp->flag */ 386 /* Set/clear disabled flag according to tp->flag */
547 if (trace_probe_is_enabled(&tk->tp)) 387 if (trace_probe_is_enabled(&tk->tp))
@@ -554,19 +394,13 @@ static int __register_trace_kprobe(struct trace_kprobe *tk)
554 else 394 else
555 ret = register_kprobe(&tk->rp.kp); 395 ret = register_kprobe(&tk->rp.kp);
556 396
557 if (ret == 0) 397 if (ret == 0) {
558 tk->tp.flags |= TP_FLAG_REGISTERED; 398 tk->tp.flags |= TP_FLAG_REGISTERED;
559 else { 399 } else if (ret == -EILSEQ) {
560 if (ret == -ENOENT && trace_kprobe_is_on_module(tk)) { 400 pr_warn("Probing address(0x%p) is not an instruction boundary.\n",
561 pr_warn("This probe might be able to register after target module is loaded. Continue.\n"); 401 tk->rp.kp.addr);
562 ret = 0; 402 ret = -EINVAL;
563 } else if (ret == -EILSEQ) {
564 pr_warn("Probing address(0x%p) is not an instruction boundary.\n",
565 tk->rp.kp.addr);
566 ret = -EINVAL;
567 }
568 } 403 }
569
570 return ret; 404 return ret;
571} 405}
572 406
@@ -629,6 +463,11 @@ static int register_trace_kprobe(struct trace_kprobe *tk)
629 463
630 /* Register k*probe */ 464 /* Register k*probe */
631 ret = __register_trace_kprobe(tk); 465 ret = __register_trace_kprobe(tk);
466 if (ret == -ENOENT && !trace_kprobe_module_exist(tk)) {
467 pr_warn("This probe might be able to register after target module is loaded. Continue.\n");
468 ret = 0;
469 }
470
632 if (ret < 0) 471 if (ret < 0)
633 unregister_kprobe_event(tk); 472 unregister_kprobe_event(tk);
634 else 473 else
@@ -713,13 +552,15 @@ static int create_trace_kprobe(int argc, char **argv)
713 long offset = 0; 552 long offset = 0;
714 void *addr = NULL; 553 void *addr = NULL;
715 char buf[MAX_EVENT_NAME_LEN]; 554 char buf[MAX_EVENT_NAME_LEN];
555 unsigned int flags = TPARG_FL_KERNEL;
716 556
717 /* argc must be >= 1 */ 557 /* argc must be >= 1 */
718 if (argv[0][0] == 'p') 558 if (argv[0][0] == 'p')
719 is_return = false; 559 is_return = false;
720 else if (argv[0][0] == 'r') 560 else if (argv[0][0] == 'r') {
721 is_return = true; 561 is_return = true;
722 else if (argv[0][0] == '-') 562 flags |= TPARG_FL_RETURN;
563 } else if (argv[0][0] == '-')
723 is_delete = true; 564 is_delete = true;
724 else { 565 else {
725 pr_info("Probe definition must be started with 'p', 'r' or" 566 pr_info("Probe definition must be started with 'p', 'r' or"
@@ -749,10 +590,13 @@ static int create_trace_kprobe(int argc, char **argv)
749 } 590 }
750 591
751 if (event) { 592 if (event) {
752 if (strchr(event, '/')) { 593 char *slash;
594
595 slash = strchr(event, '/');
596 if (slash) {
753 group = event; 597 group = event;
754 event = strchr(group, '/') + 1; 598 event = slash + 1;
755 event[-1] = '\0'; 599 slash[0] = '\0';
756 if (strlen(group) == 0) { 600 if (strlen(group) == 0) {
757 pr_info("Group name is not specified\n"); 601 pr_info("Group name is not specified\n");
758 return -EINVAL; 602 return -EINVAL;
@@ -802,8 +646,9 @@ static int create_trace_kprobe(int argc, char **argv)
802 pr_info("Failed to parse either an address or a symbol.\n"); 646 pr_info("Failed to parse either an address or a symbol.\n");
803 return ret; 647 return ret;
804 } 648 }
805 if (offset && is_return && 649 if (kprobe_on_func_entry(NULL, symbol, offset))
806 !kprobe_on_func_entry(NULL, symbol, offset)) { 650 flags |= TPARG_FL_FENTRY;
651 if (offset && is_return && !(flags & TPARG_FL_FENTRY)) {
807 pr_info("Given offset is not valid for return probe.\n"); 652 pr_info("Given offset is not valid for return probe.\n");
808 return -EINVAL; 653 return -EINVAL;
809 } 654 }
@@ -873,8 +718,7 @@ static int create_trace_kprobe(int argc, char **argv)
873 718
874 /* Parse fetch argument */ 719 /* Parse fetch argument */
875 ret = traceprobe_parse_probe_arg(arg, &tk->tp.size, parg, 720 ret = traceprobe_parse_probe_arg(arg, &tk->tp.size, parg,
876 is_return, true, 721 flags);
877 kprobes_fetch_type_table);
878 if (ret) { 722 if (ret) {
879 pr_info("Parse error at argument[%d]. (%d)\n", i, ret); 723 pr_info("Parse error at argument[%d]. (%d)\n", i, ret);
880 goto error; 724 goto error;
@@ -1028,6 +872,106 @@ static const struct file_operations kprobe_profile_ops = {
1028 .release = seq_release, 872 .release = seq_release,
1029}; 873};
1030 874
875/* Kprobe specific fetch functions */
876
877/* Return the length of string -- including null terminal byte */
878static nokprobe_inline int
879fetch_store_strlen(unsigned long addr)
880{
881 mm_segment_t old_fs;
882 int ret, len = 0;
883 u8 c;
884
885 old_fs = get_fs();
886 set_fs(KERNEL_DS);
887 pagefault_disable();
888
889 do {
890 ret = __copy_from_user_inatomic(&c, (u8 *)addr + len, 1);
891 len++;
892 } while (c && ret == 0 && len < MAX_STRING_SIZE);
893
894 pagefault_enable();
895 set_fs(old_fs);
896
897 return (ret < 0) ? ret : len;
898}
899
900/*
901 * Fetch a null-terminated string. Caller MUST set *(u32 *)buf with max
902 * length and relative data location.
903 */
904static nokprobe_inline int
905fetch_store_string(unsigned long addr, void *dest, void *base)
906{
907 int maxlen = get_loc_len(*(u32 *)dest);
908 u8 *dst = get_loc_data(dest, base);
909 long ret;
910
911 if (unlikely(!maxlen))
912 return -ENOMEM;
913 /*
914 * Try to get string again, since the string can be changed while
915 * probing.
916 */
917 ret = strncpy_from_unsafe(dst, (void *)addr, maxlen);
918
919 if (ret >= 0)
920 *(u32 *)dest = make_data_loc(ret, (void *)dst - base);
921 return ret;
922}
923
924static nokprobe_inline int
925probe_mem_read(void *dest, void *src, size_t size)
926{
927 return probe_kernel_read(dest, src, size);
928}
929
930/* Note that we don't verify it, since the code does not come from user space */
931static int
932process_fetch_insn(struct fetch_insn *code, struct pt_regs *regs, void *dest,
933 void *base)
934{
935 unsigned long val;
936
937retry:
938 /* 1st stage: get value from context */
939 switch (code->op) {
940 case FETCH_OP_REG:
941 val = regs_get_register(regs, code->param);
942 break;
943 case FETCH_OP_STACK:
944 val = regs_get_kernel_stack_nth(regs, code->param);
945 break;
946 case FETCH_OP_STACKP:
947 val = kernel_stack_pointer(regs);
948 break;
949 case FETCH_OP_RETVAL:
950 val = regs_return_value(regs);
951 break;
952 case FETCH_OP_IMM:
953 val = code->immediate;
954 break;
955 case FETCH_OP_COMM:
956 val = (unsigned long)current->comm;
957 break;
958#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
959 case FETCH_OP_ARG:
960 val = regs_get_kernel_argument(regs, code->param);
961 break;
962#endif
963 case FETCH_NOP_SYMBOL: /* Ignore a place holder */
964 code++;
965 goto retry;
966 default:
967 return -EILSEQ;
968 }
969 code++;
970
971 return process_fetch_insn_bottom(code, val, dest, base);
972}
973NOKPROBE_SYMBOL(process_fetch_insn)
974
1031/* Kprobe handler */ 975/* Kprobe handler */
1032static nokprobe_inline void 976static nokprobe_inline void
1033__kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs, 977__kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs,
@@ -1059,7 +1003,7 @@ __kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs,
1059 1003
1060 entry = ring_buffer_event_data(event); 1004 entry = ring_buffer_event_data(event);
1061 entry->ip = (unsigned long)tk->rp.kp.addr; 1005 entry->ip = (unsigned long)tk->rp.kp.addr;
1062 store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize); 1006 store_trace_args(&entry[1], &tk->tp, regs, sizeof(*entry), dsize);
1063 1007
1064 event_trigger_unlock_commit_regs(trace_file, buffer, event, 1008 event_trigger_unlock_commit_regs(trace_file, buffer, event,
1065 entry, irq_flags, pc, regs); 1009 entry, irq_flags, pc, regs);
@@ -1108,7 +1052,7 @@ __kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
1108 entry = ring_buffer_event_data(event); 1052 entry = ring_buffer_event_data(event);
1109 entry->func = (unsigned long)tk->rp.kp.addr; 1053 entry->func = (unsigned long)tk->rp.kp.addr;
1110 entry->ret_ip = (unsigned long)ri->ret_addr; 1054 entry->ret_ip = (unsigned long)ri->ret_addr;
1111 store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize); 1055 store_trace_args(&entry[1], &tk->tp, regs, sizeof(*entry), dsize);
1112 1056
1113 event_trigger_unlock_commit_regs(trace_file, buffer, event, 1057 event_trigger_unlock_commit_regs(trace_file, buffer, event,
1114 entry, irq_flags, pc, regs); 1058 entry, irq_flags, pc, regs);
@@ -1133,8 +1077,6 @@ print_kprobe_event(struct trace_iterator *iter, int flags,
1133 struct kprobe_trace_entry_head *field; 1077 struct kprobe_trace_entry_head *field;
1134 struct trace_seq *s = &iter->seq; 1078 struct trace_seq *s = &iter->seq;
1135 struct trace_probe *tp; 1079 struct trace_probe *tp;
1136 u8 *data;
1137 int i;
1138 1080
1139 field = (struct kprobe_trace_entry_head *)iter->ent; 1081 field = (struct kprobe_trace_entry_head *)iter->ent;
1140 tp = container_of(event, struct trace_probe, call.event); 1082 tp = container_of(event, struct trace_probe, call.event);
@@ -1146,11 +1088,9 @@ print_kprobe_event(struct trace_iterator *iter, int flags,
1146 1088
1147 trace_seq_putc(s, ')'); 1089 trace_seq_putc(s, ')');
1148 1090
1149 data = (u8 *)&field[1]; 1091 if (print_probe_args(s, tp->args, tp->nr_args,
1150 for (i = 0; i < tp->nr_args; i++) 1092 (u8 *)&field[1], field) < 0)
1151 if (!tp->args[i].type->print(s, tp->args[i].name, 1093 goto out;
1152 data + tp->args[i].offset, field))
1153 goto out;
1154 1094
1155 trace_seq_putc(s, '\n'); 1095 trace_seq_putc(s, '\n');
1156 out: 1096 out:
@@ -1164,8 +1104,6 @@ print_kretprobe_event(struct trace_iterator *iter, int flags,
1164 struct kretprobe_trace_entry_head *field; 1104 struct kretprobe_trace_entry_head *field;
1165 struct trace_seq *s = &iter->seq; 1105 struct trace_seq *s = &iter->seq;
1166 struct trace_probe *tp; 1106 struct trace_probe *tp;
1167 u8 *data;
1168 int i;
1169 1107
1170 field = (struct kretprobe_trace_entry_head *)iter->ent; 1108 field = (struct kretprobe_trace_entry_head *)iter->ent;
1171 tp = container_of(event, struct trace_probe, call.event); 1109 tp = container_of(event, struct trace_probe, call.event);
@@ -1182,11 +1120,9 @@ print_kretprobe_event(struct trace_iterator *iter, int flags,
1182 1120
1183 trace_seq_putc(s, ')'); 1121 trace_seq_putc(s, ')');
1184 1122
1185 data = (u8 *)&field[1]; 1123 if (print_probe_args(s, tp->args, tp->nr_args,
1186 for (i = 0; i < tp->nr_args; i++) 1124 (u8 *)&field[1], field) < 0)
1187 if (!tp->args[i].type->print(s, tp->args[i].name, 1125 goto out;
1188 data + tp->args[i].offset, field))
1189 goto out;
1190 1126
1191 trace_seq_putc(s, '\n'); 1127 trace_seq_putc(s, '\n');
1192 1128
@@ -1197,49 +1133,25 @@ print_kretprobe_event(struct trace_iterator *iter, int flags,
1197 1133
1198static int kprobe_event_define_fields(struct trace_event_call *event_call) 1134static int kprobe_event_define_fields(struct trace_event_call *event_call)
1199{ 1135{
1200 int ret, i; 1136 int ret;
1201 struct kprobe_trace_entry_head field; 1137 struct kprobe_trace_entry_head field;
1202 struct trace_kprobe *tk = (struct trace_kprobe *)event_call->data; 1138 struct trace_kprobe *tk = (struct trace_kprobe *)event_call->data;
1203 1139
1204 DEFINE_FIELD(unsigned long, ip, FIELD_STRING_IP, 0); 1140 DEFINE_FIELD(unsigned long, ip, FIELD_STRING_IP, 0);
1205 /* Set argument names as fields */
1206 for (i = 0; i < tk->tp.nr_args; i++) {
1207 struct probe_arg *parg = &tk->tp.args[i];
1208 1141
1209 ret = trace_define_field(event_call, parg->type->fmttype, 1142 return traceprobe_define_arg_fields(event_call, sizeof(field), &tk->tp);
1210 parg->name,
1211 sizeof(field) + parg->offset,
1212 parg->type->size,
1213 parg->type->is_signed,
1214 FILTER_OTHER);
1215 if (ret)
1216 return ret;
1217 }
1218 return 0;
1219} 1143}
1220 1144
1221static int kretprobe_event_define_fields(struct trace_event_call *event_call) 1145static int kretprobe_event_define_fields(struct trace_event_call *event_call)
1222{ 1146{
1223 int ret, i; 1147 int ret;
1224 struct kretprobe_trace_entry_head field; 1148 struct kretprobe_trace_entry_head field;
1225 struct trace_kprobe *tk = (struct trace_kprobe *)event_call->data; 1149 struct trace_kprobe *tk = (struct trace_kprobe *)event_call->data;
1226 1150
1227 DEFINE_FIELD(unsigned long, func, FIELD_STRING_FUNC, 0); 1151 DEFINE_FIELD(unsigned long, func, FIELD_STRING_FUNC, 0);
1228 DEFINE_FIELD(unsigned long, ret_ip, FIELD_STRING_RETIP, 0); 1152 DEFINE_FIELD(unsigned long, ret_ip, FIELD_STRING_RETIP, 0);
1229 /* Set argument names as fields */
1230 for (i = 0; i < tk->tp.nr_args; i++) {
1231 struct probe_arg *parg = &tk->tp.args[i];
1232 1153
1233 ret = trace_define_field(event_call, parg->type->fmttype, 1154 return traceprobe_define_arg_fields(event_call, sizeof(field), &tk->tp);
1234 parg->name,
1235 sizeof(field) + parg->offset,
1236 parg->type->size,
1237 parg->type->is_signed,
1238 FILTER_OTHER);
1239 if (ret)
1240 return ret;
1241 }
1242 return 0;
1243} 1155}
1244 1156
1245#ifdef CONFIG_PERF_EVENTS 1157#ifdef CONFIG_PERF_EVENTS
@@ -1286,7 +1198,7 @@ kprobe_perf_func(struct trace_kprobe *tk, struct pt_regs *regs)
1286 1198
1287 entry->ip = (unsigned long)tk->rp.kp.addr; 1199 entry->ip = (unsigned long)tk->rp.kp.addr;
1288 memset(&entry[1], 0, dsize); 1200 memset(&entry[1], 0, dsize);
1289 store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize); 1201 store_trace_args(&entry[1], &tk->tp, regs, sizeof(*entry), dsize);
1290 perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs, 1202 perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
1291 head, NULL); 1203 head, NULL);
1292 return 0; 1204 return 0;
@@ -1322,7 +1234,7 @@ kretprobe_perf_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
1322 1234
1323 entry->func = (unsigned long)tk->rp.kp.addr; 1235 entry->func = (unsigned long)tk->rp.kp.addr;
1324 entry->ret_ip = (unsigned long)ri->ret_addr; 1236 entry->ret_ip = (unsigned long)ri->ret_addr;
1325 store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize); 1237 store_trace_args(&entry[1], &tk->tp, regs, sizeof(*entry), dsize);
1326 perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs, 1238 perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
1327 head, NULL); 1239 head, NULL);
1328} 1240}
@@ -1457,7 +1369,7 @@ static int register_kprobe_event(struct trace_kprobe *tk)
1457 1369
1458 init_trace_event_call(tk, call); 1370 init_trace_event_call(tk, call);
1459 1371
1460 if (set_print_fmt(&tk->tp, trace_kprobe_is_return(tk)) < 0) 1372 if (traceprobe_set_print_fmt(&tk->tp, trace_kprobe_is_return(tk)) < 0)
1461 return -ENOMEM; 1373 return -ENOMEM;
1462 ret = register_trace_event(&call->event); 1374 ret = register_trace_event(&call->event);
1463 if (!ret) { 1375 if (!ret) {
@@ -1514,7 +1426,7 @@ create_local_trace_kprobe(char *func, void *addr, unsigned long offs,
1514 1426
1515 init_trace_event_call(tk, &tk->tp.call); 1427 init_trace_event_call(tk, &tk->tp.call);
1516 1428
1517 if (set_print_fmt(&tk->tp, trace_kprobe_is_return(tk)) < 0) { 1429 if (traceprobe_set_print_fmt(&tk->tp, trace_kprobe_is_return(tk)) < 0) {
1518 ret = -ENOMEM; 1430 ret = -ENOMEM;
1519 goto error; 1431 goto error;
1520 } 1432 }
diff --git a/kernel/trace/trace_probe.c b/kernel/trace/trace_probe.c
index e99c3ce7aa65..3ef15a6683c0 100644
--- a/kernel/trace/trace_probe.c
+++ b/kernel/trace/trace_probe.c
@@ -26,14 +26,12 @@ const char *reserved_field_names[] = {
26 26
27/* Printing in basic type function template */ 27/* Printing in basic type function template */
28#define DEFINE_BASIC_PRINT_TYPE_FUNC(tname, type, fmt) \ 28#define DEFINE_BASIC_PRINT_TYPE_FUNC(tname, type, fmt) \
29int PRINT_TYPE_FUNC_NAME(tname)(struct trace_seq *s, const char *name, \ 29int PRINT_TYPE_FUNC_NAME(tname)(struct trace_seq *s, void *data, void *ent)\
30 void *data, void *ent) \
31{ \ 30{ \
32 trace_seq_printf(s, " %s=" fmt, name, *(type *)data); \ 31 trace_seq_printf(s, fmt, *(type *)data); \
33 return !trace_seq_has_overflowed(s); \ 32 return !trace_seq_has_overflowed(s); \
34} \ 33} \
35const char PRINT_TYPE_FMT_NAME(tname)[] = fmt; \ 34const char PRINT_TYPE_FMT_NAME(tname)[] = fmt;
36NOKPROBE_SYMBOL(PRINT_TYPE_FUNC_NAME(tname));
37 35
38DEFINE_BASIC_PRINT_TYPE_FUNC(u8, u8, "%u") 36DEFINE_BASIC_PRINT_TYPE_FUNC(u8, u8, "%u")
39DEFINE_BASIC_PRINT_TYPE_FUNC(u16, u16, "%u") 37DEFINE_BASIC_PRINT_TYPE_FUNC(u16, u16, "%u")
@@ -48,193 +46,52 @@ DEFINE_BASIC_PRINT_TYPE_FUNC(x16, u16, "0x%x")
48DEFINE_BASIC_PRINT_TYPE_FUNC(x32, u32, "0x%x") 46DEFINE_BASIC_PRINT_TYPE_FUNC(x32, u32, "0x%x")
49DEFINE_BASIC_PRINT_TYPE_FUNC(x64, u64, "0x%Lx") 47DEFINE_BASIC_PRINT_TYPE_FUNC(x64, u64, "0x%Lx")
50 48
49int PRINT_TYPE_FUNC_NAME(symbol)(struct trace_seq *s, void *data, void *ent)
50{
51 trace_seq_printf(s, "%pS", (void *)*(unsigned long *)data);
52 return !trace_seq_has_overflowed(s);
53}
54const char PRINT_TYPE_FMT_NAME(symbol)[] = "%pS";
55
51/* Print type function for string type */ 56/* Print type function for string type */
52int PRINT_TYPE_FUNC_NAME(string)(struct trace_seq *s, const char *name, 57int PRINT_TYPE_FUNC_NAME(string)(struct trace_seq *s, void *data, void *ent)
53 void *data, void *ent)
54{ 58{
55 int len = *(u32 *)data >> 16; 59 int len = *(u32 *)data >> 16;
56 60
57 if (!len) 61 if (!len)
58 trace_seq_printf(s, " %s=(fault)", name); 62 trace_seq_puts(s, "(fault)");
59 else 63 else
60 trace_seq_printf(s, " %s=\"%s\"", name, 64 trace_seq_printf(s, "\"%s\"",
61 (const char *)get_loc_data(data, ent)); 65 (const char *)get_loc_data(data, ent));
62 return !trace_seq_has_overflowed(s); 66 return !trace_seq_has_overflowed(s);
63} 67}
64NOKPROBE_SYMBOL(PRINT_TYPE_FUNC_NAME(string));
65 68
66const char PRINT_TYPE_FMT_NAME(string)[] = "\\\"%s\\\""; 69const char PRINT_TYPE_FMT_NAME(string)[] = "\\\"%s\\\"";
67 70
68#define CHECK_FETCH_FUNCS(method, fn) \ 71/* Fetch type information table */
69 (((FETCH_FUNC_NAME(method, u8) == fn) || \ 72static const struct fetch_type probe_fetch_types[] = {
70 (FETCH_FUNC_NAME(method, u16) == fn) || \ 73 /* Special types */
71 (FETCH_FUNC_NAME(method, u32) == fn) || \ 74 __ASSIGN_FETCH_TYPE("string", string, string, sizeof(u32), 1,
72 (FETCH_FUNC_NAME(method, u64) == fn) || \ 75 "__data_loc char[]"),
73 (FETCH_FUNC_NAME(method, string) == fn) || \ 76 /* Basic types */
74 (FETCH_FUNC_NAME(method, string_size) == fn)) \ 77 ASSIGN_FETCH_TYPE(u8, u8, 0),
75 && (fn != NULL)) 78 ASSIGN_FETCH_TYPE(u16, u16, 0),
76 79 ASSIGN_FETCH_TYPE(u32, u32, 0),
77/* Data fetch function templates */ 80 ASSIGN_FETCH_TYPE(u64, u64, 0),
78#define DEFINE_FETCH_reg(type) \ 81 ASSIGN_FETCH_TYPE(s8, u8, 1),
79void FETCH_FUNC_NAME(reg, type)(struct pt_regs *regs, void *offset, void *dest) \ 82 ASSIGN_FETCH_TYPE(s16, u16, 1),
80{ \ 83 ASSIGN_FETCH_TYPE(s32, u32, 1),
81 *(type *)dest = (type)regs_get_register(regs, \ 84 ASSIGN_FETCH_TYPE(s64, u64, 1),
82 (unsigned int)((unsigned long)offset)); \ 85 ASSIGN_FETCH_TYPE_ALIAS(x8, u8, u8, 0),
83} \ 86 ASSIGN_FETCH_TYPE_ALIAS(x16, u16, u16, 0),
84NOKPROBE_SYMBOL(FETCH_FUNC_NAME(reg, type)); 87 ASSIGN_FETCH_TYPE_ALIAS(x32, u32, u32, 0),
85DEFINE_BASIC_FETCH_FUNCS(reg) 88 ASSIGN_FETCH_TYPE_ALIAS(x64, u64, u64, 0),
86/* No string on the register */ 89 ASSIGN_FETCH_TYPE_ALIAS(symbol, ADDR_FETCH_TYPE, ADDR_FETCH_TYPE, 0),
87#define fetch_reg_string NULL 90
88#define fetch_reg_string_size NULL 91 ASSIGN_FETCH_TYPE_END
89
90#define DEFINE_FETCH_retval(type) \
91void FETCH_FUNC_NAME(retval, type)(struct pt_regs *regs, \
92 void *dummy, void *dest) \
93{ \
94 *(type *)dest = (type)regs_return_value(regs); \
95} \
96NOKPROBE_SYMBOL(FETCH_FUNC_NAME(retval, type));
97DEFINE_BASIC_FETCH_FUNCS(retval)
98/* No string on the retval */
99#define fetch_retval_string NULL
100#define fetch_retval_string_size NULL
101
102/* Dereference memory access function */
103struct deref_fetch_param {
104 struct fetch_param orig;
105 long offset;
106 fetch_func_t fetch;
107 fetch_func_t fetch_size;
108}; 92};
109 93
110#define DEFINE_FETCH_deref(type) \ 94static const struct fetch_type *find_fetch_type(const char *type)
111void FETCH_FUNC_NAME(deref, type)(struct pt_regs *regs, \
112 void *data, void *dest) \
113{ \
114 struct deref_fetch_param *dprm = data; \
115 unsigned long addr; \
116 call_fetch(&dprm->orig, regs, &addr); \
117 if (addr) { \
118 addr += dprm->offset; \
119 dprm->fetch(regs, (void *)addr, dest); \
120 } else \
121 *(type *)dest = 0; \
122} \
123NOKPROBE_SYMBOL(FETCH_FUNC_NAME(deref, type));
124DEFINE_BASIC_FETCH_FUNCS(deref)
125DEFINE_FETCH_deref(string)
126
127void FETCH_FUNC_NAME(deref, string_size)(struct pt_regs *regs,
128 void *data, void *dest)
129{
130 struct deref_fetch_param *dprm = data;
131 unsigned long addr;
132
133 call_fetch(&dprm->orig, regs, &addr);
134 if (addr && dprm->fetch_size) {
135 addr += dprm->offset;
136 dprm->fetch_size(regs, (void *)addr, dest);
137 } else
138 *(string_size *)dest = 0;
139}
140NOKPROBE_SYMBOL(FETCH_FUNC_NAME(deref, string_size));
141
142static void update_deref_fetch_param(struct deref_fetch_param *data)
143{
144 if (CHECK_FETCH_FUNCS(deref, data->orig.fn))
145 update_deref_fetch_param(data->orig.data);
146 else if (CHECK_FETCH_FUNCS(symbol, data->orig.fn))
147 update_symbol_cache(data->orig.data);
148}
149NOKPROBE_SYMBOL(update_deref_fetch_param);
150
151static void free_deref_fetch_param(struct deref_fetch_param *data)
152{
153 if (CHECK_FETCH_FUNCS(deref, data->orig.fn))
154 free_deref_fetch_param(data->orig.data);
155 else if (CHECK_FETCH_FUNCS(symbol, data->orig.fn))
156 free_symbol_cache(data->orig.data);
157 kfree(data);
158}
159NOKPROBE_SYMBOL(free_deref_fetch_param);
160
161/* Bitfield fetch function */
162struct bitfield_fetch_param {
163 struct fetch_param orig;
164 unsigned char hi_shift;
165 unsigned char low_shift;
166};
167
168#define DEFINE_FETCH_bitfield(type) \
169void FETCH_FUNC_NAME(bitfield, type)(struct pt_regs *regs, \
170 void *data, void *dest) \
171{ \
172 struct bitfield_fetch_param *bprm = data; \
173 type buf = 0; \
174 call_fetch(&bprm->orig, regs, &buf); \
175 if (buf) { \
176 buf <<= bprm->hi_shift; \
177 buf >>= bprm->low_shift; \
178 } \
179 *(type *)dest = buf; \
180} \
181NOKPROBE_SYMBOL(FETCH_FUNC_NAME(bitfield, type));
182DEFINE_BASIC_FETCH_FUNCS(bitfield)
183#define fetch_bitfield_string NULL
184#define fetch_bitfield_string_size NULL
185
186static void
187update_bitfield_fetch_param(struct bitfield_fetch_param *data)
188{
189 /*
190 * Don't check the bitfield itself, because this must be the
191 * last fetch function.
192 */
193 if (CHECK_FETCH_FUNCS(deref, data->orig.fn))
194 update_deref_fetch_param(data->orig.data);
195 else if (CHECK_FETCH_FUNCS(symbol, data->orig.fn))
196 update_symbol_cache(data->orig.data);
197}
198
199static void
200free_bitfield_fetch_param(struct bitfield_fetch_param *data)
201{
202 /*
203 * Don't check the bitfield itself, because this must be the
204 * last fetch function.
205 */
206 if (CHECK_FETCH_FUNCS(deref, data->orig.fn))
207 free_deref_fetch_param(data->orig.data);
208 else if (CHECK_FETCH_FUNCS(symbol, data->orig.fn))
209 free_symbol_cache(data->orig.data);
210
211 kfree(data);
212}
213
214void FETCH_FUNC_NAME(comm, string)(struct pt_regs *regs,
215 void *data, void *dest)
216{
217 int maxlen = get_rloc_len(*(u32 *)dest);
218 u8 *dst = get_rloc_data(dest);
219 long ret;
220
221 if (!maxlen)
222 return;
223
224 ret = strlcpy(dst, current->comm, maxlen);
225 *(u32 *)dest = make_data_rloc(ret, get_rloc_offs(*(u32 *)dest));
226}
227NOKPROBE_SYMBOL(FETCH_FUNC_NAME(comm, string));
228
229void FETCH_FUNC_NAME(comm, string_size)(struct pt_regs *regs,
230 void *data, void *dest)
231{
232 *(u32 *)dest = strlen(current->comm) + 1;
233}
234NOKPROBE_SYMBOL(FETCH_FUNC_NAME(comm, string_size));
235
236static const struct fetch_type *find_fetch_type(const char *type,
237 const struct fetch_type *ftbl)
238{ 95{
239 int i; 96 int i;
240 97
@@ -255,58 +112,27 @@ static const struct fetch_type *find_fetch_type(const char *type,
255 112
256 switch (bs) { 113 switch (bs) {
257 case 8: 114 case 8:
258 return find_fetch_type("u8", ftbl); 115 return find_fetch_type("u8");
259 case 16: 116 case 16:
260 return find_fetch_type("u16", ftbl); 117 return find_fetch_type("u16");
261 case 32: 118 case 32:
262 return find_fetch_type("u32", ftbl); 119 return find_fetch_type("u32");
263 case 64: 120 case 64:
264 return find_fetch_type("u64", ftbl); 121 return find_fetch_type("u64");
265 default: 122 default:
266 goto fail; 123 goto fail;
267 } 124 }
268 } 125 }
269 126
270 for (i = 0; ftbl[i].name; i++) { 127 for (i = 0; probe_fetch_types[i].name; i++) {
271 if (strcmp(type, ftbl[i].name) == 0) 128 if (strcmp(type, probe_fetch_types[i].name) == 0)
272 return &ftbl[i]; 129 return &probe_fetch_types[i];
273 } 130 }
274 131
275fail: 132fail:
276 return NULL; 133 return NULL;
277} 134}
278 135
279/* Special function : only accept unsigned long */
280static void fetch_kernel_stack_address(struct pt_regs *regs, void *dummy, void *dest)
281{
282 *(unsigned long *)dest = kernel_stack_pointer(regs);
283}
284NOKPROBE_SYMBOL(fetch_kernel_stack_address);
285
286static void fetch_user_stack_address(struct pt_regs *regs, void *dummy, void *dest)
287{
288 *(unsigned long *)dest = user_stack_pointer(regs);
289}
290NOKPROBE_SYMBOL(fetch_user_stack_address);
291
292static fetch_func_t get_fetch_size_function(const struct fetch_type *type,
293 fetch_func_t orig_fn,
294 const struct fetch_type *ftbl)
295{
296 int i;
297
298 if (type != &ftbl[FETCH_TYPE_STRING])
299 return NULL; /* Only string type needs size function */
300
301 for (i = 0; i < FETCH_MTD_END; i++)
302 if (type->fetch[i] == orig_fn)
303 return ftbl[FETCH_TYPE_STRSIZE].fetch[i];
304
305 WARN_ON(1); /* This should not happen */
306
307 return NULL;
308}
309
310/* Split symbol and offset. */ 136/* Split symbol and offset. */
311int traceprobe_split_symbol_offset(char *symbol, long *offset) 137int traceprobe_split_symbol_offset(char *symbol, long *offset)
312{ 138{
@@ -331,41 +157,44 @@ int traceprobe_split_symbol_offset(char *symbol, long *offset)
331#define PARAM_MAX_STACK (THREAD_SIZE / sizeof(unsigned long)) 157#define PARAM_MAX_STACK (THREAD_SIZE / sizeof(unsigned long))
332 158
333static int parse_probe_vars(char *arg, const struct fetch_type *t, 159static int parse_probe_vars(char *arg, const struct fetch_type *t,
334 struct fetch_param *f, bool is_return, 160 struct fetch_insn *code, unsigned int flags)
335 bool is_kprobe)
336{ 161{
337 int ret = 0; 162 int ret = 0;
338 unsigned long param; 163 unsigned long param;
339 164
340 if (strcmp(arg, "retval") == 0) { 165 if (strcmp(arg, "retval") == 0) {
341 if (is_return) 166 if (flags & TPARG_FL_RETURN)
342 f->fn = t->fetch[FETCH_MTD_retval]; 167 code->op = FETCH_OP_RETVAL;
343 else 168 else
344 ret = -EINVAL; 169 ret = -EINVAL;
345 } else if (strncmp(arg, "stack", 5) == 0) { 170 } else if (strncmp(arg, "stack", 5) == 0) {
346 if (arg[5] == '\0') { 171 if (arg[5] == '\0') {
347 if (strcmp(t->name, DEFAULT_FETCH_TYPE_STR)) 172 code->op = FETCH_OP_STACKP;
348 return -EINVAL;
349
350 if (is_kprobe)
351 f->fn = fetch_kernel_stack_address;
352 else
353 f->fn = fetch_user_stack_address;
354 } else if (isdigit(arg[5])) { 173 } else if (isdigit(arg[5])) {
355 ret = kstrtoul(arg + 5, 10, &param); 174 ret = kstrtoul(arg + 5, 10, &param);
356 if (ret || (is_kprobe && param > PARAM_MAX_STACK)) 175 if (ret || ((flags & TPARG_FL_KERNEL) &&
176 param > PARAM_MAX_STACK))
357 ret = -EINVAL; 177 ret = -EINVAL;
358 else { 178 else {
359 f->fn = t->fetch[FETCH_MTD_stack]; 179 code->op = FETCH_OP_STACK;
360 f->data = (void *)param; 180 code->param = (unsigned int)param;
361 } 181 }
362 } else 182 } else
363 ret = -EINVAL; 183 ret = -EINVAL;
364 } else if (strcmp(arg, "comm") == 0) { 184 } else if (strcmp(arg, "comm") == 0) {
365 if (strcmp(t->name, "string") != 0 && 185 code->op = FETCH_OP_COMM;
366 strcmp(t->name, "string_size") != 0) 186#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
187 } else if (((flags & TPARG_FL_MASK) ==
188 (TPARG_FL_KERNEL | TPARG_FL_FENTRY)) &&
189 strncmp(arg, "arg", 3) == 0) {
190 if (!isdigit(arg[3]))
191 return -EINVAL;
192 ret = kstrtoul(arg + 3, 10, &param);
193 if (ret || !param || param > PARAM_MAX_STACK)
367 return -EINVAL; 194 return -EINVAL;
368 f->fn = t->fetch[FETCH_MTD_comm]; 195 code->op = FETCH_OP_ARG;
196 code->param = (unsigned int)param - 1;
197#endif
369 } else 198 } else
370 ret = -EINVAL; 199 ret = -EINVAL;
371 200
@@ -373,25 +202,27 @@ static int parse_probe_vars(char *arg, const struct fetch_type *t,
373} 202}
374 203
375/* Recursive argument parser */ 204/* Recursive argument parser */
376static int parse_probe_arg(char *arg, const struct fetch_type *t, 205static int
377 struct fetch_param *f, bool is_return, bool is_kprobe, 206parse_probe_arg(char *arg, const struct fetch_type *type,
378 const struct fetch_type *ftbl) 207 struct fetch_insn **pcode, struct fetch_insn *end,
208 unsigned int flags)
379{ 209{
210 struct fetch_insn *code = *pcode;
380 unsigned long param; 211 unsigned long param;
381 long offset; 212 long offset = 0;
382 char *tmp; 213 char *tmp;
383 int ret = 0; 214 int ret = 0;
384 215
385 switch (arg[0]) { 216 switch (arg[0]) {
386 case '$': 217 case '$':
387 ret = parse_probe_vars(arg + 1, t, f, is_return, is_kprobe); 218 ret = parse_probe_vars(arg + 1, type, code, flags);
388 break; 219 break;
389 220
390 case '%': /* named register */ 221 case '%': /* named register */
391 ret = regs_query_register_offset(arg + 1); 222 ret = regs_query_register_offset(arg + 1);
392 if (ret >= 0) { 223 if (ret >= 0) {
393 f->fn = t->fetch[FETCH_MTD_reg]; 224 code->op = FETCH_OP_REG;
394 f->data = (void *)(unsigned long)ret; 225 code->param = (unsigned int)ret;
395 ret = 0; 226 ret = 0;
396 } 227 }
397 break; 228 break;
@@ -401,33 +232,42 @@ static int parse_probe_arg(char *arg, const struct fetch_type *t,
401 ret = kstrtoul(arg + 1, 0, &param); 232 ret = kstrtoul(arg + 1, 0, &param);
402 if (ret) 233 if (ret)
403 break; 234 break;
404 235 /* load address */
405 f->fn = t->fetch[FETCH_MTD_memory]; 236 code->op = FETCH_OP_IMM;
406 f->data = (void *)param; 237 code->immediate = param;
407 } else if (arg[1] == '+') { 238 } else if (arg[1] == '+') {
408 /* kprobes don't support file offsets */ 239 /* kprobes don't support file offsets */
409 if (is_kprobe) 240 if (flags & TPARG_FL_KERNEL)
410 return -EINVAL; 241 return -EINVAL;
411 242
412 ret = kstrtol(arg + 2, 0, &offset); 243 ret = kstrtol(arg + 2, 0, &offset);
413 if (ret) 244 if (ret)
414 break; 245 break;
415 246
416 f->fn = t->fetch[FETCH_MTD_file_offset]; 247 code->op = FETCH_OP_FOFFS;
417 f->data = (void *)offset; 248 code->immediate = (unsigned long)offset; // imm64?
418 } else { 249 } else {
419 /* uprobes don't support symbols */ 250 /* uprobes don't support symbols */
420 if (!is_kprobe) 251 if (!(flags & TPARG_FL_KERNEL))
421 return -EINVAL; 252 return -EINVAL;
422 253
423 ret = traceprobe_split_symbol_offset(arg + 1, &offset); 254 /* Preserve symbol for updating */
424 if (ret) 255 code->op = FETCH_NOP_SYMBOL;
425 break; 256 code->data = kstrdup(arg + 1, GFP_KERNEL);
257 if (!code->data)
258 return -ENOMEM;
259 if (++code == end)
260 return -E2BIG;
426 261
427 f->data = alloc_symbol_cache(arg + 1, offset); 262 code->op = FETCH_OP_IMM;
428 if (f->data) 263 code->immediate = 0;
429 f->fn = t->fetch[FETCH_MTD_symbol];
430 } 264 }
265 /* These are fetching from memory */
266 if (++code == end)
267 return -E2BIG;
268 *pcode = code;
269 code->op = FETCH_OP_DEREF;
270 code->offset = offset;
431 break; 271 break;
432 272
433 case '+': /* deref memory */ 273 case '+': /* deref memory */
@@ -435,11 +275,10 @@ static int parse_probe_arg(char *arg, const struct fetch_type *t,
435 case '-': 275 case '-':
436 tmp = strchr(arg, '('); 276 tmp = strchr(arg, '(');
437 if (!tmp) 277 if (!tmp)
438 break; 278 return -EINVAL;
439 279
440 *tmp = '\0'; 280 *tmp = '\0';
441 ret = kstrtol(arg, 0, &offset); 281 ret = kstrtol(arg, 0, &offset);
442
443 if (ret) 282 if (ret)
444 break; 283 break;
445 284
@@ -447,36 +286,27 @@ static int parse_probe_arg(char *arg, const struct fetch_type *t,
447 tmp = strrchr(arg, ')'); 286 tmp = strrchr(arg, ')');
448 287
449 if (tmp) { 288 if (tmp) {
450 struct deref_fetch_param *dprm; 289 const struct fetch_type *t2 = find_fetch_type(NULL);
451 const struct fetch_type *t2;
452 290
453 t2 = find_fetch_type(NULL, ftbl);
454 *tmp = '\0'; 291 *tmp = '\0';
455 dprm = kzalloc(sizeof(struct deref_fetch_param), GFP_KERNEL); 292 ret = parse_probe_arg(arg, t2, &code, end, flags);
456
457 if (!dprm)
458 return -ENOMEM;
459
460 dprm->offset = offset;
461 dprm->fetch = t->fetch[FETCH_MTD_memory];
462 dprm->fetch_size = get_fetch_size_function(t,
463 dprm->fetch, ftbl);
464 ret = parse_probe_arg(arg, t2, &dprm->orig, is_return,
465 is_kprobe, ftbl);
466 if (ret) 293 if (ret)
467 kfree(dprm); 294 break;
468 else { 295 if (code->op == FETCH_OP_COMM)
469 f->fn = t->fetch[FETCH_MTD_deref]; 296 return -EINVAL;
470 f->data = (void *)dprm; 297 if (++code == end)
471 } 298 return -E2BIG;
299 *pcode = code;
300
301 code->op = FETCH_OP_DEREF;
302 code->offset = offset;
472 } 303 }
473 break; 304 break;
474 } 305 }
475 if (!ret && !f->fn) { /* Parsed, but do not find fetch method */ 306 if (!ret && code->op == FETCH_OP_NOP) {
476 pr_info("%s type has no corresponding fetch method.\n", t->name); 307 /* Parsed, but do not find fetch method */
477 ret = -EINVAL; 308 ret = -EINVAL;
478 } 309 }
479
480 return ret; 310 return ret;
481} 311}
482 312
@@ -485,22 +315,15 @@ static int parse_probe_arg(char *arg, const struct fetch_type *t,
485/* Bitfield type needs to be parsed into a fetch function */ 315/* Bitfield type needs to be parsed into a fetch function */
486static int __parse_bitfield_probe_arg(const char *bf, 316static int __parse_bitfield_probe_arg(const char *bf,
487 const struct fetch_type *t, 317 const struct fetch_type *t,
488 struct fetch_param *f) 318 struct fetch_insn **pcode)
489{ 319{
490 struct bitfield_fetch_param *bprm; 320 struct fetch_insn *code = *pcode;
491 unsigned long bw, bo; 321 unsigned long bw, bo;
492 char *tail; 322 char *tail;
493 323
494 if (*bf != 'b') 324 if (*bf != 'b')
495 return 0; 325 return 0;
496 326
497 bprm = kzalloc(sizeof(*bprm), GFP_KERNEL);
498 if (!bprm)
499 return -ENOMEM;
500
501 bprm->orig = *f;
502 f->fn = t->fetch[FETCH_MTD_bitfield];
503 f->data = (void *)bprm;
504 bw = simple_strtoul(bf + 1, &tail, 0); /* Use simple one */ 327 bw = simple_strtoul(bf + 1, &tail, 0); /* Use simple one */
505 328
506 if (bw == 0 || *tail != '@') 329 if (bw == 0 || *tail != '@')
@@ -511,20 +334,26 @@ static int __parse_bitfield_probe_arg(const char *bf,
511 334
512 if (tail == bf || *tail != '/') 335 if (tail == bf || *tail != '/')
513 return -EINVAL; 336 return -EINVAL;
337 code++;
338 if (code->op != FETCH_OP_NOP)
339 return -E2BIG;
340 *pcode = code;
514 341
515 bprm->hi_shift = BYTES_TO_BITS(t->size) - (bw + bo); 342 code->op = FETCH_OP_MOD_BF;
516 bprm->low_shift = bprm->hi_shift + bo; 343 code->lshift = BYTES_TO_BITS(t->size) - (bw + bo);
344 code->rshift = BYTES_TO_BITS(t->size) - bw;
345 code->basesize = t->size;
517 346
518 return (BYTES_TO_BITS(t->size) < (bw + bo)) ? -EINVAL : 0; 347 return (BYTES_TO_BITS(t->size) < (bw + bo)) ? -EINVAL : 0;
519} 348}
520 349
521/* String length checking wrapper */ 350/* String length checking wrapper */
522int traceprobe_parse_probe_arg(char *arg, ssize_t *size, 351int traceprobe_parse_probe_arg(char *arg, ssize_t *size,
523 struct probe_arg *parg, bool is_return, bool is_kprobe, 352 struct probe_arg *parg, unsigned int flags)
524 const struct fetch_type *ftbl)
525{ 353{
526 const char *t; 354 struct fetch_insn *code, *scode, *tmp = NULL;
527 int ret; 355 char *t, *t2;
356 int ret, len;
528 357
529 if (strlen(arg) > MAX_ARGSTR_LEN) { 358 if (strlen(arg) > MAX_ARGSTR_LEN) {
530 pr_info("Argument is too long.: %s\n", arg); 359 pr_info("Argument is too long.: %s\n", arg);
@@ -535,36 +364,128 @@ int traceprobe_parse_probe_arg(char *arg, ssize_t *size,
535 pr_info("Failed to allocate memory for command '%s'.\n", arg); 364 pr_info("Failed to allocate memory for command '%s'.\n", arg);
536 return -ENOMEM; 365 return -ENOMEM;
537 } 366 }
538 t = strchr(parg->comm, ':'); 367 t = strchr(arg, ':');
539 if (t) { 368 if (t) {
540 arg[t - parg->comm] = '\0'; 369 *t = '\0';
541 t++; 370 t2 = strchr(++t, '[');
371 if (t2) {
372 *t2 = '\0';
373 parg->count = simple_strtoul(t2 + 1, &t2, 0);
374 if (strcmp(t2, "]") || parg->count == 0)
375 return -EINVAL;
376 if (parg->count > MAX_ARRAY_LEN)
377 return -E2BIG;
378 }
542 } 379 }
543 /* 380 /*
544 * The default type of $comm should be "string", and it can't be 381 * The default type of $comm should be "string", and it can't be
545 * dereferenced. 382 * dereferenced.
546 */ 383 */
547 if (!t && strcmp(arg, "$comm") == 0) 384 if (!t && strcmp(arg, "$comm") == 0)
548 t = "string"; 385 parg->type = find_fetch_type("string");
549 parg->type = find_fetch_type(t, ftbl); 386 else
387 parg->type = find_fetch_type(t);
550 if (!parg->type) { 388 if (!parg->type) {
551 pr_info("Unsupported type: %s\n", t); 389 pr_info("Unsupported type: %s\n", t);
552 return -EINVAL; 390 return -EINVAL;
553 } 391 }
554 parg->offset = *size; 392 parg->offset = *size;
555 *size += parg->type->size; 393 *size += parg->type->size * (parg->count ?: 1);
556 ret = parse_probe_arg(arg, parg->type, &parg->fetch, is_return, 394
557 is_kprobe, ftbl); 395 if (parg->count) {
558 396 len = strlen(parg->type->fmttype) + 6;
559 if (ret >= 0 && t != NULL) 397 parg->fmt = kmalloc(len, GFP_KERNEL);
560 ret = __parse_bitfield_probe_arg(t, parg->type, &parg->fetch); 398 if (!parg->fmt)
561 399 return -ENOMEM;
562 if (ret >= 0) { 400 snprintf(parg->fmt, len, "%s[%d]", parg->type->fmttype,
563 parg->fetch_size.fn = get_fetch_size_function(parg->type, 401 parg->count);
564 parg->fetch.fn, 402 }
565 ftbl); 403
566 parg->fetch_size.data = parg->fetch.data; 404 code = tmp = kzalloc(sizeof(*code) * FETCH_INSN_MAX, GFP_KERNEL);
405 if (!code)
406 return -ENOMEM;
407 code[FETCH_INSN_MAX - 1].op = FETCH_OP_END;
408
409 ret = parse_probe_arg(arg, parg->type, &code, &code[FETCH_INSN_MAX - 1],
410 flags);
411 if (ret)
412 goto fail;
413
414 /* Store operation */
415 if (!strcmp(parg->type->name, "string")) {
416 if (code->op != FETCH_OP_DEREF && code->op != FETCH_OP_IMM &&
417 code->op != FETCH_OP_COMM) {
418 pr_info("string only accepts memory or address.\n");
419 ret = -EINVAL;
420 goto fail;
421 }
422 if (code->op != FETCH_OP_DEREF || parg->count) {
423 /*
424 * IMM and COMM is pointing actual address, those must
425 * be kept, and if parg->count != 0, this is an array
426 * of string pointers instead of string address itself.
427 */
428 code++;
429 if (code->op != FETCH_OP_NOP) {
430 ret = -E2BIG;
431 goto fail;
432 }
433 }
434 code->op = FETCH_OP_ST_STRING; /* In DEREF case, replace it */
435 code->size = parg->type->size;
436 parg->dynamic = true;
437 } else if (code->op == FETCH_OP_DEREF) {
438 code->op = FETCH_OP_ST_MEM;
439 code->size = parg->type->size;
440 } else {
441 code++;
442 if (code->op != FETCH_OP_NOP) {
443 ret = -E2BIG;
444 goto fail;
445 }
446 code->op = FETCH_OP_ST_RAW;
447 code->size = parg->type->size;
448 }
449 scode = code;
450 /* Modify operation */
451 if (t != NULL) {
452 ret = __parse_bitfield_probe_arg(t, parg->type, &code);
453 if (ret)
454 goto fail;
567 } 455 }
456 /* Loop(Array) operation */
457 if (parg->count) {
458 if (scode->op != FETCH_OP_ST_MEM &&
459 scode->op != FETCH_OP_ST_STRING) {
460 pr_info("array only accepts memory or address\n");
461 ret = -EINVAL;
462 goto fail;
463 }
464 code++;
465 if (code->op != FETCH_OP_NOP) {
466 ret = -E2BIG;
467 goto fail;
468 }
469 code->op = FETCH_OP_LP_ARRAY;
470 code->param = parg->count;
471 }
472 code++;
473 code->op = FETCH_OP_END;
474
475 /* Shrink down the code buffer */
476 parg->code = kzalloc(sizeof(*code) * (code - tmp + 1), GFP_KERNEL);
477 if (!parg->code)
478 ret = -ENOMEM;
479 else
480 memcpy(parg->code, tmp, sizeof(*code) * (code - tmp + 1));
481
482fail:
483 if (ret) {
484 for (code = tmp; code < tmp + FETCH_INSN_MAX; code++)
485 if (code->op == FETCH_NOP_SYMBOL)
486 kfree(code->data);
487 }
488 kfree(tmp);
568 489
569 return ret; 490 return ret;
570} 491}
@@ -586,35 +507,63 @@ int traceprobe_conflict_field_name(const char *name,
586 return 0; 507 return 0;
587} 508}
588 509
589void traceprobe_update_arg(struct probe_arg *arg)
590{
591 if (CHECK_FETCH_FUNCS(bitfield, arg->fetch.fn))
592 update_bitfield_fetch_param(arg->fetch.data);
593 else if (CHECK_FETCH_FUNCS(deref, arg->fetch.fn))
594 update_deref_fetch_param(arg->fetch.data);
595 else if (CHECK_FETCH_FUNCS(symbol, arg->fetch.fn))
596 update_symbol_cache(arg->fetch.data);
597}
598
599void traceprobe_free_probe_arg(struct probe_arg *arg) 510void traceprobe_free_probe_arg(struct probe_arg *arg)
600{ 511{
601 if (CHECK_FETCH_FUNCS(bitfield, arg->fetch.fn)) 512 struct fetch_insn *code = arg->code;
602 free_bitfield_fetch_param(arg->fetch.data);
603 else if (CHECK_FETCH_FUNCS(deref, arg->fetch.fn))
604 free_deref_fetch_param(arg->fetch.data);
605 else if (CHECK_FETCH_FUNCS(symbol, arg->fetch.fn))
606 free_symbol_cache(arg->fetch.data);
607 513
514 while (code && code->op != FETCH_OP_END) {
515 if (code->op == FETCH_NOP_SYMBOL)
516 kfree(code->data);
517 code++;
518 }
519 kfree(arg->code);
608 kfree(arg->name); 520 kfree(arg->name);
609 kfree(arg->comm); 521 kfree(arg->comm);
522 kfree(arg->fmt);
610} 523}
611 524
525int traceprobe_update_arg(struct probe_arg *arg)
526{
527 struct fetch_insn *code = arg->code;
528 long offset;
529 char *tmp;
530 char c;
531 int ret = 0;
532
533 while (code && code->op != FETCH_OP_END) {
534 if (code->op == FETCH_NOP_SYMBOL) {
535 if (code[1].op != FETCH_OP_IMM)
536 return -EINVAL;
537
538 tmp = strpbrk("+-", code->data);
539 if (tmp)
540 c = *tmp;
541 ret = traceprobe_split_symbol_offset(code->data,
542 &offset);
543 if (ret)
544 return ret;
545
546 code[1].immediate =
547 (unsigned long)kallsyms_lookup_name(code->data);
548 if (tmp)
549 *tmp = c;
550 if (!code[1].immediate)
551 return -ENOENT;
552 code[1].immediate += offset;
553 }
554 code++;
555 }
556 return 0;
557}
558
559/* When len=0, we just calculate the needed length */
560#define LEN_OR_ZERO (len ? len - pos : 0)
612static int __set_print_fmt(struct trace_probe *tp, char *buf, int len, 561static int __set_print_fmt(struct trace_probe *tp, char *buf, int len,
613 bool is_return) 562 bool is_return)
614{ 563{
615 int i; 564 struct probe_arg *parg;
565 int i, j;
616 int pos = 0; 566 int pos = 0;
617
618 const char *fmt, *arg; 567 const char *fmt, *arg;
619 568
620 if (!is_return) { 569 if (!is_return) {
@@ -625,35 +574,51 @@ static int __set_print_fmt(struct trace_probe *tp, char *buf, int len,
625 arg = "REC->" FIELD_STRING_FUNC ", REC->" FIELD_STRING_RETIP; 574 arg = "REC->" FIELD_STRING_FUNC ", REC->" FIELD_STRING_RETIP;
626 } 575 }
627 576
628 /* When len=0, we just calculate the needed length */
629#define LEN_OR_ZERO (len ? len - pos : 0)
630
631 pos += snprintf(buf + pos, LEN_OR_ZERO, "\"%s", fmt); 577 pos += snprintf(buf + pos, LEN_OR_ZERO, "\"%s", fmt);
632 578
633 for (i = 0; i < tp->nr_args; i++) { 579 for (i = 0; i < tp->nr_args; i++) {
634 pos += snprintf(buf + pos, LEN_OR_ZERO, " %s=%s", 580 parg = tp->args + i;
635 tp->args[i].name, tp->args[i].type->fmt); 581 pos += snprintf(buf + pos, LEN_OR_ZERO, " %s=", parg->name);
582 if (parg->count) {
583 pos += snprintf(buf + pos, LEN_OR_ZERO, "{%s",
584 parg->type->fmt);
585 for (j = 1; j < parg->count; j++)
586 pos += snprintf(buf + pos, LEN_OR_ZERO, ",%s",
587 parg->type->fmt);
588 pos += snprintf(buf + pos, LEN_OR_ZERO, "}");
589 } else
590 pos += snprintf(buf + pos, LEN_OR_ZERO, "%s",
591 parg->type->fmt);
636 } 592 }
637 593
638 pos += snprintf(buf + pos, LEN_OR_ZERO, "\", %s", arg); 594 pos += snprintf(buf + pos, LEN_OR_ZERO, "\", %s", arg);
639 595
640 for (i = 0; i < tp->nr_args; i++) { 596 for (i = 0; i < tp->nr_args; i++) {
641 if (strcmp(tp->args[i].type->name, "string") == 0) 597 parg = tp->args + i;
598 if (parg->count) {
599 if (strcmp(parg->type->name, "string") == 0)
600 fmt = ", __get_str(%s[%d])";
601 else
602 fmt = ", REC->%s[%d]";
603 for (j = 0; j < parg->count; j++)
604 pos += snprintf(buf + pos, LEN_OR_ZERO,
605 fmt, parg->name, j);
606 } else {
607 if (strcmp(parg->type->name, "string") == 0)
608 fmt = ", __get_str(%s)";
609 else
610 fmt = ", REC->%s";
642 pos += snprintf(buf + pos, LEN_OR_ZERO, 611 pos += snprintf(buf + pos, LEN_OR_ZERO,
643 ", __get_str(%s)", 612 fmt, parg->name);
644 tp->args[i].name); 613 }
645 else
646 pos += snprintf(buf + pos, LEN_OR_ZERO, ", REC->%s",
647 tp->args[i].name);
648 } 614 }
649 615
650#undef LEN_OR_ZERO
651
652 /* return the length of print_fmt */ 616 /* return the length of print_fmt */
653 return pos; 617 return pos;
654} 618}
619#undef LEN_OR_ZERO
655 620
656int set_print_fmt(struct trace_probe *tp, bool is_return) 621int traceprobe_set_print_fmt(struct trace_probe *tp, bool is_return)
657{ 622{
658 int len; 623 int len;
659 char *print_fmt; 624 char *print_fmt;
@@ -670,3 +635,28 @@ int set_print_fmt(struct trace_probe *tp, bool is_return)
670 635
671 return 0; 636 return 0;
672} 637}
638
639int traceprobe_define_arg_fields(struct trace_event_call *event_call,
640 size_t offset, struct trace_probe *tp)
641{
642 int ret, i;
643
644 /* Set argument names as fields */
645 for (i = 0; i < tp->nr_args; i++) {
646 struct probe_arg *parg = &tp->args[i];
647 const char *fmt = parg->type->fmttype;
648 int size = parg->type->size;
649
650 if (parg->fmt)
651 fmt = parg->fmt;
652 if (parg->count)
653 size *= parg->count;
654 ret = trace_define_field(event_call, fmt, parg->name,
655 offset + parg->offset, size,
656 parg->type->is_signed,
657 FILTER_OTHER);
658 if (ret)
659 return ret;
660 }
661 return 0;
662}
diff --git a/kernel/trace/trace_probe.h b/kernel/trace/trace_probe.h
index 5f52668e165d..974afc1a3e73 100644
--- a/kernel/trace/trace_probe.h
+++ b/kernel/trace/trace_probe.h
@@ -23,6 +23,7 @@
23#include <linux/stringify.h> 23#include <linux/stringify.h>
24#include <linux/limits.h> 24#include <linux/limits.h>
25#include <linux/uaccess.h> 25#include <linux/uaccess.h>
26#include <linux/bitops.h>
26#include <asm/bitsperlong.h> 27#include <asm/bitsperlong.h>
27 28
28#include "trace.h" 29#include "trace.h"
@@ -30,6 +31,7 @@
30 31
31#define MAX_TRACE_ARGS 128 32#define MAX_TRACE_ARGS 128
32#define MAX_ARGSTR_LEN 63 33#define MAX_ARGSTR_LEN 63
34#define MAX_ARRAY_LEN 64
33#define MAX_STRING_SIZE PATH_MAX 35#define MAX_STRING_SIZE PATH_MAX
34 36
35/* Reserved field names */ 37/* Reserved field names */
@@ -54,50 +56,74 @@
54#define TP_FLAG_PROFILE 2 56#define TP_FLAG_PROFILE 2
55#define TP_FLAG_REGISTERED 4 57#define TP_FLAG_REGISTERED 4
56 58
59/* data_loc: data location, compatible with u32 */
60#define make_data_loc(len, offs) \
61 (((u32)(len) << 16) | ((u32)(offs) & 0xffff))
62#define get_loc_len(dl) ((u32)(dl) >> 16)
63#define get_loc_offs(dl) ((u32)(dl) & 0xffff)
57 64
58/* data_rloc: data relative location, compatible with u32 */ 65static nokprobe_inline void *get_loc_data(u32 *dl, void *ent)
59#define make_data_rloc(len, roffs) \
60 (((u32)(len) << 16) | ((u32)(roffs) & 0xffff))
61#define get_rloc_len(dl) ((u32)(dl) >> 16)
62#define get_rloc_offs(dl) ((u32)(dl) & 0xffff)
63
64/*
65 * Convert data_rloc to data_loc:
66 * data_rloc stores the offset from data_rloc itself, but data_loc
67 * stores the offset from event entry.
68 */
69#define convert_rloc_to_loc(dl, offs) ((u32)(dl) + (offs))
70
71static nokprobe_inline void *get_rloc_data(u32 *dl)
72{ 66{
73 return (u8 *)dl + get_rloc_offs(*dl); 67 return (u8 *)ent + get_loc_offs(*dl);
74} 68}
75 69
76/* For data_loc conversion */ 70static nokprobe_inline u32 update_data_loc(u32 loc, int consumed)
77static nokprobe_inline void *get_loc_data(u32 *dl, void *ent)
78{ 71{
79 return (u8 *)ent + get_rloc_offs(*dl); 72 u32 maxlen = get_loc_len(loc);
73 u32 offset = get_loc_offs(loc);
74
75 return make_data_loc(maxlen - consumed, offset + consumed);
80} 76}
81 77
82/* Data fetch function type */
83typedef void (*fetch_func_t)(struct pt_regs *, void *, void *);
84/* Printing function type */ 78/* Printing function type */
85typedef int (*print_type_func_t)(struct trace_seq *, const char *, void *, void *); 79typedef int (*print_type_func_t)(struct trace_seq *, void *, void *);
86 80
87/* Fetch types */ 81enum fetch_op {
88enum { 82 FETCH_OP_NOP = 0,
89 FETCH_MTD_reg = 0, 83 // Stage 1 (load) ops
90 FETCH_MTD_stack, 84 FETCH_OP_REG, /* Register : .param = offset */
91 FETCH_MTD_retval, 85 FETCH_OP_STACK, /* Stack : .param = index */
92 FETCH_MTD_comm, 86 FETCH_OP_STACKP, /* Stack pointer */
93 FETCH_MTD_memory, 87 FETCH_OP_RETVAL, /* Return value */
94 FETCH_MTD_symbol, 88 FETCH_OP_IMM, /* Immediate : .immediate */
95 FETCH_MTD_deref, 89 FETCH_OP_COMM, /* Current comm */
96 FETCH_MTD_bitfield, 90 FETCH_OP_ARG, /* Function argument : .param */
97 FETCH_MTD_file_offset, 91 FETCH_OP_FOFFS, /* File offset: .immediate */
98 FETCH_MTD_END, 92 // Stage 2 (dereference) op
93 FETCH_OP_DEREF, /* Dereference: .offset */
94 // Stage 3 (store) ops
95 FETCH_OP_ST_RAW, /* Raw: .size */
96 FETCH_OP_ST_MEM, /* Mem: .offset, .size */
97 FETCH_OP_ST_STRING, /* String: .offset, .size */
98 // Stage 4 (modify) op
99 FETCH_OP_MOD_BF, /* Bitfield: .basesize, .lshift, .rshift */
100 // Stage 5 (loop) op
101 FETCH_OP_LP_ARRAY, /* Array: .param = loop count */
102 FETCH_OP_END,
103 FETCH_NOP_SYMBOL, /* Unresolved Symbol holder */
99}; 104};
100 105
106struct fetch_insn {
107 enum fetch_op op;
108 union {
109 unsigned int param;
110 struct {
111 unsigned int size;
112 int offset;
113 };
114 struct {
115 unsigned char basesize;
116 unsigned char lshift;
117 unsigned char rshift;
118 };
119 unsigned long immediate;
120 void *data;
121 };
122};
123
124/* fetch + deref*N + store + mod + end <= 16, this allows N=12, enough */
125#define FETCH_INSN_MAX 16
126
101/* Fetch type information table */ 127/* Fetch type information table */
102struct fetch_type { 128struct fetch_type {
103 const char *name; /* Name of type */ 129 const char *name; /* Name of type */
@@ -106,13 +132,6 @@ struct fetch_type {
106 print_type_func_t print; /* Print functions */ 132 print_type_func_t print; /* Print functions */
107 const char *fmt; /* Fromat string */ 133 const char *fmt; /* Fromat string */
108 const char *fmttype; /* Name in format file */ 134 const char *fmttype; /* Name in format file */
109 /* Fetch functions */
110 fetch_func_t fetch[FETCH_MTD_END];
111};
112
113struct fetch_param {
114 fetch_func_t fn;
115 void *data;
116}; 135};
117 136
118/* For defining macros, define string/string_size types */ 137/* For defining macros, define string/string_size types */
@@ -124,8 +143,7 @@ typedef u32 string_size;
124 143
125/* Printing in basic type function template */ 144/* Printing in basic type function template */
126#define DECLARE_BASIC_PRINT_TYPE_FUNC(type) \ 145#define DECLARE_BASIC_PRINT_TYPE_FUNC(type) \
127int PRINT_TYPE_FUNC_NAME(type)(struct trace_seq *s, const char *name, \ 146int PRINT_TYPE_FUNC_NAME(type)(struct trace_seq *s, void *data, void *ent);\
128 void *data, void *ent); \
129extern const char PRINT_TYPE_FMT_NAME(type)[] 147extern const char PRINT_TYPE_FMT_NAME(type)[]
130 148
131DECLARE_BASIC_PRINT_TYPE_FUNC(u8); 149DECLARE_BASIC_PRINT_TYPE_FUNC(u8);
@@ -142,57 +160,7 @@ DECLARE_BASIC_PRINT_TYPE_FUNC(x32);
142DECLARE_BASIC_PRINT_TYPE_FUNC(x64); 160DECLARE_BASIC_PRINT_TYPE_FUNC(x64);
143 161
144DECLARE_BASIC_PRINT_TYPE_FUNC(string); 162DECLARE_BASIC_PRINT_TYPE_FUNC(string);
145 163DECLARE_BASIC_PRINT_TYPE_FUNC(symbol);
146#define FETCH_FUNC_NAME(method, type) fetch_##method##_##type
147
148/* Declare macro for basic types */
149#define DECLARE_FETCH_FUNC(method, type) \
150extern void FETCH_FUNC_NAME(method, type)(struct pt_regs *regs, \
151 void *data, void *dest)
152
153#define DECLARE_BASIC_FETCH_FUNCS(method) \
154DECLARE_FETCH_FUNC(method, u8); \
155DECLARE_FETCH_FUNC(method, u16); \
156DECLARE_FETCH_FUNC(method, u32); \
157DECLARE_FETCH_FUNC(method, u64)
158
159DECLARE_BASIC_FETCH_FUNCS(reg);
160#define fetch_reg_string NULL
161#define fetch_reg_string_size NULL
162
163DECLARE_BASIC_FETCH_FUNCS(retval);
164#define fetch_retval_string NULL
165#define fetch_retval_string_size NULL
166
167DECLARE_BASIC_FETCH_FUNCS(symbol);
168DECLARE_FETCH_FUNC(symbol, string);
169DECLARE_FETCH_FUNC(symbol, string_size);
170
171DECLARE_BASIC_FETCH_FUNCS(deref);
172DECLARE_FETCH_FUNC(deref, string);
173DECLARE_FETCH_FUNC(deref, string_size);
174
175DECLARE_BASIC_FETCH_FUNCS(bitfield);
176#define fetch_bitfield_string NULL
177#define fetch_bitfield_string_size NULL
178
179/* comm only makes sense as a string */
180#define fetch_comm_u8 NULL
181#define fetch_comm_u16 NULL
182#define fetch_comm_u32 NULL
183#define fetch_comm_u64 NULL
184DECLARE_FETCH_FUNC(comm, string);
185DECLARE_FETCH_FUNC(comm, string_size);
186
187/*
188 * Define macro for basic types - we don't need to define s* types, because
189 * we have to care only about bitwidth at recording time.
190 */
191#define DEFINE_BASIC_FETCH_FUNCS(method) \
192DEFINE_FETCH_##method(u8) \
193DEFINE_FETCH_##method(u16) \
194DEFINE_FETCH_##method(u32) \
195DEFINE_FETCH_##method(u64)
196 164
197/* Default (unsigned long) fetch type */ 165/* Default (unsigned long) fetch type */
198#define __DEFAULT_FETCH_TYPE(t) x##t 166#define __DEFAULT_FETCH_TYPE(t) x##t
@@ -200,8 +168,9 @@ DEFINE_FETCH_##method(u64)
200#define DEFAULT_FETCH_TYPE _DEFAULT_FETCH_TYPE(BITS_PER_LONG) 168#define DEFAULT_FETCH_TYPE _DEFAULT_FETCH_TYPE(BITS_PER_LONG)
201#define DEFAULT_FETCH_TYPE_STR __stringify(DEFAULT_FETCH_TYPE) 169#define DEFAULT_FETCH_TYPE_STR __stringify(DEFAULT_FETCH_TYPE)
202 170
203#define ASSIGN_FETCH_FUNC(method, type) \ 171#define __ADDR_FETCH_TYPE(t) u##t
204 [FETCH_MTD_##method] = FETCH_FUNC_NAME(method, type) 172#define _ADDR_FETCH_TYPE(t) __ADDR_FETCH_TYPE(t)
173#define ADDR_FETCH_TYPE _ADDR_FETCH_TYPE(BITS_PER_LONG)
205 174
206#define __ASSIGN_FETCH_TYPE(_name, ptype, ftype, _size, sign, _fmttype) \ 175#define __ASSIGN_FETCH_TYPE(_name, ptype, ftype, _size, sign, _fmttype) \
207 {.name = _name, \ 176 {.name = _name, \
@@ -210,64 +179,23 @@ DEFINE_FETCH_##method(u64)
210 .print = PRINT_TYPE_FUNC_NAME(ptype), \ 179 .print = PRINT_TYPE_FUNC_NAME(ptype), \
211 .fmt = PRINT_TYPE_FMT_NAME(ptype), \ 180 .fmt = PRINT_TYPE_FMT_NAME(ptype), \
212 .fmttype = _fmttype, \ 181 .fmttype = _fmttype, \
213 .fetch = { \
214ASSIGN_FETCH_FUNC(reg, ftype), \
215ASSIGN_FETCH_FUNC(stack, ftype), \
216ASSIGN_FETCH_FUNC(retval, ftype), \
217ASSIGN_FETCH_FUNC(comm, ftype), \
218ASSIGN_FETCH_FUNC(memory, ftype), \
219ASSIGN_FETCH_FUNC(symbol, ftype), \
220ASSIGN_FETCH_FUNC(deref, ftype), \
221ASSIGN_FETCH_FUNC(bitfield, ftype), \
222ASSIGN_FETCH_FUNC(file_offset, ftype), \
223 } \
224 } 182 }
225 183#define _ASSIGN_FETCH_TYPE(_name, ptype, ftype, _size, sign, _fmttype) \
184 __ASSIGN_FETCH_TYPE(_name, ptype, ftype, _size, sign, #_fmttype)
226#define ASSIGN_FETCH_TYPE(ptype, ftype, sign) \ 185#define ASSIGN_FETCH_TYPE(ptype, ftype, sign) \
227 __ASSIGN_FETCH_TYPE(#ptype, ptype, ftype, sizeof(ftype), sign, #ptype) 186 _ASSIGN_FETCH_TYPE(#ptype, ptype, ftype, sizeof(ftype), sign, ptype)
228 187
229/* If ptype is an alias of atype, use this macro (show atype in format) */ 188/* If ptype is an alias of atype, use this macro (show atype in format) */
230#define ASSIGN_FETCH_TYPE_ALIAS(ptype, atype, ftype, sign) \ 189#define ASSIGN_FETCH_TYPE_ALIAS(ptype, atype, ftype, sign) \
231 __ASSIGN_FETCH_TYPE(#ptype, ptype, ftype, sizeof(ftype), sign, #atype) 190 _ASSIGN_FETCH_TYPE(#ptype, ptype, ftype, sizeof(ftype), sign, atype)
232 191
233#define ASSIGN_FETCH_TYPE_END {} 192#define ASSIGN_FETCH_TYPE_END {}
234 193#define MAX_ARRAY_LEN 64
235#define FETCH_TYPE_STRING 0
236#define FETCH_TYPE_STRSIZE 1
237 194
238#ifdef CONFIG_KPROBE_EVENTS 195#ifdef CONFIG_KPROBE_EVENTS
239struct symbol_cache;
240unsigned long update_symbol_cache(struct symbol_cache *sc);
241void free_symbol_cache(struct symbol_cache *sc);
242struct symbol_cache *alloc_symbol_cache(const char *sym, long offset);
243bool trace_kprobe_on_func_entry(struct trace_event_call *call); 196bool trace_kprobe_on_func_entry(struct trace_event_call *call);
244bool trace_kprobe_error_injectable(struct trace_event_call *call); 197bool trace_kprobe_error_injectable(struct trace_event_call *call);
245#else 198#else
246/* uprobes do not support symbol fetch methods */
247#define fetch_symbol_u8 NULL
248#define fetch_symbol_u16 NULL
249#define fetch_symbol_u32 NULL
250#define fetch_symbol_u64 NULL
251#define fetch_symbol_string NULL
252#define fetch_symbol_string_size NULL
253
254struct symbol_cache {
255};
256static inline unsigned long __used update_symbol_cache(struct symbol_cache *sc)
257{
258 return 0;
259}
260
261static inline void __used free_symbol_cache(struct symbol_cache *sc)
262{
263}
264
265static inline struct symbol_cache * __used
266alloc_symbol_cache(const char *sym, long offset)
267{
268 return NULL;
269}
270
271static inline bool trace_kprobe_on_func_entry(struct trace_event_call *call) 199static inline bool trace_kprobe_on_func_entry(struct trace_event_call *call)
272{ 200{
273 return false; 201 return false;
@@ -280,11 +208,13 @@ static inline bool trace_kprobe_error_injectable(struct trace_event_call *call)
280#endif /* CONFIG_KPROBE_EVENTS */ 208#endif /* CONFIG_KPROBE_EVENTS */
281 209
282struct probe_arg { 210struct probe_arg {
283 struct fetch_param fetch; 211 struct fetch_insn *code;
284 struct fetch_param fetch_size; 212 bool dynamic;/* Dynamic array (string) is used */
285 unsigned int offset; /* Offset from argument entry */ 213 unsigned int offset; /* Offset from argument entry */
214 unsigned int count; /* Array count */
286 const char *name; /* Name of this argument */ 215 const char *name; /* Name of this argument */
287 const char *comm; /* Command of this argument */ 216 const char *comm; /* Command of this argument */
217 char *fmt; /* Format string if needed */
288 const struct fetch_type *type; /* Type of this argument */ 218 const struct fetch_type *type; /* Type of this argument */
289}; 219};
290 220
@@ -313,12 +243,6 @@ static inline bool trace_probe_is_registered(struct trace_probe *tp)
313 return !!(tp->flags & TP_FLAG_REGISTERED); 243 return !!(tp->flags & TP_FLAG_REGISTERED);
314} 244}
315 245
316static nokprobe_inline void call_fetch(struct fetch_param *fprm,
317 struct pt_regs *regs, void *dest)
318{
319 return fprm->fn(regs, fprm->data, dest);
320}
321
322/* Check the name is good for event/group/fields */ 246/* Check the name is good for event/group/fields */
323static inline bool is_good_name(const char *name) 247static inline bool is_good_name(const char *name)
324{ 248{
@@ -343,67 +267,23 @@ find_event_file_link(struct trace_probe *tp, struct trace_event_file *file)
343 return NULL; 267 return NULL;
344} 268}
345 269
270#define TPARG_FL_RETURN BIT(0)
271#define TPARG_FL_KERNEL BIT(1)
272#define TPARG_FL_FENTRY BIT(2)
273#define TPARG_FL_MASK GENMASK(2, 0)
274
346extern int traceprobe_parse_probe_arg(char *arg, ssize_t *size, 275extern int traceprobe_parse_probe_arg(char *arg, ssize_t *size,
347 struct probe_arg *parg, bool is_return, bool is_kprobe, 276 struct probe_arg *parg, unsigned int flags);
348 const struct fetch_type *ftbl);
349 277
350extern int traceprobe_conflict_field_name(const char *name, 278extern int traceprobe_conflict_field_name(const char *name,
351 struct probe_arg *args, int narg); 279 struct probe_arg *args, int narg);
352 280
353extern void traceprobe_update_arg(struct probe_arg *arg); 281extern int traceprobe_update_arg(struct probe_arg *arg);
354extern void traceprobe_free_probe_arg(struct probe_arg *arg); 282extern void traceprobe_free_probe_arg(struct probe_arg *arg);
355 283
356extern int traceprobe_split_symbol_offset(char *symbol, long *offset); 284extern int traceprobe_split_symbol_offset(char *symbol, long *offset);
357 285
358/* Sum up total data length for dynamic arraies (strings) */ 286extern int traceprobe_set_print_fmt(struct trace_probe *tp, bool is_return);
359static nokprobe_inline int
360__get_data_size(struct trace_probe *tp, struct pt_regs *regs)
361{
362 int i, ret = 0;
363 u32 len;
364
365 for (i = 0; i < tp->nr_args; i++)
366 if (unlikely(tp->args[i].fetch_size.fn)) {
367 call_fetch(&tp->args[i].fetch_size, regs, &len);
368 ret += len;
369 }
370
371 return ret;
372}
373
374/* Store the value of each argument */
375static nokprobe_inline void
376store_trace_args(int ent_size, struct trace_probe *tp, struct pt_regs *regs,
377 u8 *data, int maxlen)
378{
379 int i;
380 u32 end = tp->size;
381 u32 *dl; /* Data (relative) location */
382
383 for (i = 0; i < tp->nr_args; i++) {
384 if (unlikely(tp->args[i].fetch_size.fn)) {
385 /*
386 * First, we set the relative location and
387 * maximum data length to *dl
388 */
389 dl = (u32 *)(data + tp->args[i].offset);
390 *dl = make_data_rloc(maxlen, end - tp->args[i].offset);
391 /* Then try to fetch string or dynamic array data */
392 call_fetch(&tp->args[i].fetch, regs, dl);
393 /* Reduce maximum length */
394 end += get_rloc_len(*dl);
395 maxlen -= get_rloc_len(*dl);
396 /* Trick here, convert data_rloc to data_loc */
397 *dl = convert_rloc_to_loc(*dl,
398 ent_size + tp->args[i].offset);
399 } else
400 /* Just fetching data normally */
401 call_fetch(&tp->args[i].fetch, regs,
402 data + tp->args[i].offset);
403 }
404}
405
406extern int set_print_fmt(struct trace_probe *tp, bool is_return);
407 287
408#ifdef CONFIG_PERF_EVENTS 288#ifdef CONFIG_PERF_EVENTS
409extern struct trace_event_call * 289extern struct trace_event_call *
@@ -412,6 +292,9 @@ create_local_trace_kprobe(char *func, void *addr, unsigned long offs,
412extern void destroy_local_trace_kprobe(struct trace_event_call *event_call); 292extern void destroy_local_trace_kprobe(struct trace_event_call *event_call);
413 293
414extern struct trace_event_call * 294extern struct trace_event_call *
415create_local_trace_uprobe(char *name, unsigned long offs, bool is_return); 295create_local_trace_uprobe(char *name, unsigned long offs,
296 unsigned long ref_ctr_offset, bool is_return);
416extern void destroy_local_trace_uprobe(struct trace_event_call *event_call); 297extern void destroy_local_trace_uprobe(struct trace_event_call *event_call);
417#endif 298#endif
299extern int traceprobe_define_arg_fields(struct trace_event_call *event_call,
300 size_t offset, struct trace_probe *tp);
diff --git a/kernel/trace/trace_probe_tmpl.h b/kernel/trace/trace_probe_tmpl.h
new file mode 100644
index 000000000000..5c56afc17cf8
--- /dev/null
+++ b/kernel/trace/trace_probe_tmpl.h
@@ -0,0 +1,216 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Traceprobe fetch helper inlines
4 */
5
6static nokprobe_inline void
7fetch_store_raw(unsigned long val, struct fetch_insn *code, void *buf)
8{
9 switch (code->size) {
10 case 1:
11 *(u8 *)buf = (u8)val;
12 break;
13 case 2:
14 *(u16 *)buf = (u16)val;
15 break;
16 case 4:
17 *(u32 *)buf = (u32)val;
18 break;
19 case 8:
20 //TBD: 32bit signed
21 *(u64 *)buf = (u64)val;
22 break;
23 default:
24 *(unsigned long *)buf = val;
25 }
26}
27
28static nokprobe_inline void
29fetch_apply_bitfield(struct fetch_insn *code, void *buf)
30{
31 switch (code->basesize) {
32 case 1:
33 *(u8 *)buf <<= code->lshift;
34 *(u8 *)buf >>= code->rshift;
35 break;
36 case 2:
37 *(u16 *)buf <<= code->lshift;
38 *(u16 *)buf >>= code->rshift;
39 break;
40 case 4:
41 *(u32 *)buf <<= code->lshift;
42 *(u32 *)buf >>= code->rshift;
43 break;
44 case 8:
45 *(u64 *)buf <<= code->lshift;
46 *(u64 *)buf >>= code->rshift;
47 break;
48 }
49}
50
51/*
52 * These functions must be defined for each callsite.
53 * Return consumed dynamic data size (>= 0), or error (< 0).
54 * If dest is NULL, don't store result and return required dynamic data size.
55 */
56static int
57process_fetch_insn(struct fetch_insn *code, struct pt_regs *regs,
58 void *dest, void *base);
59static nokprobe_inline int fetch_store_strlen(unsigned long addr);
60static nokprobe_inline int
61fetch_store_string(unsigned long addr, void *dest, void *base);
62static nokprobe_inline int
63probe_mem_read(void *dest, void *src, size_t size);
64
65/* From the 2nd stage, routine is same */
66static nokprobe_inline int
67process_fetch_insn_bottom(struct fetch_insn *code, unsigned long val,
68 void *dest, void *base)
69{
70 struct fetch_insn *s3 = NULL;
71 int total = 0, ret = 0, i = 0;
72 u32 loc = 0;
73 unsigned long lval = val;
74
75stage2:
76 /* 2nd stage: dereference memory if needed */
77 while (code->op == FETCH_OP_DEREF) {
78 lval = val;
79 ret = probe_mem_read(&val, (void *)val + code->offset,
80 sizeof(val));
81 if (ret)
82 return ret;
83 code++;
84 }
85
86 s3 = code;
87stage3:
88 /* 3rd stage: store value to buffer */
89 if (unlikely(!dest)) {
90 if (code->op == FETCH_OP_ST_STRING) {
91 ret += fetch_store_strlen(val + code->offset);
92 code++;
93 goto array;
94 } else
95 return -EILSEQ;
96 }
97
98 switch (code->op) {
99 case FETCH_OP_ST_RAW:
100 fetch_store_raw(val, code, dest);
101 break;
102 case FETCH_OP_ST_MEM:
103 probe_mem_read(dest, (void *)val + code->offset, code->size);
104 break;
105 case FETCH_OP_ST_STRING:
106 loc = *(u32 *)dest;
107 ret = fetch_store_string(val + code->offset, dest, base);
108 break;
109 default:
110 return -EILSEQ;
111 }
112 code++;
113
114 /* 4th stage: modify stored value if needed */
115 if (code->op == FETCH_OP_MOD_BF) {
116 fetch_apply_bitfield(code, dest);
117 code++;
118 }
119
120array:
121 /* the last stage: Loop on array */
122 if (code->op == FETCH_OP_LP_ARRAY) {
123 total += ret;
124 if (++i < code->param) {
125 code = s3;
126 if (s3->op != FETCH_OP_ST_STRING) {
127 dest += s3->size;
128 val += s3->size;
129 goto stage3;
130 }
131 code--;
132 val = lval + sizeof(char *);
133 if (dest) {
134 dest += sizeof(u32);
135 *(u32 *)dest = update_data_loc(loc, ret);
136 }
137 goto stage2;
138 }
139 code++;
140 ret = total;
141 }
142
143 return code->op == FETCH_OP_END ? ret : -EILSEQ;
144}
145
146/* Sum up total data length for dynamic arraies (strings) */
147static nokprobe_inline int
148__get_data_size(struct trace_probe *tp, struct pt_regs *regs)
149{
150 struct probe_arg *arg;
151 int i, len, ret = 0;
152
153 for (i = 0; i < tp->nr_args; i++) {
154 arg = tp->args + i;
155 if (unlikely(arg->dynamic)) {
156 len = process_fetch_insn(arg->code, regs, NULL, NULL);
157 if (len > 0)
158 ret += len;
159 }
160 }
161
162 return ret;
163}
164
165/* Store the value of each argument */
166static nokprobe_inline void
167store_trace_args(void *data, struct trace_probe *tp, struct pt_regs *regs,
168 int header_size, int maxlen)
169{
170 struct probe_arg *arg;
171 void *base = data - header_size;
172 void *dyndata = data + tp->size;
173 u32 *dl; /* Data location */
174 int ret, i;
175
176 for (i = 0; i < tp->nr_args; i++) {
177 arg = tp->args + i;
178 dl = data + arg->offset;
179 /* Point the dynamic data area if needed */
180 if (unlikely(arg->dynamic))
181 *dl = make_data_loc(maxlen, dyndata - base);
182 ret = process_fetch_insn(arg->code, regs, dl, base);
183 if (unlikely(ret < 0 && arg->dynamic))
184 *dl = make_data_loc(0, dyndata - base);
185 else
186 dyndata += ret;
187 }
188}
189
190static inline int
191print_probe_args(struct trace_seq *s, struct probe_arg *args, int nr_args,
192 u8 *data, void *field)
193{
194 void *p;
195 int i, j;
196
197 for (i = 0; i < nr_args; i++) {
198 struct probe_arg *a = args + i;
199
200 trace_seq_printf(s, " %s=", a->name);
201 if (likely(!a->count)) {
202 if (!a->type->print(s, data + a->offset, field))
203 return -ENOMEM;
204 continue;
205 }
206 trace_seq_putc(s, '{');
207 p = data + a->offset;
208 for (j = 0; j < a->count; j++) {
209 if (!a->type->print(s, p, field))
210 return -ENOMEM;
211 trace_seq_putc(s, j == a->count - 1 ? '}' : ',');
212 p += a->type->size;
213 }
214 }
215 return 0;
216}
diff --git a/kernel/trace/trace_stack.c b/kernel/trace/trace_stack.c
index 4237eba4ef20..2b0d1ee3241c 100644
--- a/kernel/trace/trace_stack.c
+++ b/kernel/trace/trace_stack.c
@@ -111,7 +111,7 @@ check_stack(unsigned long ip, unsigned long *stack)
111 stack_trace_max_size = this_size; 111 stack_trace_max_size = this_size;
112 112
113 stack_trace_max.nr_entries = 0; 113 stack_trace_max.nr_entries = 0;
114 stack_trace_max.skip = 3; 114 stack_trace_max.skip = 0;
115 115
116 save_stack_trace(&stack_trace_max); 116 save_stack_trace(&stack_trace_max);
117 117
diff --git a/kernel/trace/trace_uprobe.c b/kernel/trace/trace_uprobe.c
index e696667da29a..31ea48eceda1 100644
--- a/kernel/trace/trace_uprobe.c
+++ b/kernel/trace/trace_uprobe.c
@@ -15,6 +15,7 @@
15#include <linux/rculist.h> 15#include <linux/rculist.h>
16 16
17#include "trace_probe.h" 17#include "trace_probe.h"
18#include "trace_probe_tmpl.h"
18 19
19#define UPROBE_EVENT_SYSTEM "uprobes" 20#define UPROBE_EVENT_SYSTEM "uprobes"
20 21
@@ -47,6 +48,7 @@ struct trace_uprobe {
47 struct inode *inode; 48 struct inode *inode;
48 char *filename; 49 char *filename;
49 unsigned long offset; 50 unsigned long offset;
51 unsigned long ref_ctr_offset;
50 unsigned long nhit; 52 unsigned long nhit;
51 struct trace_probe tp; 53 struct trace_probe tp;
52}; 54};
@@ -98,74 +100,52 @@ static unsigned long get_user_stack_nth(struct pt_regs *regs, unsigned int n)
98/* 100/*
99 * Uprobes-specific fetch functions 101 * Uprobes-specific fetch functions
100 */ 102 */
101#define DEFINE_FETCH_stack(type) \ 103static nokprobe_inline int
102static void FETCH_FUNC_NAME(stack, type)(struct pt_regs *regs, \ 104probe_mem_read(void *dest, void *src, size_t size)
103 void *offset, void *dest) \ 105{
104{ \ 106 void __user *vaddr = (void __force __user *)src;
105 *(type *)dest = (type)get_user_stack_nth(regs, \ 107
106 ((unsigned long)offset)); \ 108 return copy_from_user(dest, vaddr, size) ? -EFAULT : 0;
107}
108DEFINE_BASIC_FETCH_FUNCS(stack)
109/* No string on the stack entry */
110#define fetch_stack_string NULL
111#define fetch_stack_string_size NULL
112
113#define DEFINE_FETCH_memory(type) \
114static void FETCH_FUNC_NAME(memory, type)(struct pt_regs *regs, \
115 void *addr, void *dest) \
116{ \
117 type retval; \
118 void __user *vaddr = (void __force __user *) addr; \
119 \
120 if (copy_from_user(&retval, vaddr, sizeof(type))) \
121 *(type *)dest = 0; \
122 else \
123 *(type *) dest = retval; \
124} 109}
125DEFINE_BASIC_FETCH_FUNCS(memory)
126/* 110/*
127 * Fetch a null-terminated string. Caller MUST set *(u32 *)dest with max 111 * Fetch a null-terminated string. Caller MUST set *(u32 *)dest with max
128 * length and relative data location. 112 * length and relative data location.
129 */ 113 */
130static void FETCH_FUNC_NAME(memory, string)(struct pt_regs *regs, 114static nokprobe_inline int
131 void *addr, void *dest) 115fetch_store_string(unsigned long addr, void *dest, void *base)
132{ 116{
133 long ret; 117 long ret;
134 u32 rloc = *(u32 *)dest; 118 u32 loc = *(u32 *)dest;
135 int maxlen = get_rloc_len(rloc); 119 int maxlen = get_loc_len(loc);
136 u8 *dst = get_rloc_data(dest); 120 u8 *dst = get_loc_data(dest, base);
137 void __user *src = (void __force __user *) addr; 121 void __user *src = (void __force __user *) addr;
138 122
139 if (!maxlen) 123 if (unlikely(!maxlen))
140 return; 124 return -ENOMEM;
141 125
142 ret = strncpy_from_user(dst, src, maxlen); 126 ret = strncpy_from_user(dst, src, maxlen);
143 if (ret == maxlen) 127 if (ret >= 0) {
144 dst[--ret] = '\0'; 128 if (ret == maxlen)
145 129 dst[ret - 1] = '\0';
146 if (ret < 0) { /* Failed to fetch string */ 130 *(u32 *)dest = make_data_loc(ret, (void *)dst - base);
147 ((u8 *)get_rloc_data(dest))[0] = '\0';
148 *(u32 *)dest = make_data_rloc(0, get_rloc_offs(rloc));
149 } else {
150 *(u32 *)dest = make_data_rloc(ret, get_rloc_offs(rloc));
151 } 131 }
132
133 return ret;
152} 134}
153 135
154static void FETCH_FUNC_NAME(memory, string_size)(struct pt_regs *regs, 136/* Return the length of string -- including null terminal byte */
155 void *addr, void *dest) 137static nokprobe_inline int
138fetch_store_strlen(unsigned long addr)
156{ 139{
157 int len; 140 int len;
158 void __user *vaddr = (void __force __user *) addr; 141 void __user *vaddr = (void __force __user *) addr;
159 142
160 len = strnlen_user(vaddr, MAX_STRING_SIZE); 143 len = strnlen_user(vaddr, MAX_STRING_SIZE);
161 144
162 if (len == 0 || len > MAX_STRING_SIZE) /* Failed to check length */ 145 return (len > MAX_STRING_SIZE) ? 0 : len;
163 *(u32 *)dest = 0;
164 else
165 *(u32 *)dest = len;
166} 146}
167 147
168static unsigned long translate_user_vaddr(void *file_offset) 148static unsigned long translate_user_vaddr(unsigned long file_offset)
169{ 149{
170 unsigned long base_addr; 150 unsigned long base_addr;
171 struct uprobe_dispatch_data *udd; 151 struct uprobe_dispatch_data *udd;
@@ -173,44 +153,44 @@ static unsigned long translate_user_vaddr(void *file_offset)
173 udd = (void *) current->utask->vaddr; 153 udd = (void *) current->utask->vaddr;
174 154
175 base_addr = udd->bp_addr - udd->tu->offset; 155 base_addr = udd->bp_addr - udd->tu->offset;
176 return base_addr + (unsigned long)file_offset; 156 return base_addr + file_offset;
177} 157}
178 158
179#define DEFINE_FETCH_file_offset(type) \ 159/* Note that we don't verify it, since the code does not come from user space */
180static void FETCH_FUNC_NAME(file_offset, type)(struct pt_regs *regs, \ 160static int
181 void *offset, void *dest)\ 161process_fetch_insn(struct fetch_insn *code, struct pt_regs *regs, void *dest,
182{ \ 162 void *base)
183 void *vaddr = (void *)translate_user_vaddr(offset); \ 163{
184 \ 164 unsigned long val;
185 FETCH_FUNC_NAME(memory, type)(regs, vaddr, dest); \ 165
166 /* 1st stage: get value from context */
167 switch (code->op) {
168 case FETCH_OP_REG:
169 val = regs_get_register(regs, code->param);
170 break;
171 case FETCH_OP_STACK:
172 val = get_user_stack_nth(regs, code->param);
173 break;
174 case FETCH_OP_STACKP:
175 val = user_stack_pointer(regs);
176 break;
177 case FETCH_OP_RETVAL:
178 val = regs_return_value(regs);
179 break;
180 case FETCH_OP_IMM:
181 val = code->immediate;
182 break;
183 case FETCH_OP_FOFFS:
184 val = translate_user_vaddr(code->immediate);
185 break;
186 default:
187 return -EILSEQ;
188 }
189 code++;
190
191 return process_fetch_insn_bottom(code, val, dest, base);
186} 192}
187DEFINE_BASIC_FETCH_FUNCS(file_offset) 193NOKPROBE_SYMBOL(process_fetch_insn)
188DEFINE_FETCH_file_offset(string)
189DEFINE_FETCH_file_offset(string_size)
190
191/* Fetch type information table */
192static const struct fetch_type uprobes_fetch_type_table[] = {
193 /* Special types */
194 [FETCH_TYPE_STRING] = __ASSIGN_FETCH_TYPE("string", string, string,
195 sizeof(u32), 1, "__data_loc char[]"),
196 [FETCH_TYPE_STRSIZE] = __ASSIGN_FETCH_TYPE("string_size", u32,
197 string_size, sizeof(u32), 0, "u32"),
198 /* Basic types */
199 ASSIGN_FETCH_TYPE(u8, u8, 0),
200 ASSIGN_FETCH_TYPE(u16, u16, 0),
201 ASSIGN_FETCH_TYPE(u32, u32, 0),
202 ASSIGN_FETCH_TYPE(u64, u64, 0),
203 ASSIGN_FETCH_TYPE(s8, u8, 1),
204 ASSIGN_FETCH_TYPE(s16, u16, 1),
205 ASSIGN_FETCH_TYPE(s32, u32, 1),
206 ASSIGN_FETCH_TYPE(s64, u64, 1),
207 ASSIGN_FETCH_TYPE_ALIAS(x8, u8, u8, 0),
208 ASSIGN_FETCH_TYPE_ALIAS(x16, u16, u16, 0),
209 ASSIGN_FETCH_TYPE_ALIAS(x32, u32, u32, 0),
210 ASSIGN_FETCH_TYPE_ALIAS(x64, u64, u64, 0),
211
212 ASSIGN_FETCH_TYPE_END
213};
214 194
215static inline void init_trace_uprobe_filter(struct trace_uprobe_filter *filter) 195static inline void init_trace_uprobe_filter(struct trace_uprobe_filter *filter)
216{ 196{
@@ -311,6 +291,35 @@ static int unregister_trace_uprobe(struct trace_uprobe *tu)
311 return 0; 291 return 0;
312} 292}
313 293
294/*
295 * Uprobe with multiple reference counter is not allowed. i.e.
296 * If inode and offset matches, reference counter offset *must*
297 * match as well. Though, there is one exception: If user is
298 * replacing old trace_uprobe with new one(same group/event),
299 * then we allow same uprobe with new reference counter as far
300 * as the new one does not conflict with any other existing
301 * ones.
302 */
303static struct trace_uprobe *find_old_trace_uprobe(struct trace_uprobe *new)
304{
305 struct trace_uprobe *tmp, *old = NULL;
306 struct inode *new_inode = d_real_inode(new->path.dentry);
307
308 old = find_probe_event(trace_event_name(&new->tp.call),
309 new->tp.call.class->system);
310
311 list_for_each_entry(tmp, &uprobe_list, list) {
312 if ((old ? old != tmp : true) &&
313 new_inode == d_real_inode(tmp->path.dentry) &&
314 new->offset == tmp->offset &&
315 new->ref_ctr_offset != tmp->ref_ctr_offset) {
316 pr_warn("Reference counter offset mismatch.");
317 return ERR_PTR(-EINVAL);
318 }
319 }
320 return old;
321}
322
314/* Register a trace_uprobe and probe_event */ 323/* Register a trace_uprobe and probe_event */
315static int register_trace_uprobe(struct trace_uprobe *tu) 324static int register_trace_uprobe(struct trace_uprobe *tu)
316{ 325{
@@ -320,8 +329,12 @@ static int register_trace_uprobe(struct trace_uprobe *tu)
320 mutex_lock(&uprobe_lock); 329 mutex_lock(&uprobe_lock);
321 330
322 /* register as an event */ 331 /* register as an event */
323 old_tu = find_probe_event(trace_event_name(&tu->tp.call), 332 old_tu = find_old_trace_uprobe(tu);
324 tu->tp.call.class->system); 333 if (IS_ERR(old_tu)) {
334 ret = PTR_ERR(old_tu);
335 goto end;
336 }
337
325 if (old_tu) { 338 if (old_tu) {
326 /* delete old event */ 339 /* delete old event */
327 ret = unregister_trace_uprobe(old_tu); 340 ret = unregister_trace_uprobe(old_tu);
@@ -352,10 +365,10 @@ end:
352static int create_trace_uprobe(int argc, char **argv) 365static int create_trace_uprobe(int argc, char **argv)
353{ 366{
354 struct trace_uprobe *tu; 367 struct trace_uprobe *tu;
355 char *arg, *event, *group, *filename; 368 char *arg, *event, *group, *filename, *rctr, *rctr_end;
356 char buf[MAX_EVENT_NAME_LEN]; 369 char buf[MAX_EVENT_NAME_LEN];
357 struct path path; 370 struct path path;
358 unsigned long offset; 371 unsigned long offset, ref_ctr_offset;
359 bool is_delete, is_return; 372 bool is_delete, is_return;
360 int i, ret; 373 int i, ret;
361 374
@@ -364,6 +377,7 @@ static int create_trace_uprobe(int argc, char **argv)
364 is_return = false; 377 is_return = false;
365 event = NULL; 378 event = NULL;
366 group = NULL; 379 group = NULL;
380 ref_ctr_offset = 0;
367 381
368 /* argc must be >= 1 */ 382 /* argc must be >= 1 */
369 if (argv[0][0] == '-') 383 if (argv[0][0] == '-')
@@ -438,6 +452,26 @@ static int create_trace_uprobe(int argc, char **argv)
438 goto fail_address_parse; 452 goto fail_address_parse;
439 } 453 }
440 454
455 /* Parse reference counter offset if specified. */
456 rctr = strchr(arg, '(');
457 if (rctr) {
458 rctr_end = strchr(rctr, ')');
459 if (rctr > rctr_end || *(rctr_end + 1) != 0) {
460 ret = -EINVAL;
461 pr_info("Invalid reference counter offset.\n");
462 goto fail_address_parse;
463 }
464
465 *rctr++ = '\0';
466 *rctr_end = '\0';
467 ret = kstrtoul(rctr, 0, &ref_ctr_offset);
468 if (ret) {
469 pr_info("Invalid reference counter offset.\n");
470 goto fail_address_parse;
471 }
472 }
473
474 /* Parse uprobe offset. */
441 ret = kstrtoul(arg, 0, &offset); 475 ret = kstrtoul(arg, 0, &offset);
442 if (ret) 476 if (ret)
443 goto fail_address_parse; 477 goto fail_address_parse;
@@ -472,6 +506,7 @@ static int create_trace_uprobe(int argc, char **argv)
472 goto fail_address_parse; 506 goto fail_address_parse;
473 } 507 }
474 tu->offset = offset; 508 tu->offset = offset;
509 tu->ref_ctr_offset = ref_ctr_offset;
475 tu->path = path; 510 tu->path = path;
476 tu->filename = kstrdup(filename, GFP_KERNEL); 511 tu->filename = kstrdup(filename, GFP_KERNEL);
477 512
@@ -522,8 +557,7 @@ static int create_trace_uprobe(int argc, char **argv)
522 557
523 /* Parse fetch argument */ 558 /* Parse fetch argument */
524 ret = traceprobe_parse_probe_arg(arg, &tu->tp.size, parg, 559 ret = traceprobe_parse_probe_arg(arg, &tu->tp.size, parg,
525 is_return, false, 560 is_return ? TPARG_FL_RETURN : 0);
526 uprobes_fetch_type_table);
527 if (ret) { 561 if (ret) {
528 pr_info("Parse error at argument[%d]. (%d)\n", i, ret); 562 pr_info("Parse error at argument[%d]. (%d)\n", i, ret);
529 goto error; 563 goto error;
@@ -590,6 +624,9 @@ static int probes_seq_show(struct seq_file *m, void *v)
590 trace_event_name(&tu->tp.call), tu->filename, 624 trace_event_name(&tu->tp.call), tu->filename,
591 (int)(sizeof(void *) * 2), tu->offset); 625 (int)(sizeof(void *) * 2), tu->offset);
592 626
627 if (tu->ref_ctr_offset)
628 seq_printf(m, "(0x%lx)", tu->ref_ctr_offset);
629
593 for (i = 0; i < tu->tp.nr_args; i++) 630 for (i = 0; i < tu->tp.nr_args; i++)
594 seq_printf(m, " %s=%s", tu->tp.args[i].name, tu->tp.args[i].comm); 631 seq_printf(m, " %s=%s", tu->tp.args[i].name, tu->tp.args[i].comm);
595 632
@@ -833,7 +870,6 @@ print_uprobe_event(struct trace_iterator *iter, int flags, struct trace_event *e
833 struct trace_seq *s = &iter->seq; 870 struct trace_seq *s = &iter->seq;
834 struct trace_uprobe *tu; 871 struct trace_uprobe *tu;
835 u8 *data; 872 u8 *data;
836 int i;
837 873
838 entry = (struct uprobe_trace_entry_head *)iter->ent; 874 entry = (struct uprobe_trace_entry_head *)iter->ent;
839 tu = container_of(event, struct trace_uprobe, tp.call.event); 875 tu = container_of(event, struct trace_uprobe, tp.call.event);
@@ -850,12 +886,8 @@ print_uprobe_event(struct trace_iterator *iter, int flags, struct trace_event *e
850 data = DATAOF_TRACE_ENTRY(entry, false); 886 data = DATAOF_TRACE_ENTRY(entry, false);
851 } 887 }
852 888
853 for (i = 0; i < tu->tp.nr_args; i++) { 889 if (print_probe_args(s, tu->tp.args, tu->tp.nr_args, data, entry) < 0)
854 struct probe_arg *parg = &tu->tp.args[i]; 890 goto out;
855
856 if (!parg->type->print(s, parg->name, data + parg->offset, entry))
857 goto out;
858 }
859 891
860 trace_seq_putc(s, '\n'); 892 trace_seq_putc(s, '\n');
861 893
@@ -905,7 +937,13 @@ probe_event_enable(struct trace_uprobe *tu, struct trace_event_file *file,
905 937
906 tu->consumer.filter = filter; 938 tu->consumer.filter = filter;
907 tu->inode = d_real_inode(tu->path.dentry); 939 tu->inode = d_real_inode(tu->path.dentry);
908 ret = uprobe_register(tu->inode, tu->offset, &tu->consumer); 940 if (tu->ref_ctr_offset) {
941 ret = uprobe_register_refctr(tu->inode, tu->offset,
942 tu->ref_ctr_offset, &tu->consumer);
943 } else {
944 ret = uprobe_register(tu->inode, tu->offset, &tu->consumer);
945 }
946
909 if (ret) 947 if (ret)
910 goto err_buffer; 948 goto err_buffer;
911 949
@@ -958,7 +996,7 @@ probe_event_disable(struct trace_uprobe *tu, struct trace_event_file *file)
958 996
959static int uprobe_event_define_fields(struct trace_event_call *event_call) 997static int uprobe_event_define_fields(struct trace_event_call *event_call)
960{ 998{
961 int ret, i, size; 999 int ret, size;
962 struct uprobe_trace_entry_head field; 1000 struct uprobe_trace_entry_head field;
963 struct trace_uprobe *tu = event_call->data; 1001 struct trace_uprobe *tu = event_call->data;
964 1002
@@ -970,19 +1008,8 @@ static int uprobe_event_define_fields(struct trace_event_call *event_call)
970 DEFINE_FIELD(unsigned long, vaddr[0], FIELD_STRING_IP, 0); 1008 DEFINE_FIELD(unsigned long, vaddr[0], FIELD_STRING_IP, 0);
971 size = SIZEOF_TRACE_ENTRY(false); 1009 size = SIZEOF_TRACE_ENTRY(false);
972 } 1010 }
973 /* Set argument names as fields */
974 for (i = 0; i < tu->tp.nr_args; i++) {
975 struct probe_arg *parg = &tu->tp.args[i];
976
977 ret = trace_define_field(event_call, parg->type->fmttype,
978 parg->name, size + parg->offset,
979 parg->type->size, parg->type->is_signed,
980 FILTER_OTHER);
981 1011
982 if (ret) 1012 return traceprobe_define_arg_fields(event_call, size, &tu->tp);
983 return ret;
984 }
985 return 0;
986} 1013}
987 1014
988#ifdef CONFIG_PERF_EVENTS 1015#ifdef CONFIG_PERF_EVENTS
@@ -1233,7 +1260,7 @@ static int uprobe_dispatcher(struct uprobe_consumer *con, struct pt_regs *regs)
1233 esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu)); 1260 esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));
1234 1261
1235 ucb = uprobe_buffer_get(); 1262 ucb = uprobe_buffer_get();
1236 store_trace_args(esize, &tu->tp, regs, ucb->buf, dsize); 1263 store_trace_args(ucb->buf, &tu->tp, regs, esize, dsize);
1237 1264
1238 if (tu->tp.flags & TP_FLAG_TRACE) 1265 if (tu->tp.flags & TP_FLAG_TRACE)
1239 ret |= uprobe_trace_func(tu, regs, ucb, dsize); 1266 ret |= uprobe_trace_func(tu, regs, ucb, dsize);
@@ -1268,7 +1295,7 @@ static int uretprobe_dispatcher(struct uprobe_consumer *con,
1268 esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu)); 1295 esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));
1269 1296
1270 ucb = uprobe_buffer_get(); 1297 ucb = uprobe_buffer_get();
1271 store_trace_args(esize, &tu->tp, regs, ucb->buf, dsize); 1298 store_trace_args(ucb->buf, &tu->tp, regs, esize, dsize);
1272 1299
1273 if (tu->tp.flags & TP_FLAG_TRACE) 1300 if (tu->tp.flags & TP_FLAG_TRACE)
1274 uretprobe_trace_func(tu, func, regs, ucb, dsize); 1301 uretprobe_trace_func(tu, func, regs, ucb, dsize);
@@ -1304,7 +1331,7 @@ static int register_uprobe_event(struct trace_uprobe *tu)
1304 1331
1305 init_trace_event_call(tu, call); 1332 init_trace_event_call(tu, call);
1306 1333
1307 if (set_print_fmt(&tu->tp, is_ret_probe(tu)) < 0) 1334 if (traceprobe_set_print_fmt(&tu->tp, is_ret_probe(tu)) < 0)
1308 return -ENOMEM; 1335 return -ENOMEM;
1309 1336
1310 ret = register_trace_event(&call->event); 1337 ret = register_trace_event(&call->event);
@@ -1340,7 +1367,8 @@ static int unregister_uprobe_event(struct trace_uprobe *tu)
1340 1367
1341#ifdef CONFIG_PERF_EVENTS 1368#ifdef CONFIG_PERF_EVENTS
1342struct trace_event_call * 1369struct trace_event_call *
1343create_local_trace_uprobe(char *name, unsigned long offs, bool is_return) 1370create_local_trace_uprobe(char *name, unsigned long offs,
1371 unsigned long ref_ctr_offset, bool is_return)
1344{ 1372{
1345 struct trace_uprobe *tu; 1373 struct trace_uprobe *tu;
1346 struct path path; 1374 struct path path;
@@ -1372,10 +1400,11 @@ create_local_trace_uprobe(char *name, unsigned long offs, bool is_return)
1372 1400
1373 tu->offset = offs; 1401 tu->offset = offs;
1374 tu->path = path; 1402 tu->path = path;
1403 tu->ref_ctr_offset = ref_ctr_offset;
1375 tu->filename = kstrdup(name, GFP_KERNEL); 1404 tu->filename = kstrdup(name, GFP_KERNEL);
1376 init_trace_event_call(tu, &tu->tp.call); 1405 init_trace_event_call(tu, &tu->tp.call);
1377 1406
1378 if (set_print_fmt(&tu->tp, is_ret_probe(tu)) < 0) { 1407 if (traceprobe_set_print_fmt(&tu->tp, is_ret_probe(tu)) < 0) {
1379 ret = -ENOMEM; 1408 ret = -ENOMEM;
1380 goto error; 1409 goto error;
1381 } 1410 }
diff --git a/tools/perf/util/probe-event.c b/tools/perf/util/probe-event.c
index f119eb628dbb..e86f8be89157 100644
--- a/tools/perf/util/probe-event.c
+++ b/tools/perf/util/probe-event.c
@@ -1819,6 +1819,12 @@ int parse_probe_trace_command(const char *cmd, struct probe_trace_event *tev)
1819 tp->offset = strtoul(fmt2_str, NULL, 10); 1819 tp->offset = strtoul(fmt2_str, NULL, 10);
1820 } 1820 }
1821 1821
1822 if (tev->uprobes) {
1823 fmt2_str = strchr(p, '(');
1824 if (fmt2_str)
1825 tp->ref_ctr_offset = strtoul(fmt2_str + 1, NULL, 0);
1826 }
1827
1822 tev->nargs = argc - 2; 1828 tev->nargs = argc - 2;
1823 tev->args = zalloc(sizeof(struct probe_trace_arg) * tev->nargs); 1829 tev->args = zalloc(sizeof(struct probe_trace_arg) * tev->nargs);
1824 if (tev->args == NULL) { 1830 if (tev->args == NULL) {
@@ -2012,6 +2018,22 @@ static int synthesize_probe_trace_arg(struct probe_trace_arg *arg,
2012 return err; 2018 return err;
2013} 2019}
2014 2020
2021static int
2022synthesize_uprobe_trace_def(struct probe_trace_event *tev, struct strbuf *buf)
2023{
2024 struct probe_trace_point *tp = &tev->point;
2025 int err;
2026
2027 err = strbuf_addf(buf, "%s:0x%lx", tp->module, tp->address);
2028
2029 if (err >= 0 && tp->ref_ctr_offset) {
2030 if (!uprobe_ref_ctr_is_supported())
2031 return -1;
2032 err = strbuf_addf(buf, "(0x%lx)", tp->ref_ctr_offset);
2033 }
2034 return err >= 0 ? 0 : -1;
2035}
2036
2015char *synthesize_probe_trace_command(struct probe_trace_event *tev) 2037char *synthesize_probe_trace_command(struct probe_trace_event *tev)
2016{ 2038{
2017 struct probe_trace_point *tp = &tev->point; 2039 struct probe_trace_point *tp = &tev->point;
@@ -2041,15 +2063,17 @@ char *synthesize_probe_trace_command(struct probe_trace_event *tev)
2041 } 2063 }
2042 2064
2043 /* Use the tp->address for uprobes */ 2065 /* Use the tp->address for uprobes */
2044 if (tev->uprobes) 2066 if (tev->uprobes) {
2045 err = strbuf_addf(&buf, "%s:0x%lx", tp->module, tp->address); 2067 err = synthesize_uprobe_trace_def(tev, &buf);
2046 else if (!strncmp(tp->symbol, "0x", 2)) 2068 } else if (!strncmp(tp->symbol, "0x", 2)) {
2047 /* Absolute address. See try_to_find_absolute_address() */ 2069 /* Absolute address. See try_to_find_absolute_address() */
2048 err = strbuf_addf(&buf, "%s%s0x%lx", tp->module ?: "", 2070 err = strbuf_addf(&buf, "%s%s0x%lx", tp->module ?: "",
2049 tp->module ? ":" : "", tp->address); 2071 tp->module ? ":" : "", tp->address);
2050 else 2072 } else {
2051 err = strbuf_addf(&buf, "%s%s%s+%lu", tp->module ?: "", 2073 err = strbuf_addf(&buf, "%s%s%s+%lu", tp->module ?: "",
2052 tp->module ? ":" : "", tp->symbol, tp->offset); 2074 tp->module ? ":" : "", tp->symbol, tp->offset);
2075 }
2076
2053 if (err) 2077 if (err)
2054 goto error; 2078 goto error;
2055 2079
@@ -2633,6 +2657,13 @@ static void warn_uprobe_event_compat(struct probe_trace_event *tev)
2633{ 2657{
2634 int i; 2658 int i;
2635 char *buf = synthesize_probe_trace_command(tev); 2659 char *buf = synthesize_probe_trace_command(tev);
2660 struct probe_trace_point *tp = &tev->point;
2661
2662 if (tp->ref_ctr_offset && !uprobe_ref_ctr_is_supported()) {
2663 pr_warning("A semaphore is associated with %s:%s and "
2664 "seems your kernel doesn't support it.\n",
2665 tev->group, tev->event);
2666 }
2636 2667
2637 /* Old uprobe event doesn't support memory dereference */ 2668 /* Old uprobe event doesn't support memory dereference */
2638 if (!tev->uprobes || tev->nargs == 0 || !buf) 2669 if (!tev->uprobes || tev->nargs == 0 || !buf)
diff --git a/tools/perf/util/probe-event.h b/tools/perf/util/probe-event.h
index 45b14f020558..15a98c3a2a2f 100644
--- a/tools/perf/util/probe-event.h
+++ b/tools/perf/util/probe-event.h
@@ -27,6 +27,7 @@ struct probe_trace_point {
27 char *symbol; /* Base symbol */ 27 char *symbol; /* Base symbol */
28 char *module; /* Module name */ 28 char *module; /* Module name */
29 unsigned long offset; /* Offset from symbol */ 29 unsigned long offset; /* Offset from symbol */
30 unsigned long ref_ctr_offset; /* SDT reference counter offset */
30 unsigned long address; /* Actual address of the trace point */ 31 unsigned long address; /* Actual address of the trace point */
31 bool retprobe; /* Return probe flag */ 32 bool retprobe; /* Return probe flag */
32}; 33};
diff --git a/tools/perf/util/probe-file.c b/tools/perf/util/probe-file.c
index b76088fadf3d..aac7817d9e14 100644
--- a/tools/perf/util/probe-file.c
+++ b/tools/perf/util/probe-file.c
@@ -696,8 +696,16 @@ out_err:
696#ifdef HAVE_GELF_GETNOTE_SUPPORT 696#ifdef HAVE_GELF_GETNOTE_SUPPORT
697static unsigned long long sdt_note__get_addr(struct sdt_note *note) 697static unsigned long long sdt_note__get_addr(struct sdt_note *note)
698{ 698{
699 return note->bit32 ? (unsigned long long)note->addr.a32[0] 699 return note->bit32 ?
700 : (unsigned long long)note->addr.a64[0]; 700 (unsigned long long)note->addr.a32[SDT_NOTE_IDX_LOC] :
701 (unsigned long long)note->addr.a64[SDT_NOTE_IDX_LOC];
702}
703
704static unsigned long long sdt_note__get_ref_ctr_offset(struct sdt_note *note)
705{
706 return note->bit32 ?
707 (unsigned long long)note->addr.a32[SDT_NOTE_IDX_REFCTR] :
708 (unsigned long long)note->addr.a64[SDT_NOTE_IDX_REFCTR];
701} 709}
702 710
703static const char * const type_to_suffix[] = { 711static const char * const type_to_suffix[] = {
@@ -775,14 +783,21 @@ static char *synthesize_sdt_probe_command(struct sdt_note *note,
775{ 783{
776 struct strbuf buf; 784 struct strbuf buf;
777 char *ret = NULL, **args; 785 char *ret = NULL, **args;
778 int i, args_count; 786 int i, args_count, err;
787 unsigned long long ref_ctr_offset;
779 788
780 if (strbuf_init(&buf, 32) < 0) 789 if (strbuf_init(&buf, 32) < 0)
781 return NULL; 790 return NULL;
782 791
783 if (strbuf_addf(&buf, "p:%s/%s %s:0x%llx", 792 err = strbuf_addf(&buf, "p:%s/%s %s:0x%llx",
784 sdtgrp, note->name, pathname, 793 sdtgrp, note->name, pathname,
785 sdt_note__get_addr(note)) < 0) 794 sdt_note__get_addr(note));
795
796 ref_ctr_offset = sdt_note__get_ref_ctr_offset(note);
797 if (ref_ctr_offset && err >= 0)
798 err = strbuf_addf(&buf, "(0x%llx)", ref_ctr_offset);
799
800 if (err < 0)
786 goto error; 801 goto error;
787 802
788 if (!note->args) 803 if (!note->args)
@@ -998,6 +1013,7 @@ int probe_cache__show_all_caches(struct strfilter *filter)
998enum ftrace_readme { 1013enum ftrace_readme {
999 FTRACE_README_PROBE_TYPE_X = 0, 1014 FTRACE_README_PROBE_TYPE_X = 0,
1000 FTRACE_README_KRETPROBE_OFFSET, 1015 FTRACE_README_KRETPROBE_OFFSET,
1016 FTRACE_README_UPROBE_REF_CTR,
1001 FTRACE_README_END, 1017 FTRACE_README_END,
1002}; 1018};
1003 1019
@@ -1009,6 +1025,7 @@ static struct {
1009 [idx] = {.pattern = pat, .avail = false} 1025 [idx] = {.pattern = pat, .avail = false}
1010 DEFINE_TYPE(FTRACE_README_PROBE_TYPE_X, "*type: * x8/16/32/64,*"), 1026 DEFINE_TYPE(FTRACE_README_PROBE_TYPE_X, "*type: * x8/16/32/64,*"),
1011 DEFINE_TYPE(FTRACE_README_KRETPROBE_OFFSET, "*place (kretprobe): *"), 1027 DEFINE_TYPE(FTRACE_README_KRETPROBE_OFFSET, "*place (kretprobe): *"),
1028 DEFINE_TYPE(FTRACE_README_UPROBE_REF_CTR, "*ref_ctr_offset*"),
1012}; 1029};
1013 1030
1014static bool scan_ftrace_readme(enum ftrace_readme type) 1031static bool scan_ftrace_readme(enum ftrace_readme type)
@@ -1064,3 +1081,8 @@ bool kretprobe_offset_is_supported(void)
1064{ 1081{
1065 return scan_ftrace_readme(FTRACE_README_KRETPROBE_OFFSET); 1082 return scan_ftrace_readme(FTRACE_README_KRETPROBE_OFFSET);
1066} 1083}
1084
1085bool uprobe_ref_ctr_is_supported(void)
1086{
1087 return scan_ftrace_readme(FTRACE_README_UPROBE_REF_CTR);
1088}
diff --git a/tools/perf/util/probe-file.h b/tools/perf/util/probe-file.h
index 63f29b1d22c1..2a249182f2a6 100644
--- a/tools/perf/util/probe-file.h
+++ b/tools/perf/util/probe-file.h
@@ -69,6 +69,7 @@ struct probe_cache_entry *probe_cache__find_by_name(struct probe_cache *pcache,
69int probe_cache__show_all_caches(struct strfilter *filter); 69int probe_cache__show_all_caches(struct strfilter *filter);
70bool probe_type_is_available(enum probe_type type); 70bool probe_type_is_available(enum probe_type type);
71bool kretprobe_offset_is_supported(void); 71bool kretprobe_offset_is_supported(void);
72bool uprobe_ref_ctr_is_supported(void);
72#else /* ! HAVE_LIBELF_SUPPORT */ 73#else /* ! HAVE_LIBELF_SUPPORT */
73static inline struct probe_cache *probe_cache__new(const char *tgt __maybe_unused, struct nsinfo *nsi __maybe_unused) 74static inline struct probe_cache *probe_cache__new(const char *tgt __maybe_unused, struct nsinfo *nsi __maybe_unused)
74{ 75{
diff --git a/tools/perf/util/symbol-elf.c b/tools/perf/util/symbol-elf.c
index 29770ea61768..0281d5e2cd67 100644
--- a/tools/perf/util/symbol-elf.c
+++ b/tools/perf/util/symbol-elf.c
@@ -1947,6 +1947,34 @@ void kcore_extract__delete(struct kcore_extract *kce)
1947} 1947}
1948 1948
1949#ifdef HAVE_GELF_GETNOTE_SUPPORT 1949#ifdef HAVE_GELF_GETNOTE_SUPPORT
1950
1951static void sdt_adjust_loc(struct sdt_note *tmp, GElf_Addr base_off)
1952{
1953 if (!base_off)
1954 return;
1955
1956 if (tmp->bit32)
1957 tmp->addr.a32[SDT_NOTE_IDX_LOC] =
1958 tmp->addr.a32[SDT_NOTE_IDX_LOC] + base_off -
1959 tmp->addr.a32[SDT_NOTE_IDX_BASE];
1960 else
1961 tmp->addr.a64[SDT_NOTE_IDX_LOC] =
1962 tmp->addr.a64[SDT_NOTE_IDX_LOC] + base_off -
1963 tmp->addr.a64[SDT_NOTE_IDX_BASE];
1964}
1965
1966static void sdt_adjust_refctr(struct sdt_note *tmp, GElf_Addr base_addr,
1967 GElf_Addr base_off)
1968{
1969 if (!base_off)
1970 return;
1971
1972 if (tmp->bit32 && tmp->addr.a32[SDT_NOTE_IDX_REFCTR])
1973 tmp->addr.a32[SDT_NOTE_IDX_REFCTR] -= (base_addr - base_off);
1974 else if (tmp->addr.a64[SDT_NOTE_IDX_REFCTR])
1975 tmp->addr.a64[SDT_NOTE_IDX_REFCTR] -= (base_addr - base_off);
1976}
1977
1950/** 1978/**
1951 * populate_sdt_note : Parse raw data and identify SDT note 1979 * populate_sdt_note : Parse raw data and identify SDT note
1952 * @elf: elf of the opened file 1980 * @elf: elf of the opened file
@@ -1964,7 +1992,6 @@ static int populate_sdt_note(Elf **elf, const char *data, size_t len,
1964 const char *provider, *name, *args; 1992 const char *provider, *name, *args;
1965 struct sdt_note *tmp = NULL; 1993 struct sdt_note *tmp = NULL;
1966 GElf_Ehdr ehdr; 1994 GElf_Ehdr ehdr;
1967 GElf_Addr base_off = 0;
1968 GElf_Shdr shdr; 1995 GElf_Shdr shdr;
1969 int ret = -EINVAL; 1996 int ret = -EINVAL;
1970 1997
@@ -2060,17 +2087,12 @@ static int populate_sdt_note(Elf **elf, const char *data, size_t len,
2060 * base address in the description of the SDT note. If its different, 2087 * base address in the description of the SDT note. If its different,
2061 * then accordingly, adjust the note location. 2088 * then accordingly, adjust the note location.
2062 */ 2089 */
2063 if (elf_section_by_name(*elf, &ehdr, &shdr, SDT_BASE_SCN, NULL)) { 2090 if (elf_section_by_name(*elf, &ehdr, &shdr, SDT_BASE_SCN, NULL))
2064 base_off = shdr.sh_offset; 2091 sdt_adjust_loc(tmp, shdr.sh_offset);
2065 if (base_off) { 2092
2066 if (tmp->bit32) 2093 /* Adjust reference counter offset */
2067 tmp->addr.a32[0] = tmp->addr.a32[0] + base_off - 2094 if (elf_section_by_name(*elf, &ehdr, &shdr, SDT_PROBES_SCN, NULL))
2068 tmp->addr.a32[1]; 2095 sdt_adjust_refctr(tmp, shdr.sh_addr, shdr.sh_offset);
2069 else
2070 tmp->addr.a64[0] = tmp->addr.a64[0] + base_off -
2071 tmp->addr.a64[1];
2072 }
2073 }
2074 2096
2075 list_add_tail(&tmp->note_list, sdt_notes); 2097 list_add_tail(&tmp->note_list, sdt_notes);
2076 return 0; 2098 return 0;
diff --git a/tools/perf/util/symbol.h b/tools/perf/util/symbol.h
index f25fae4b5743..20f49779116b 100644
--- a/tools/perf/util/symbol.h
+++ b/tools/perf/util/symbol.h
@@ -379,12 +379,19 @@ int get_sdt_note_list(struct list_head *head, const char *target);
379int cleanup_sdt_note_list(struct list_head *sdt_notes); 379int cleanup_sdt_note_list(struct list_head *sdt_notes);
380int sdt_notes__get_count(struct list_head *start); 380int sdt_notes__get_count(struct list_head *start);
381 381
382#define SDT_PROBES_SCN ".probes"
382#define SDT_BASE_SCN ".stapsdt.base" 383#define SDT_BASE_SCN ".stapsdt.base"
383#define SDT_NOTE_SCN ".note.stapsdt" 384#define SDT_NOTE_SCN ".note.stapsdt"
384#define SDT_NOTE_TYPE 3 385#define SDT_NOTE_TYPE 3
385#define SDT_NOTE_NAME "stapsdt" 386#define SDT_NOTE_NAME "stapsdt"
386#define NR_ADDR 3 387#define NR_ADDR 3
387 388
389enum {
390 SDT_NOTE_IDX_LOC = 0,
391 SDT_NOTE_IDX_BASE,
392 SDT_NOTE_IDX_REFCTR,
393};
394
388struct mem_info *mem_info__new(void); 395struct mem_info *mem_info__new(void);
389struct mem_info *mem_info__get(struct mem_info *mi); 396struct mem_info *mem_info__get(struct mem_info *mi);
390void mem_info__put(struct mem_info *mi); 397void mem_info__put(struct mem_info *mi);
generated by cgit v1.2.2 (git 2.25.0) at 2025-09-11 04:27:47 -0400