aboutsummaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
authorIngo Molnar <mingo@kernel.org>2018-11-03 18:42:16 -0400
committerIngo Molnar <mingo@kernel.org>2018-11-03 18:42:16 -0400
commit23a12ddee1ce28065b71f14ccc695b5a0c8a64ff (patch)
treecedaa1cde5b2557116e523c31552187804704093 /kernel
parent98f76206b33504b934209d16196477dfa519a807 (diff)
parentbcb6fb5da77c2a228adf07cc9cb1a0c2aa2001c6 (diff)
Merge branch 'core/urgent' into x86/urgent, to pick up objtool fix
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Diffstat (limited to 'kernel')
-rw-r--r--kernel/bounds.c4
-rw-r--r--kernel/bpf/btf.c58
-rw-r--r--kernel/bpf/core.c49
-rw-r--r--kernel/bpf/devmap.c3
-rw-r--r--kernel/bpf/helpers.c2
-rw-r--r--kernel/bpf/queue_stack_maps.c2
-rw-r--r--kernel/bpf/verifier.c13
-rw-r--r--kernel/dma/direct.c2
-rw-r--r--kernel/dma/swiotlb.c8
-rw-r--r--kernel/events/core.c49
-rw-r--r--kernel/events/uprobes.c278
-rw-r--r--kernel/fail_function.c3
-rw-r--r--kernel/futex.c2
-rw-r--r--kernel/hung_task.c30
-rw-r--r--kernel/kallsyms.c4
-rw-r--r--kernel/locking/qspinlock_paravirt.h2
-rw-r--r--kernel/memremap.c75
-rw-r--r--kernel/panic.c10
-rw-r--r--kernel/pid.c2
-rw-r--r--kernel/power/snapshot.c5
-rw-r--r--kernel/printk/printk.c5
-rw-r--r--kernel/profile.c2
-rw-r--r--kernel/sched/core.c34
-rw-r--r--kernel/signal.c2
-rw-r--r--kernel/trace/trace.c12
-rw-r--r--kernel/trace/trace_event_perf.c7
-rw-r--r--kernel/trace/trace_events_hist.c4
-rw-r--r--kernel/trace/trace_kprobe.c412
-rw-r--r--kernel/trace/trace_probe.c672
-rw-r--r--kernel/trace/trace_probe.h289
-rw-r--r--kernel/trace/trace_probe_tmpl.h216
-rw-r--r--kernel/trace/trace_stack.c2
-rw-r--r--kernel/trace/trace_uprobe.c255
33 files changed, 1433 insertions, 1080 deletions
diff --git a/kernel/bounds.c b/kernel/bounds.c
index c373e887c066..9795d75b09b2 100644
--- a/kernel/bounds.c
+++ b/kernel/bounds.c
@@ -13,7 +13,7 @@
13#include <linux/log2.h> 13#include <linux/log2.h>
14#include <linux/spinlock_types.h> 14#include <linux/spinlock_types.h>
15 15
16void foo(void) 16int main(void)
17{ 17{
18 /* The enum constants to put into include/generated/bounds.h */ 18 /* The enum constants to put into include/generated/bounds.h */
19 DEFINE(NR_PAGEFLAGS, __NR_PAGEFLAGS); 19 DEFINE(NR_PAGEFLAGS, __NR_PAGEFLAGS);
@@ -23,4 +23,6 @@ void foo(void)
23#endif 23#endif
24 DEFINE(SPINLOCK_SIZE, sizeof(spinlock_t)); 24 DEFINE(SPINLOCK_SIZE, sizeof(spinlock_t));
25 /* End of constants */ 25 /* End of constants */
26
27 return 0;
26} 28}
diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c
index 378cef70341c..ee4c82667d65 100644
--- a/kernel/bpf/btf.c
+++ b/kernel/bpf/btf.c
@@ -2067,56 +2067,47 @@ static int btf_check_sec_info(struct btf_verifier_env *env,
2067 return 0; 2067 return 0;
2068} 2068}
2069 2069
2070static int btf_parse_hdr(struct btf_verifier_env *env, void __user *btf_data, 2070static int btf_parse_hdr(struct btf_verifier_env *env)
2071 u32 btf_data_size)
2072{ 2071{
2072 u32 hdr_len, hdr_copy, btf_data_size;
2073 const struct btf_header *hdr; 2073 const struct btf_header *hdr;
2074 u32 hdr_len, hdr_copy;
2075 /*
2076 * Minimal part of the "struct btf_header" that
2077 * contains the hdr_len.
2078 */
2079 struct btf_min_header {
2080 u16 magic;
2081 u8 version;
2082 u8 flags;
2083 u32 hdr_len;
2084 } __user *min_hdr;
2085 struct btf *btf; 2074 struct btf *btf;
2086 int err; 2075 int err;
2087 2076
2088 btf = env->btf; 2077 btf = env->btf;
2089 min_hdr = btf_data; 2078 btf_data_size = btf->data_size;
2090 2079
2091 if (btf_data_size < sizeof(*min_hdr)) { 2080 if (btf_data_size <
2081 offsetof(struct btf_header, hdr_len) + sizeof(hdr->hdr_len)) {
2092 btf_verifier_log(env, "hdr_len not found"); 2082 btf_verifier_log(env, "hdr_len not found");
2093 return -EINVAL; 2083 return -EINVAL;
2094 } 2084 }
2095 2085
2096 if (get_user(hdr_len, &min_hdr->hdr_len)) 2086 hdr = btf->data;
2097 return -EFAULT; 2087 hdr_len = hdr->hdr_len;
2098
2099 if (btf_data_size < hdr_len) { 2088 if (btf_data_size < hdr_len) {
2100 btf_verifier_log(env, "btf_header not found"); 2089 btf_verifier_log(env, "btf_header not found");
2101 return -EINVAL; 2090 return -EINVAL;
2102 } 2091 }
2103 2092
2104 err = bpf_check_uarg_tail_zero(btf_data, sizeof(btf->hdr), hdr_len); 2093 /* Ensure the unsupported header fields are zero */
2105 if (err) { 2094 if (hdr_len > sizeof(btf->hdr)) {
2106 if (err == -E2BIG) 2095 u8 *expected_zero = btf->data + sizeof(btf->hdr);
2107 btf_verifier_log(env, "Unsupported btf_header"); 2096 u8 *end = btf->data + hdr_len;
2108 return err; 2097
2098 for (; expected_zero < end; expected_zero++) {
2099 if (*expected_zero) {
2100 btf_verifier_log(env, "Unsupported btf_header");
2101 return -E2BIG;
2102 }
2103 }
2109 } 2104 }
2110 2105
2111 hdr_copy = min_t(u32, hdr_len, sizeof(btf->hdr)); 2106 hdr_copy = min_t(u32, hdr_len, sizeof(btf->hdr));
2112 if (copy_from_user(&btf->hdr, btf_data, hdr_copy)) 2107 memcpy(&btf->hdr, btf->data, hdr_copy);
2113 return -EFAULT;
2114 2108
2115 hdr = &btf->hdr; 2109 hdr = &btf->hdr;
2116 2110
2117 if (hdr->hdr_len != hdr_len)
2118 return -EINVAL;
2119
2120 btf_verifier_log_hdr(env, btf_data_size); 2111 btf_verifier_log_hdr(env, btf_data_size);
2121 2112
2122 if (hdr->magic != BTF_MAGIC) { 2113 if (hdr->magic != BTF_MAGIC) {
@@ -2186,10 +2177,6 @@ static struct btf *btf_parse(void __user *btf_data, u32 btf_data_size,
2186 } 2177 }
2187 env->btf = btf; 2178 env->btf = btf;
2188 2179
2189 err = btf_parse_hdr(env, btf_data, btf_data_size);
2190 if (err)
2191 goto errout;
2192
2193 data = kvmalloc(btf_data_size, GFP_KERNEL | __GFP_NOWARN); 2180 data = kvmalloc(btf_data_size, GFP_KERNEL | __GFP_NOWARN);
2194 if (!data) { 2181 if (!data) {
2195 err = -ENOMEM; 2182 err = -ENOMEM;
@@ -2198,13 +2185,18 @@ static struct btf *btf_parse(void __user *btf_data, u32 btf_data_size,
2198 2185
2199 btf->data = data; 2186 btf->data = data;
2200 btf->data_size = btf_data_size; 2187 btf->data_size = btf_data_size;
2201 btf->nohdr_data = btf->data + btf->hdr.hdr_len;
2202 2188
2203 if (copy_from_user(data, btf_data, btf_data_size)) { 2189 if (copy_from_user(data, btf_data, btf_data_size)) {
2204 err = -EFAULT; 2190 err = -EFAULT;
2205 goto errout; 2191 goto errout;
2206 } 2192 }
2207 2193
2194 err = btf_parse_hdr(env);
2195 if (err)
2196 goto errout;
2197
2198 btf->nohdr_data = btf->data + btf->hdr.hdr_len;
2199
2208 err = btf_parse_str_sec(env); 2200 err = btf_parse_str_sec(env);
2209 if (err) 2201 if (err)
2210 goto errout; 2202 goto errout;
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 7c7eeea8cffc..6377225b2082 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -365,10 +365,13 @@ void bpf_prog_kallsyms_del_all(struct bpf_prog *fp)
365} 365}
366 366
367#ifdef CONFIG_BPF_JIT 367#ifdef CONFIG_BPF_JIT
368# define BPF_JIT_LIMIT_DEFAULT (PAGE_SIZE * 40000)
369
368/* All BPF JIT sysctl knobs here. */ 370/* All BPF JIT sysctl knobs here. */
369int bpf_jit_enable __read_mostly = IS_BUILTIN(CONFIG_BPF_JIT_ALWAYS_ON); 371int bpf_jit_enable __read_mostly = IS_BUILTIN(CONFIG_BPF_JIT_ALWAYS_ON);
370int bpf_jit_harden __read_mostly; 372int bpf_jit_harden __read_mostly;
371int bpf_jit_kallsyms __read_mostly; 373int bpf_jit_kallsyms __read_mostly;
374int bpf_jit_limit __read_mostly = BPF_JIT_LIMIT_DEFAULT;
372 375
373static __always_inline void 376static __always_inline void
374bpf_get_prog_addr_region(const struct bpf_prog *prog, 377bpf_get_prog_addr_region(const struct bpf_prog *prog,
@@ -577,27 +580,64 @@ int bpf_get_kallsym(unsigned int symnum, unsigned long *value, char *type,
577 return ret; 580 return ret;
578} 581}
579 582
583static atomic_long_t bpf_jit_current;
584
585#if defined(MODULES_VADDR)
586static int __init bpf_jit_charge_init(void)
587{
588 /* Only used as heuristic here to derive limit. */
589 bpf_jit_limit = min_t(u64, round_up((MODULES_END - MODULES_VADDR) >> 2,
590 PAGE_SIZE), INT_MAX);
591 return 0;
592}
593pure_initcall(bpf_jit_charge_init);
594#endif
595
596static int bpf_jit_charge_modmem(u32 pages)
597{
598 if (atomic_long_add_return(pages, &bpf_jit_current) >
599 (bpf_jit_limit >> PAGE_SHIFT)) {
600 if (!capable(CAP_SYS_ADMIN)) {
601 atomic_long_sub(pages, &bpf_jit_current);
602 return -EPERM;
603 }
604 }
605
606 return 0;
607}
608
609static void bpf_jit_uncharge_modmem(u32 pages)
610{
611 atomic_long_sub(pages, &bpf_jit_current);
612}
613
580struct bpf_binary_header * 614struct bpf_binary_header *
581bpf_jit_binary_alloc(unsigned int proglen, u8 **image_ptr, 615bpf_jit_binary_alloc(unsigned int proglen, u8 **image_ptr,
582 unsigned int alignment, 616 unsigned int alignment,
583 bpf_jit_fill_hole_t bpf_fill_ill_insns) 617 bpf_jit_fill_hole_t bpf_fill_ill_insns)
584{ 618{
585 struct bpf_binary_header *hdr; 619 struct bpf_binary_header *hdr;
586 unsigned int size, hole, start; 620 u32 size, hole, start, pages;
587 621
588 /* Most of BPF filters are really small, but if some of them 622 /* Most of BPF filters are really small, but if some of them
589 * fill a page, allow at least 128 extra bytes to insert a 623 * fill a page, allow at least 128 extra bytes to insert a
590 * random section of illegal instructions. 624 * random section of illegal instructions.
591 */ 625 */
592 size = round_up(proglen + sizeof(*hdr) + 128, PAGE_SIZE); 626 size = round_up(proglen + sizeof(*hdr) + 128, PAGE_SIZE);
627 pages = size / PAGE_SIZE;
628
629 if (bpf_jit_charge_modmem(pages))
630 return NULL;
593 hdr = module_alloc(size); 631 hdr = module_alloc(size);
594 if (hdr == NULL) 632 if (!hdr) {
633 bpf_jit_uncharge_modmem(pages);
595 return NULL; 634 return NULL;
635 }
596 636
597 /* Fill space with illegal/arch-dep instructions. */ 637 /* Fill space with illegal/arch-dep instructions. */
598 bpf_fill_ill_insns(hdr, size); 638 bpf_fill_ill_insns(hdr, size);
599 639
600 hdr->pages = size / PAGE_SIZE; 640 hdr->pages = pages;
601 hole = min_t(unsigned int, size - (proglen + sizeof(*hdr)), 641 hole = min_t(unsigned int, size - (proglen + sizeof(*hdr)),
602 PAGE_SIZE - sizeof(*hdr)); 642 PAGE_SIZE - sizeof(*hdr));
603 start = (get_random_int() % hole) & ~(alignment - 1); 643 start = (get_random_int() % hole) & ~(alignment - 1);
@@ -610,7 +650,10 @@ bpf_jit_binary_alloc(unsigned int proglen, u8 **image_ptr,
610 650
611void bpf_jit_binary_free(struct bpf_binary_header *hdr) 651void bpf_jit_binary_free(struct bpf_binary_header *hdr)
612{ 652{
653 u32 pages = hdr->pages;
654
613 module_memfree(hdr); 655 module_memfree(hdr);
656 bpf_jit_uncharge_modmem(pages);
614} 657}
615 658
616/* This symbol is only overridden by archs that have different 659/* This symbol is only overridden by archs that have different
diff --git a/kernel/bpf/devmap.c b/kernel/bpf/devmap.c
index 141710b82a6c..191b79948424 100644
--- a/kernel/bpf/devmap.c
+++ b/kernel/bpf/devmap.c
@@ -512,8 +512,7 @@ static int dev_map_notification(struct notifier_block *notifier,
512 struct bpf_dtab_netdev *dev, *odev; 512 struct bpf_dtab_netdev *dev, *odev;
513 513
514 dev = READ_ONCE(dtab->netdev_map[i]); 514 dev = READ_ONCE(dtab->netdev_map[i]);
515 if (!dev || 515 if (!dev || netdev != dev->dev)
516 dev->dev->ifindex != netdev->ifindex)
517 continue; 516 continue;
518 odev = cmpxchg(&dtab->netdev_map[i], dev, NULL); 517 odev = cmpxchg(&dtab->netdev_map[i], dev, NULL);
519 if (dev == odev) 518 if (dev == odev)
diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c
index ab0d5e3f9892..a74972b07e74 100644
--- a/kernel/bpf/helpers.c
+++ b/kernel/bpf/helpers.c
@@ -99,7 +99,6 @@ BPF_CALL_2(bpf_map_pop_elem, struct bpf_map *, map, void *, value)
99const struct bpf_func_proto bpf_map_pop_elem_proto = { 99const struct bpf_func_proto bpf_map_pop_elem_proto = {
100 .func = bpf_map_pop_elem, 100 .func = bpf_map_pop_elem,
101 .gpl_only = false, 101 .gpl_only = false,
102 .pkt_access = true,
103 .ret_type = RET_INTEGER, 102 .ret_type = RET_INTEGER,
104 .arg1_type = ARG_CONST_MAP_PTR, 103 .arg1_type = ARG_CONST_MAP_PTR,
105 .arg2_type = ARG_PTR_TO_UNINIT_MAP_VALUE, 104 .arg2_type = ARG_PTR_TO_UNINIT_MAP_VALUE,
@@ -113,7 +112,6 @@ BPF_CALL_2(bpf_map_peek_elem, struct bpf_map *, map, void *, value)
113const struct bpf_func_proto bpf_map_peek_elem_proto = { 112const struct bpf_func_proto bpf_map_peek_elem_proto = {
114 .func = bpf_map_pop_elem, 113 .func = bpf_map_pop_elem,
115 .gpl_only = false, 114 .gpl_only = false,
116 .pkt_access = true,
117 .ret_type = RET_INTEGER, 115 .ret_type = RET_INTEGER,
118 .arg1_type = ARG_CONST_MAP_PTR, 116 .arg1_type = ARG_CONST_MAP_PTR,
119 .arg2_type = ARG_PTR_TO_UNINIT_MAP_VALUE, 117 .arg2_type = ARG_PTR_TO_UNINIT_MAP_VALUE,
diff --git a/kernel/bpf/queue_stack_maps.c b/kernel/bpf/queue_stack_maps.c
index 12a93fb37449..8bbd72d3a121 100644
--- a/kernel/bpf/queue_stack_maps.c
+++ b/kernel/bpf/queue_stack_maps.c
@@ -122,6 +122,7 @@ static int __queue_map_get(struct bpf_map *map, void *value, bool delete)
122 raw_spin_lock_irqsave(&qs->lock, flags); 122 raw_spin_lock_irqsave(&qs->lock, flags);
123 123
124 if (queue_stack_map_is_empty(qs)) { 124 if (queue_stack_map_is_empty(qs)) {
125 memset(value, 0, qs->map.value_size);
125 err = -ENOENT; 126 err = -ENOENT;
126 goto out; 127 goto out;
127 } 128 }
@@ -151,6 +152,7 @@ static int __stack_map_get(struct bpf_map *map, void *value, bool delete)
151 raw_spin_lock_irqsave(&qs->lock, flags); 152 raw_spin_lock_irqsave(&qs->lock, flags);
152 153
153 if (queue_stack_map_is_empty(qs)) { 154 if (queue_stack_map_is_empty(qs)) {
155 memset(value, 0, qs->map.value_size);
154 err = -ENOENT; 156 err = -ENOENT;
155 goto out; 157 goto out;
156 } 158 }
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 98fa0be35370..171a2c88e77d 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -1387,21 +1387,24 @@ static bool may_access_direct_pkt_data(struct bpf_verifier_env *env,
1387 enum bpf_access_type t) 1387 enum bpf_access_type t)
1388{ 1388{
1389 switch (env->prog->type) { 1389 switch (env->prog->type) {
1390 /* Program types only with direct read access go here! */
1390 case BPF_PROG_TYPE_LWT_IN: 1391 case BPF_PROG_TYPE_LWT_IN:
1391 case BPF_PROG_TYPE_LWT_OUT: 1392 case BPF_PROG_TYPE_LWT_OUT:
1392 case BPF_PROG_TYPE_LWT_SEG6LOCAL: 1393 case BPF_PROG_TYPE_LWT_SEG6LOCAL:
1393 case BPF_PROG_TYPE_SK_REUSEPORT: 1394 case BPF_PROG_TYPE_SK_REUSEPORT:
1394 /* dst_input() and dst_output() can't write for now */ 1395 case BPF_PROG_TYPE_FLOW_DISSECTOR:
1396 case BPF_PROG_TYPE_CGROUP_SKB:
1395 if (t == BPF_WRITE) 1397 if (t == BPF_WRITE)
1396 return false; 1398 return false;
1397 /* fallthrough */ 1399 /* fallthrough */
1400
1401 /* Program types with direct read + write access go here! */
1398 case BPF_PROG_TYPE_SCHED_CLS: 1402 case BPF_PROG_TYPE_SCHED_CLS:
1399 case BPF_PROG_TYPE_SCHED_ACT: 1403 case BPF_PROG_TYPE_SCHED_ACT:
1400 case BPF_PROG_TYPE_XDP: 1404 case BPF_PROG_TYPE_XDP:
1401 case BPF_PROG_TYPE_LWT_XMIT: 1405 case BPF_PROG_TYPE_LWT_XMIT:
1402 case BPF_PROG_TYPE_SK_SKB: 1406 case BPF_PROG_TYPE_SK_SKB:
1403 case BPF_PROG_TYPE_SK_MSG: 1407 case BPF_PROG_TYPE_SK_MSG:
1404 case BPF_PROG_TYPE_FLOW_DISSECTOR:
1405 if (meta) 1408 if (meta)
1406 return meta->pkt_access; 1409 return meta->pkt_access;
1407 1410
@@ -5706,7 +5709,11 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env)
5706 bool is_narrower_load; 5709 bool is_narrower_load;
5707 u32 target_size; 5710 u32 target_size;
5708 5711
5709 if (ops->gen_prologue) { 5712 if (ops->gen_prologue || env->seen_direct_write) {
5713 if (!ops->gen_prologue) {
5714 verbose(env, "bpf verifier is misconfigured\n");
5715 return -EINVAL;
5716 }
5710 cnt = ops->gen_prologue(insn_buf, env->seen_direct_write, 5717 cnt = ops->gen_prologue(insn_buf, env->seen_direct_write,
5711 env->prog); 5718 env->prog);
5712 if (cnt >= ARRAY_SIZE(insn_buf)) { 5719 if (cnt >= ARRAY_SIZE(insn_buf)) {
diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c
index f14c376937e5..22a12ab5a5e9 100644
--- a/kernel/dma/direct.c
+++ b/kernel/dma/direct.c
@@ -4,7 +4,7 @@
4 * 4 *
5 * DMA operations that map physical memory directly without using an IOMMU. 5 * DMA operations that map physical memory directly without using an IOMMU.
6 */ 6 */
7#include <linux/bootmem.h> /* for max_pfn */ 7#include <linux/memblock.h> /* for max_pfn */
8#include <linux/export.h> 8#include <linux/export.h>
9#include <linux/mm.h> 9#include <linux/mm.h>
10#include <linux/dma-direct.h> 10#include <linux/dma-direct.h>
diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c
index ebecaf255ea2..5731daa09a32 100644
--- a/kernel/dma/swiotlb.c
+++ b/kernel/dma/swiotlb.c
@@ -40,7 +40,7 @@
40#include <asm/dma.h> 40#include <asm/dma.h>
41 41
42#include <linux/init.h> 42#include <linux/init.h>
43#include <linux/bootmem.h> 43#include <linux/memblock.h>
44#include <linux/iommu-helper.h> 44#include <linux/iommu-helper.h>
45 45
46#define CREATE_TRACE_POINTS 46#define CREATE_TRACE_POINTS
@@ -204,10 +204,10 @@ int __init swiotlb_init_with_tbl(char *tlb, unsigned long nslabs, int verbose)
204 * to find contiguous free memory regions of size up to IO_TLB_SEGSIZE 204 * to find contiguous free memory regions of size up to IO_TLB_SEGSIZE
205 * between io_tlb_start and io_tlb_end. 205 * between io_tlb_start and io_tlb_end.
206 */ 206 */
207 io_tlb_list = memblock_virt_alloc( 207 io_tlb_list = memblock_alloc(
208 PAGE_ALIGN(io_tlb_nslabs * sizeof(int)), 208 PAGE_ALIGN(io_tlb_nslabs * sizeof(int)),
209 PAGE_SIZE); 209 PAGE_SIZE);
210 io_tlb_orig_addr = memblock_virt_alloc( 210 io_tlb_orig_addr = memblock_alloc(
211 PAGE_ALIGN(io_tlb_nslabs * sizeof(phys_addr_t)), 211 PAGE_ALIGN(io_tlb_nslabs * sizeof(phys_addr_t)),
212 PAGE_SIZE); 212 PAGE_SIZE);
213 for (i = 0; i < io_tlb_nslabs; i++) { 213 for (i = 0; i < io_tlb_nslabs; i++) {
@@ -242,7 +242,7 @@ swiotlb_init(int verbose)
242 bytes = io_tlb_nslabs << IO_TLB_SHIFT; 242 bytes = io_tlb_nslabs << IO_TLB_SHIFT;
243 243
244 /* Get IO TLB memory from the low pages */ 244 /* Get IO TLB memory from the low pages */
245 vstart = memblock_virt_alloc_low_nopanic(PAGE_ALIGN(bytes), PAGE_SIZE); 245 vstart = memblock_alloc_low_nopanic(PAGE_ALIGN(bytes), PAGE_SIZE);
246 if (vstart && !swiotlb_init_with_tbl(vstart, io_tlb_nslabs, verbose)) 246 if (vstart && !swiotlb_init_with_tbl(vstart, io_tlb_nslabs, verbose))
247 return; 247 return;
248 248
diff --git a/kernel/events/core.c b/kernel/events/core.c
index 5a97f34bc14c..8c490130c4fb 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -8376,30 +8376,39 @@ static struct pmu perf_tracepoint = {
8376 * 8376 *
8377 * PERF_PROBE_CONFIG_IS_RETPROBE if set, create kretprobe/uretprobe 8377 * PERF_PROBE_CONFIG_IS_RETPROBE if set, create kretprobe/uretprobe
8378 * if not set, create kprobe/uprobe 8378 * if not set, create kprobe/uprobe
8379 *
8380 * The following values specify a reference counter (or semaphore in the
8381 * terminology of tools like dtrace, systemtap, etc.) Userspace Statically
8382 * Defined Tracepoints (USDT). Currently, we use 40 bit for the offset.
8383 *
8384 * PERF_UPROBE_REF_CTR_OFFSET_BITS # of bits in config as th offset
8385 * PERF_UPROBE_REF_CTR_OFFSET_SHIFT # of bits to shift left
8379 */ 8386 */
8380enum perf_probe_config { 8387enum perf_probe_config {
8381 PERF_PROBE_CONFIG_IS_RETPROBE = 1U << 0, /* [k,u]retprobe */ 8388 PERF_PROBE_CONFIG_IS_RETPROBE = 1U << 0, /* [k,u]retprobe */
8389 PERF_UPROBE_REF_CTR_OFFSET_BITS = 32,
8390 PERF_UPROBE_REF_CTR_OFFSET_SHIFT = 64 - PERF_UPROBE_REF_CTR_OFFSET_BITS,
8382}; 8391};
8383 8392
8384PMU_FORMAT_ATTR(retprobe, "config:0"); 8393PMU_FORMAT_ATTR(retprobe, "config:0");
8394#endif
8385 8395
8386static struct attribute *probe_attrs[] = { 8396#ifdef CONFIG_KPROBE_EVENTS
8397static struct attribute *kprobe_attrs[] = {
8387 &format_attr_retprobe.attr, 8398 &format_attr_retprobe.attr,
8388 NULL, 8399 NULL,
8389}; 8400};
8390 8401
8391static struct attribute_group probe_format_group = { 8402static struct attribute_group kprobe_format_group = {
8392 .name = "format", 8403 .name = "format",
8393 .attrs = probe_attrs, 8404 .attrs = kprobe_attrs,
8394}; 8405};
8395 8406
8396static const struct attribute_group *probe_attr_groups[] = { 8407static const struct attribute_group *kprobe_attr_groups[] = {
8397 &probe_format_group, 8408 &kprobe_format_group,
8398 NULL, 8409 NULL,
8399}; 8410};
8400#endif
8401 8411
8402#ifdef CONFIG_KPROBE_EVENTS
8403static int perf_kprobe_event_init(struct perf_event *event); 8412static int perf_kprobe_event_init(struct perf_event *event);
8404static struct pmu perf_kprobe = { 8413static struct pmu perf_kprobe = {
8405 .task_ctx_nr = perf_sw_context, 8414 .task_ctx_nr = perf_sw_context,
@@ -8409,7 +8418,7 @@ static struct pmu perf_kprobe = {
8409 .start = perf_swevent_start, 8418 .start = perf_swevent_start,
8410 .stop = perf_swevent_stop, 8419 .stop = perf_swevent_stop,
8411 .read = perf_swevent_read, 8420 .read = perf_swevent_read,
8412 .attr_groups = probe_attr_groups, 8421 .attr_groups = kprobe_attr_groups,
8413}; 8422};
8414 8423
8415static int perf_kprobe_event_init(struct perf_event *event) 8424static int perf_kprobe_event_init(struct perf_event *event)
@@ -8441,6 +8450,24 @@ static int perf_kprobe_event_init(struct perf_event *event)
8441#endif /* CONFIG_KPROBE_EVENTS */ 8450#endif /* CONFIG_KPROBE_EVENTS */
8442 8451
8443#ifdef CONFIG_UPROBE_EVENTS 8452#ifdef CONFIG_UPROBE_EVENTS
8453PMU_FORMAT_ATTR(ref_ctr_offset, "config:32-63");
8454
8455static struct attribute *uprobe_attrs[] = {
8456 &format_attr_retprobe.attr,
8457 &format_attr_ref_ctr_offset.attr,
8458 NULL,
8459};
8460
8461static struct attribute_group uprobe_format_group = {
8462 .name = "format",
8463 .attrs = uprobe_attrs,
8464};
8465
8466static const struct attribute_group *uprobe_attr_groups[] = {
8467 &uprobe_format_group,
8468 NULL,
8469};
8470
8444static int perf_uprobe_event_init(struct perf_event *event); 8471static int perf_uprobe_event_init(struct perf_event *event);
8445static struct pmu perf_uprobe = { 8472static struct pmu perf_uprobe = {
8446 .task_ctx_nr = perf_sw_context, 8473 .task_ctx_nr = perf_sw_context,
@@ -8450,12 +8477,13 @@ static struct pmu perf_uprobe = {
8450 .start = perf_swevent_start, 8477 .start = perf_swevent_start,
8451 .stop = perf_swevent_stop, 8478 .stop = perf_swevent_stop,
8452 .read = perf_swevent_read, 8479 .read = perf_swevent_read,
8453 .attr_groups = probe_attr_groups, 8480 .attr_groups = uprobe_attr_groups,
8454}; 8481};
8455 8482
8456static int perf_uprobe_event_init(struct perf_event *event) 8483static int perf_uprobe_event_init(struct perf_event *event)
8457{ 8484{
8458 int err; 8485 int err;
8486 unsigned long ref_ctr_offset;
8459 bool is_retprobe; 8487 bool is_retprobe;
8460 8488
8461 if (event->attr.type != perf_uprobe.type) 8489 if (event->attr.type != perf_uprobe.type)
@@ -8471,7 +8499,8 @@ static int perf_uprobe_event_init(struct perf_event *event)
8471 return -EOPNOTSUPP; 8499 return -EOPNOTSUPP;
8472 8500
8473 is_retprobe = event->attr.config & PERF_PROBE_CONFIG_IS_RETPROBE; 8501 is_retprobe = event->attr.config & PERF_PROBE_CONFIG_IS_RETPROBE;
8474 err = perf_uprobe_init(event, is_retprobe); 8502 ref_ctr_offset = event->attr.config >> PERF_UPROBE_REF_CTR_OFFSET_SHIFT;
8503 err = perf_uprobe_init(event, ref_ctr_offset, is_retprobe);
8475 if (err) 8504 if (err)
8476 return err; 8505 return err;
8477 8506
diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c
index 2bf792d22087..96d4bee83489 100644
--- a/kernel/events/uprobes.c
+++ b/kernel/events/uprobes.c
@@ -73,6 +73,7 @@ struct uprobe {
73 struct uprobe_consumer *consumers; 73 struct uprobe_consumer *consumers;
74 struct inode *inode; /* Also hold a ref to inode */ 74 struct inode *inode; /* Also hold a ref to inode */
75 loff_t offset; 75 loff_t offset;
76 loff_t ref_ctr_offset;
76 unsigned long flags; 77 unsigned long flags;
77 78
78 /* 79 /*
@@ -88,6 +89,15 @@ struct uprobe {
88 struct arch_uprobe arch; 89 struct arch_uprobe arch;
89}; 90};
90 91
92struct delayed_uprobe {
93 struct list_head list;
94 struct uprobe *uprobe;
95 struct mm_struct *mm;
96};
97
98static DEFINE_MUTEX(delayed_uprobe_lock);
99static LIST_HEAD(delayed_uprobe_list);
100
91/* 101/*
92 * Execute out of line area: anonymous executable mapping installed 102 * Execute out of line area: anonymous executable mapping installed
93 * by the probed task to execute the copy of the original instruction 103 * by the probed task to execute the copy of the original instruction
@@ -282,6 +292,166 @@ static int verify_opcode(struct page *page, unsigned long vaddr, uprobe_opcode_t
282 return 1; 292 return 1;
283} 293}
284 294
295static struct delayed_uprobe *
296delayed_uprobe_check(struct uprobe *uprobe, struct mm_struct *mm)
297{
298 struct delayed_uprobe *du;
299
300 list_for_each_entry(du, &delayed_uprobe_list, list)
301 if (du->uprobe == uprobe && du->mm == mm)
302 return du;
303 return NULL;
304}
305
306static int delayed_uprobe_add(struct uprobe *uprobe, struct mm_struct *mm)
307{
308 struct delayed_uprobe *du;
309
310 if (delayed_uprobe_check(uprobe, mm))
311 return 0;
312
313 du = kzalloc(sizeof(*du), GFP_KERNEL);
314 if (!du)
315 return -ENOMEM;
316
317 du->uprobe = uprobe;
318 du->mm = mm;
319 list_add(&du->list, &delayed_uprobe_list);
320 return 0;
321}
322
323static void delayed_uprobe_delete(struct delayed_uprobe *du)
324{
325 if (WARN_ON(!du))
326 return;
327 list_del(&du->list);
328 kfree(du);
329}
330
331static void delayed_uprobe_remove(struct uprobe *uprobe, struct mm_struct *mm)
332{
333 struct list_head *pos, *q;
334 struct delayed_uprobe *du;
335
336 if (!uprobe && !mm)
337 return;
338
339 list_for_each_safe(pos, q, &delayed_uprobe_list) {
340 du = list_entry(pos, struct delayed_uprobe, list);
341
342 if (uprobe && du->uprobe != uprobe)
343 continue;
344 if (mm && du->mm != mm)
345 continue;
346
347 delayed_uprobe_delete(du);
348 }
349}
350
351static bool valid_ref_ctr_vma(struct uprobe *uprobe,
352 struct vm_area_struct *vma)
353{
354 unsigned long vaddr = offset_to_vaddr(vma, uprobe->ref_ctr_offset);
355
356 return uprobe->ref_ctr_offset &&
357 vma->vm_file &&
358 file_inode(vma->vm_file) == uprobe->inode &&
359 (vma->vm_flags & (VM_WRITE|VM_SHARED)) == VM_WRITE &&
360 vma->vm_start <= vaddr &&
361 vma->vm_end > vaddr;
362}
363
364static struct vm_area_struct *
365find_ref_ctr_vma(struct uprobe *uprobe, struct mm_struct *mm)
366{
367 struct vm_area_struct *tmp;
368
369 for (tmp = mm->mmap; tmp; tmp = tmp->vm_next)
370 if (valid_ref_ctr_vma(uprobe, tmp))
371 return tmp;
372
373 return NULL;
374}
375
376static int
377__update_ref_ctr(struct mm_struct *mm, unsigned long vaddr, short d)
378{
379 void *kaddr;
380 struct page *page;
381 struct vm_area_struct *vma;
382 int ret;
383 short *ptr;
384
385 if (!vaddr || !d)
386 return -EINVAL;
387
388 ret = get_user_pages_remote(NULL, mm, vaddr, 1,
389 FOLL_WRITE, &page, &vma, NULL);
390 if (unlikely(ret <= 0)) {
391 /*
392 * We are asking for 1 page. If get_user_pages_remote() fails,
393 * it may return 0, in that case we have to return error.
394 */
395 return ret == 0 ? -EBUSY : ret;
396 }
397
398 kaddr = kmap_atomic(page);
399 ptr = kaddr + (vaddr & ~PAGE_MASK);
400
401 if (unlikely(*ptr + d < 0)) {
402 pr_warn("ref_ctr going negative. vaddr: 0x%lx, "
403 "curr val: %d, delta: %d\n", vaddr, *ptr, d);
404 ret = -EINVAL;
405 goto out;
406 }
407
408 *ptr += d;
409 ret = 0;
410out:
411 kunmap_atomic(kaddr);
412 put_page(page);
413 return ret;
414}
415
416static void update_ref_ctr_warn(struct uprobe *uprobe,
417 struct mm_struct *mm, short d)
418{
419 pr_warn("ref_ctr %s failed for inode: 0x%lx offset: "
420 "0x%llx ref_ctr_offset: 0x%llx of mm: 0x%pK\n",
421 d > 0 ? "increment" : "decrement", uprobe->inode->i_ino,
422 (unsigned long long) uprobe->offset,
423 (unsigned long long) uprobe->ref_ctr_offset, mm);
424}
425
426static int update_ref_ctr(struct uprobe *uprobe, struct mm_struct *mm,
427 short d)
428{
429 struct vm_area_struct *rc_vma;
430 unsigned long rc_vaddr;
431 int ret = 0;
432
433 rc_vma = find_ref_ctr_vma(uprobe, mm);
434
435 if (rc_vma) {
436 rc_vaddr = offset_to_vaddr(rc_vma, uprobe->ref_ctr_offset);
437 ret = __update_ref_ctr(mm, rc_vaddr, d);
438 if (ret)
439 update_ref_ctr_warn(uprobe, mm, d);
440
441 if (d > 0)
442 return ret;
443 }
444
445 mutex_lock(&delayed_uprobe_lock);
446 if (d > 0)
447 ret = delayed_uprobe_add(uprobe, mm);
448 else
449 delayed_uprobe_remove(uprobe, mm);
450 mutex_unlock(&delayed_uprobe_lock);
451
452 return ret;
453}
454
285/* 455/*
286 * NOTE: 456 * NOTE:
287 * Expect the breakpoint instruction to be the smallest size instruction for 457 * Expect the breakpoint instruction to be the smallest size instruction for
@@ -302,9 +472,13 @@ static int verify_opcode(struct page *page, unsigned long vaddr, uprobe_opcode_t
302int uprobe_write_opcode(struct arch_uprobe *auprobe, struct mm_struct *mm, 472int uprobe_write_opcode(struct arch_uprobe *auprobe, struct mm_struct *mm,
303 unsigned long vaddr, uprobe_opcode_t opcode) 473 unsigned long vaddr, uprobe_opcode_t opcode)
304{ 474{
475 struct uprobe *uprobe;
305 struct page *old_page, *new_page; 476 struct page *old_page, *new_page;
306 struct vm_area_struct *vma; 477 struct vm_area_struct *vma;
307 int ret; 478 int ret, is_register, ref_ctr_updated = 0;
479
480 is_register = is_swbp_insn(&opcode);
481 uprobe = container_of(auprobe, struct uprobe, arch);
308 482
309retry: 483retry:
310 /* Read the page with vaddr into memory */ 484 /* Read the page with vaddr into memory */
@@ -317,6 +491,15 @@ retry:
317 if (ret <= 0) 491 if (ret <= 0)
318 goto put_old; 492 goto put_old;
319 493
494 /* We are going to replace instruction, update ref_ctr. */
495 if (!ref_ctr_updated && uprobe->ref_ctr_offset) {
496 ret = update_ref_ctr(uprobe, mm, is_register ? 1 : -1);
497 if (ret)
498 goto put_old;
499
500 ref_ctr_updated = 1;
501 }
502
320 ret = anon_vma_prepare(vma); 503 ret = anon_vma_prepare(vma);
321 if (ret) 504 if (ret)
322 goto put_old; 505 goto put_old;
@@ -337,6 +520,11 @@ put_old:
337 520
338 if (unlikely(ret == -EAGAIN)) 521 if (unlikely(ret == -EAGAIN))
339 goto retry; 522 goto retry;
523
524 /* Revert back reference counter if instruction update failed. */
525 if (ret && is_register && ref_ctr_updated)
526 update_ref_ctr(uprobe, mm, -1);
527
340 return ret; 528 return ret;
341} 529}
342 530
@@ -378,8 +566,15 @@ static struct uprobe *get_uprobe(struct uprobe *uprobe)
378 566
379static void put_uprobe(struct uprobe *uprobe) 567static void put_uprobe(struct uprobe *uprobe)
380{ 568{
381 if (atomic_dec_and_test(&uprobe->ref)) 569 if (atomic_dec_and_test(&uprobe->ref)) {
570 /*
571 * If application munmap(exec_vma) before uprobe_unregister()
572 * gets called, we don't get a chance to remove uprobe from
573 * delayed_uprobe_list from remove_breakpoint(). Do it here.
574 */
575 delayed_uprobe_remove(uprobe, NULL);
382 kfree(uprobe); 576 kfree(uprobe);
577 }
383} 578}
384 579
385static int match_uprobe(struct uprobe *l, struct uprobe *r) 580static int match_uprobe(struct uprobe *l, struct uprobe *r)
@@ -484,7 +679,18 @@ static struct uprobe *insert_uprobe(struct uprobe *uprobe)
484 return u; 679 return u;
485} 680}
486 681
487static struct uprobe *alloc_uprobe(struct inode *inode, loff_t offset) 682static void
683ref_ctr_mismatch_warn(struct uprobe *cur_uprobe, struct uprobe *uprobe)
684{
685 pr_warn("ref_ctr_offset mismatch. inode: 0x%lx offset: 0x%llx "
686 "ref_ctr_offset(old): 0x%llx ref_ctr_offset(new): 0x%llx\n",
687 uprobe->inode->i_ino, (unsigned long long) uprobe->offset,
688 (unsigned long long) cur_uprobe->ref_ctr_offset,
689 (unsigned long long) uprobe->ref_ctr_offset);
690}
691
692static struct uprobe *alloc_uprobe(struct inode *inode, loff_t offset,
693 loff_t ref_ctr_offset)
488{ 694{
489 struct uprobe *uprobe, *cur_uprobe; 695 struct uprobe *uprobe, *cur_uprobe;
490 696
@@ -494,6 +700,7 @@ static struct uprobe *alloc_uprobe(struct inode *inode, loff_t offset)
494 700
495 uprobe->inode = inode; 701 uprobe->inode = inode;
496 uprobe->offset = offset; 702 uprobe->offset = offset;
703 uprobe->ref_ctr_offset = ref_ctr_offset;
497 init_rwsem(&uprobe->register_rwsem); 704 init_rwsem(&uprobe->register_rwsem);
498 init_rwsem(&uprobe->consumer_rwsem); 705 init_rwsem(&uprobe->consumer_rwsem);
499 706
@@ -501,6 +708,12 @@ static struct uprobe *alloc_uprobe(struct inode *inode, loff_t offset)
501 cur_uprobe = insert_uprobe(uprobe); 708 cur_uprobe = insert_uprobe(uprobe);
502 /* a uprobe exists for this inode:offset combination */ 709 /* a uprobe exists for this inode:offset combination */
503 if (cur_uprobe) { 710 if (cur_uprobe) {
711 if (cur_uprobe->ref_ctr_offset != uprobe->ref_ctr_offset) {
712 ref_ctr_mismatch_warn(cur_uprobe, uprobe);
713 put_uprobe(cur_uprobe);
714 kfree(uprobe);
715 return ERR_PTR(-EINVAL);
716 }
504 kfree(uprobe); 717 kfree(uprobe);
505 uprobe = cur_uprobe; 718 uprobe = cur_uprobe;
506 } 719 }
@@ -895,7 +1108,7 @@ EXPORT_SYMBOL_GPL(uprobe_unregister);
895 * else return 0 (success) 1108 * else return 0 (success)
896 */ 1109 */
897static int __uprobe_register(struct inode *inode, loff_t offset, 1110static int __uprobe_register(struct inode *inode, loff_t offset,
898 struct uprobe_consumer *uc) 1111 loff_t ref_ctr_offset, struct uprobe_consumer *uc)
899{ 1112{
900 struct uprobe *uprobe; 1113 struct uprobe *uprobe;
901 int ret; 1114 int ret;
@@ -912,9 +1125,12 @@ static int __uprobe_register(struct inode *inode, loff_t offset,
912 return -EINVAL; 1125 return -EINVAL;
913 1126
914 retry: 1127 retry:
915 uprobe = alloc_uprobe(inode, offset); 1128 uprobe = alloc_uprobe(inode, offset, ref_ctr_offset);
916 if (!uprobe) 1129 if (!uprobe)
917 return -ENOMEM; 1130 return -ENOMEM;
1131 if (IS_ERR(uprobe))
1132 return PTR_ERR(uprobe);
1133
918 /* 1134 /*
919 * We can race with uprobe_unregister()->delete_uprobe(). 1135 * We can race with uprobe_unregister()->delete_uprobe().
920 * Check uprobe_is_active() and retry if it is false. 1136 * Check uprobe_is_active() and retry if it is false.
@@ -938,10 +1154,17 @@ static int __uprobe_register(struct inode *inode, loff_t offset,
938int uprobe_register(struct inode *inode, loff_t offset, 1154int uprobe_register(struct inode *inode, loff_t offset,
939 struct uprobe_consumer *uc) 1155 struct uprobe_consumer *uc)
940{ 1156{
941 return __uprobe_register(inode, offset, uc); 1157 return __uprobe_register(inode, offset, 0, uc);
942} 1158}
943EXPORT_SYMBOL_GPL(uprobe_register); 1159EXPORT_SYMBOL_GPL(uprobe_register);
944 1160
1161int uprobe_register_refctr(struct inode *inode, loff_t offset,
1162 loff_t ref_ctr_offset, struct uprobe_consumer *uc)
1163{
1164 return __uprobe_register(inode, offset, ref_ctr_offset, uc);
1165}
1166EXPORT_SYMBOL_GPL(uprobe_register_refctr);
1167
945/* 1168/*
946 * uprobe_apply - unregister an already registered probe. 1169 * uprobe_apply - unregister an already registered probe.
947 * @inode: the file in which the probe has to be removed. 1170 * @inode: the file in which the probe has to be removed.
@@ -1060,6 +1283,35 @@ static void build_probe_list(struct inode *inode,
1060 spin_unlock(&uprobes_treelock); 1283 spin_unlock(&uprobes_treelock);
1061} 1284}
1062 1285
1286/* @vma contains reference counter, not the probed instruction. */
1287static int delayed_ref_ctr_inc(struct vm_area_struct *vma)
1288{
1289 struct list_head *pos, *q;
1290 struct delayed_uprobe *du;
1291 unsigned long vaddr;
1292 int ret = 0, err = 0;
1293
1294 mutex_lock(&delayed_uprobe_lock);
1295 list_for_each_safe(pos, q, &delayed_uprobe_list) {
1296 du = list_entry(pos, struct delayed_uprobe, list);
1297
1298 if (du->mm != vma->vm_mm ||
1299 !valid_ref_ctr_vma(du->uprobe, vma))
1300 continue;
1301
1302 vaddr = offset_to_vaddr(vma, du->uprobe->ref_ctr_offset);
1303 ret = __update_ref_ctr(vma->vm_mm, vaddr, 1);
1304 if (ret) {
1305 update_ref_ctr_warn(du->uprobe, vma->vm_mm, 1);
1306 if (!err)
1307 err = ret;
1308 }
1309 delayed_uprobe_delete(du);
1310 }
1311 mutex_unlock(&delayed_uprobe_lock);
1312 return err;
1313}
1314
1063/* 1315/*
1064 * Called from mmap_region/vma_adjust with mm->mmap_sem acquired. 1316 * Called from mmap_region/vma_adjust with mm->mmap_sem acquired.
1065 * 1317 *
@@ -1072,7 +1324,15 @@ int uprobe_mmap(struct vm_area_struct *vma)
1072 struct uprobe *uprobe, *u; 1324 struct uprobe *uprobe, *u;
1073 struct inode *inode; 1325 struct inode *inode;
1074 1326
1075 if (no_uprobe_events() || !valid_vma(vma, true)) 1327 if (no_uprobe_events())
1328 return 0;
1329
1330 if (vma->vm_file &&
1331 (vma->vm_flags & (VM_WRITE|VM_SHARED)) == VM_WRITE &&
1332 test_bit(MMF_HAS_UPROBES, &vma->vm_mm->flags))
1333 delayed_ref_ctr_inc(vma);
1334
1335 if (!valid_vma(vma, true))
1076 return 0; 1336 return 0;
1077 1337
1078 inode = file_inode(vma->vm_file); 1338 inode = file_inode(vma->vm_file);
@@ -1246,6 +1506,10 @@ void uprobe_clear_state(struct mm_struct *mm)
1246{ 1506{
1247 struct xol_area *area = mm->uprobes_state.xol_area; 1507 struct xol_area *area = mm->uprobes_state.xol_area;
1248 1508
1509 mutex_lock(&delayed_uprobe_lock);
1510 delayed_uprobe_remove(NULL, mm);
1511 mutex_unlock(&delayed_uprobe_lock);
1512
1249 if (!area) 1513 if (!area)
1250 return; 1514 return;
1251 1515
diff --git a/kernel/fail_function.c b/kernel/fail_function.c
index bc80a4e268c0..17f75b545f66 100644
--- a/kernel/fail_function.c
+++ b/kernel/fail_function.c
@@ -173,8 +173,7 @@ static void fei_debugfs_remove_attr(struct fei_attr *attr)
173 struct dentry *dir; 173 struct dentry *dir;
174 174
175 dir = debugfs_lookup(attr->kp.symbol_name, fei_debugfs_dir); 175 dir = debugfs_lookup(attr->kp.symbol_name, fei_debugfs_dir);
176 if (dir) 176 debugfs_remove_recursive(dir);
177 debugfs_remove_recursive(dir);
178} 177}
179 178
180static int fei_kprobe_handler(struct kprobe *kp, struct pt_regs *regs) 179static int fei_kprobe_handler(struct kprobe *kp, struct pt_regs *regs)
diff --git a/kernel/futex.c b/kernel/futex.c
index 3e2de8fc1891..f423f9b6577e 100644
--- a/kernel/futex.c
+++ b/kernel/futex.c
@@ -65,7 +65,7 @@
65#include <linux/sched/mm.h> 65#include <linux/sched/mm.h>
66#include <linux/hugetlb.h> 66#include <linux/hugetlb.h>
67#include <linux/freezer.h> 67#include <linux/freezer.h>
68#include <linux/bootmem.h> 68#include <linux/memblock.h>
69#include <linux/fault-inject.h> 69#include <linux/fault-inject.h>
70 70
71#include <asm/futex.h> 71#include <asm/futex.h>
diff --git a/kernel/hung_task.c b/kernel/hung_task.c
index b9132d1269ef..cb8e3e8ac7b9 100644
--- a/kernel/hung_task.c
+++ b/kernel/hung_task.c
@@ -15,6 +15,7 @@
15#include <linux/lockdep.h> 15#include <linux/lockdep.h>
16#include <linux/export.h> 16#include <linux/export.h>
17#include <linux/sysctl.h> 17#include <linux/sysctl.h>
18#include <linux/suspend.h>
18#include <linux/utsname.h> 19#include <linux/utsname.h>
19#include <linux/sched/signal.h> 20#include <linux/sched/signal.h>
20#include <linux/sched/debug.h> 21#include <linux/sched/debug.h>
@@ -242,6 +243,28 @@ void reset_hung_task_detector(void)
242} 243}
243EXPORT_SYMBOL_GPL(reset_hung_task_detector); 244EXPORT_SYMBOL_GPL(reset_hung_task_detector);
244 245
246static bool hung_detector_suspended;
247
248static int hungtask_pm_notify(struct notifier_block *self,
249 unsigned long action, void *hcpu)
250{
251 switch (action) {
252 case PM_SUSPEND_PREPARE:
253 case PM_HIBERNATION_PREPARE:
254 case PM_RESTORE_PREPARE:
255 hung_detector_suspended = true;
256 break;
257 case PM_POST_SUSPEND:
258 case PM_POST_HIBERNATION:
259 case PM_POST_RESTORE:
260 hung_detector_suspended = false;
261 break;
262 default:
263 break;
264 }
265 return NOTIFY_OK;
266}
267
245/* 268/*
246 * kthread which checks for tasks stuck in D state 269 * kthread which checks for tasks stuck in D state
247 */ 270 */
@@ -261,7 +284,8 @@ static int watchdog(void *dummy)
261 interval = min_t(unsigned long, interval, timeout); 284 interval = min_t(unsigned long, interval, timeout);
262 t = hung_timeout_jiffies(hung_last_checked, interval); 285 t = hung_timeout_jiffies(hung_last_checked, interval);
263 if (t <= 0) { 286 if (t <= 0) {
264 if (!atomic_xchg(&reset_hung_task, 0)) 287 if (!atomic_xchg(&reset_hung_task, 0) &&
288 !hung_detector_suspended)
265 check_hung_uninterruptible_tasks(timeout); 289 check_hung_uninterruptible_tasks(timeout);
266 hung_last_checked = jiffies; 290 hung_last_checked = jiffies;
267 continue; 291 continue;
@@ -275,6 +299,10 @@ static int watchdog(void *dummy)
275static int __init hung_task_init(void) 299static int __init hung_task_init(void)
276{ 300{
277 atomic_notifier_chain_register(&panic_notifier_list, &panic_block); 301 atomic_notifier_chain_register(&panic_notifier_list, &panic_block);
302
303 /* Disable hung task detector on suspend */
304 pm_notifier(hungtask_pm_notify, 0);
305
278 watchdog_task = kthread_run(watchdog, NULL, "khungtaskd"); 306 watchdog_task = kthread_run(watchdog, NULL, "khungtaskd");
279 307
280 return 0; 308 return 0;
diff --git a/kernel/kallsyms.c b/kernel/kallsyms.c
index 02a0b01380d8..f3a04994e063 100644
--- a/kernel/kallsyms.c
+++ b/kernel/kallsyms.c
@@ -37,7 +37,7 @@ extern const u8 kallsyms_names[] __weak;
37 * Tell the compiler that the count isn't in the small data section if the arch 37 * Tell the compiler that the count isn't in the small data section if the arch
38 * has one (eg: FRV). 38 * has one (eg: FRV).
39 */ 39 */
40extern const unsigned long kallsyms_num_syms 40extern const unsigned int kallsyms_num_syms
41__attribute__((weak, section(".rodata"))); 41__attribute__((weak, section(".rodata")));
42 42
43extern const unsigned long kallsyms_relative_base 43extern const unsigned long kallsyms_relative_base
@@ -46,7 +46,7 @@ __attribute__((weak, section(".rodata")));
46extern const u8 kallsyms_token_table[] __weak; 46extern const u8 kallsyms_token_table[] __weak;
47extern const u16 kallsyms_token_index[] __weak; 47extern const u16 kallsyms_token_index[] __weak;
48 48
49extern const unsigned long kallsyms_markers[] __weak; 49extern const unsigned int kallsyms_markers[] __weak;
50 50
51/* 51/*
52 * Expand a compressed symbol data into the resulting uncompressed string, 52 * Expand a compressed symbol data into the resulting uncompressed string,
diff --git a/kernel/locking/qspinlock_paravirt.h b/kernel/locking/qspinlock_paravirt.h
index 0130e488ebfe..8f36c27c1794 100644
--- a/kernel/locking/qspinlock_paravirt.h
+++ b/kernel/locking/qspinlock_paravirt.h
@@ -4,7 +4,7 @@
4#endif 4#endif
5 5
6#include <linux/hash.h> 6#include <linux/hash.h>
7#include <linux/bootmem.h> 7#include <linux/memblock.h>
8#include <linux/debug_locks.h> 8#include <linux/debug_locks.h>
9 9
10/* 10/*
diff --git a/kernel/memremap.c b/kernel/memremap.c
index 620fc4d2559a..9eced2cc9f94 100644
--- a/kernel/memremap.c
+++ b/kernel/memremap.c
@@ -1,47 +1,21 @@
1/* SPDX-License-Identifier: GPL-2.0 */ 1/* SPDX-License-Identifier: GPL-2.0 */
2/* Copyright(c) 2015 Intel Corporation. All rights reserved. */ 2/* Copyright(c) 2015 Intel Corporation. All rights reserved. */
3#include <linux/radix-tree.h>
4#include <linux/device.h> 3#include <linux/device.h>
5#include <linux/types.h>
6#include <linux/pfn_t.h>
7#include <linux/io.h> 4#include <linux/io.h>
8#include <linux/kasan.h> 5#include <linux/kasan.h>
9#include <linux/mm.h>
10#include <linux/memory_hotplug.h> 6#include <linux/memory_hotplug.h>
7#include <linux/mm.h>
8#include <linux/pfn_t.h>
11#include <linux/swap.h> 9#include <linux/swap.h>
12#include <linux/swapops.h> 10#include <linux/swapops.h>
11#include <linux/types.h>
13#include <linux/wait_bit.h> 12#include <linux/wait_bit.h>
13#include <linux/xarray.h>
14 14
15static DEFINE_MUTEX(pgmap_lock); 15static DEFINE_XARRAY(pgmap_array);
16static RADIX_TREE(pgmap_radix, GFP_KERNEL);
17#define SECTION_MASK ~((1UL << PA_SECTION_SHIFT) - 1) 16#define SECTION_MASK ~((1UL << PA_SECTION_SHIFT) - 1)
18#define SECTION_SIZE (1UL << PA_SECTION_SHIFT) 17#define SECTION_SIZE (1UL << PA_SECTION_SHIFT)
19 18
20static unsigned long order_at(struct resource *res, unsigned long pgoff)
21{
22 unsigned long phys_pgoff = PHYS_PFN(res->start) + pgoff;
23 unsigned long nr_pages, mask;
24
25 nr_pages = PHYS_PFN(resource_size(res));
26 if (nr_pages == pgoff)
27 return ULONG_MAX;
28
29 /*
30 * What is the largest aligned power-of-2 range available from
31 * this resource pgoff to the end of the resource range,
32 * considering the alignment of the current pgoff?
33 */
34 mask = phys_pgoff | rounddown_pow_of_two(nr_pages - pgoff);
35 if (!mask)
36 return ULONG_MAX;
37
38 return find_first_bit(&mask, BITS_PER_LONG);
39}
40
41#define foreach_order_pgoff(res, order, pgoff) \
42 for (pgoff = 0, order = order_at((res), pgoff); order < ULONG_MAX; \
43 pgoff += 1UL << order, order = order_at((res), pgoff))
44
45#if IS_ENABLED(CONFIG_DEVICE_PRIVATE) 19#if IS_ENABLED(CONFIG_DEVICE_PRIVATE)
46vm_fault_t device_private_entry_fault(struct vm_area_struct *vma, 20vm_fault_t device_private_entry_fault(struct vm_area_struct *vma,
47 unsigned long addr, 21 unsigned long addr,
@@ -70,18 +44,10 @@ vm_fault_t device_private_entry_fault(struct vm_area_struct *vma,
70EXPORT_SYMBOL(device_private_entry_fault); 44EXPORT_SYMBOL(device_private_entry_fault);
71#endif /* CONFIG_DEVICE_PRIVATE */ 45#endif /* CONFIG_DEVICE_PRIVATE */
72 46
73static void pgmap_radix_release(struct resource *res, unsigned long end_pgoff) 47static void pgmap_array_delete(struct resource *res)
74{ 48{
75 unsigned long pgoff, order; 49 xa_store_range(&pgmap_array, PHYS_PFN(res->start), PHYS_PFN(res->end),
76 50 NULL, GFP_KERNEL);
77 mutex_lock(&pgmap_lock);
78 foreach_order_pgoff(res, order, pgoff) {
79 if (pgoff >= end_pgoff)
80 break;
81 radix_tree_delete(&pgmap_radix, PHYS_PFN(res->start) + pgoff);
82 }
83 mutex_unlock(&pgmap_lock);
84
85 synchronize_rcu(); 51 synchronize_rcu();
86} 52}
87 53
@@ -142,7 +108,7 @@ static void devm_memremap_pages_release(void *data)
142 mem_hotplug_done(); 108 mem_hotplug_done();
143 109
144 untrack_pfn(NULL, PHYS_PFN(align_start), align_size); 110 untrack_pfn(NULL, PHYS_PFN(align_start), align_size);
145 pgmap_radix_release(res, -1); 111 pgmap_array_delete(res);
146 dev_WARN_ONCE(dev, pgmap->altmap.alloc, 112 dev_WARN_ONCE(dev, pgmap->altmap.alloc,
147 "%s: failed to free all reserved pages\n", __func__); 113 "%s: failed to free all reserved pages\n", __func__);
148} 114}
@@ -177,7 +143,6 @@ void *devm_memremap_pages(struct device *dev, struct dev_pagemap *pgmap)
177 struct resource *res = &pgmap->res; 143 struct resource *res = &pgmap->res;
178 struct dev_pagemap *conflict_pgmap; 144 struct dev_pagemap *conflict_pgmap;
179 pgprot_t pgprot = PAGE_KERNEL; 145 pgprot_t pgprot = PAGE_KERNEL;
180 unsigned long pgoff, order;
181 int error, nid, is_ram; 146 int error, nid, is_ram;
182 147
183 align_start = res->start & ~(SECTION_SIZE - 1); 148 align_start = res->start & ~(SECTION_SIZE - 1);
@@ -216,20 +181,10 @@ void *devm_memremap_pages(struct device *dev, struct dev_pagemap *pgmap)
216 181
217 pgmap->dev = dev; 182 pgmap->dev = dev;
218 183
219 mutex_lock(&pgmap_lock); 184 error = xa_err(xa_store_range(&pgmap_array, PHYS_PFN(res->start),
220 error = 0; 185 PHYS_PFN(res->end), pgmap, GFP_KERNEL));
221
222 foreach_order_pgoff(res, order, pgoff) {
223 error = __radix_tree_insert(&pgmap_radix,
224 PHYS_PFN(res->start) + pgoff, order, pgmap);
225 if (error) {
226 dev_err(dev, "%s: failed: %d\n", __func__, error);
227 break;
228 }
229 }
230 mutex_unlock(&pgmap_lock);
231 if (error) 186 if (error)
232 goto err_radix; 187 goto err_array;
233 188
234 nid = dev_to_node(dev); 189 nid = dev_to_node(dev);
235 if (nid < 0) 190 if (nid < 0)
@@ -274,8 +229,8 @@ void *devm_memremap_pages(struct device *dev, struct dev_pagemap *pgmap)
274 err_kasan: 229 err_kasan:
275 untrack_pfn(NULL, PHYS_PFN(align_start), align_size); 230 untrack_pfn(NULL, PHYS_PFN(align_start), align_size);
276 err_pfn_remap: 231 err_pfn_remap:
277 err_radix: 232 pgmap_array_delete(res);
278 pgmap_radix_release(res, pgoff); 233 err_array:
279 return ERR_PTR(error); 234 return ERR_PTR(error);
280} 235}
281EXPORT_SYMBOL(devm_memremap_pages); 236EXPORT_SYMBOL(devm_memremap_pages);
@@ -315,7 +270,7 @@ struct dev_pagemap *get_dev_pagemap(unsigned long pfn,
315 270
316 /* fall back to slow path lookup */ 271 /* fall back to slow path lookup */
317 rcu_read_lock(); 272 rcu_read_lock();
318 pgmap = radix_tree_lookup(&pgmap_radix, PHYS_PFN(phys)); 273 pgmap = xa_load(&pgmap_array, PHYS_PFN(phys));
319 if (pgmap && !percpu_ref_tryget_live(pgmap->ref)) 274 if (pgmap && !percpu_ref_tryget_live(pgmap->ref))
320 pgmap = NULL; 275 pgmap = NULL;
321 rcu_read_unlock(); 276 rcu_read_unlock();
diff --git a/kernel/panic.c b/kernel/panic.c
index 8b2e002d52eb..f6d549a29a5c 100644
--- a/kernel/panic.c
+++ b/kernel/panic.c
@@ -136,7 +136,7 @@ void panic(const char *fmt, ...)
136{ 136{
137 static char buf[1024]; 137 static char buf[1024];
138 va_list args; 138 va_list args;
139 long i, i_next = 0; 139 long i, i_next = 0, len;
140 int state = 0; 140 int state = 0;
141 int old_cpu, this_cpu; 141 int old_cpu, this_cpu;
142 bool _crash_kexec_post_notifiers = crash_kexec_post_notifiers; 142 bool _crash_kexec_post_notifiers = crash_kexec_post_notifiers;
@@ -173,8 +173,12 @@ void panic(const char *fmt, ...)
173 console_verbose(); 173 console_verbose();
174 bust_spinlocks(1); 174 bust_spinlocks(1);
175 va_start(args, fmt); 175 va_start(args, fmt);
176 vsnprintf(buf, sizeof(buf), fmt, args); 176 len = vscnprintf(buf, sizeof(buf), fmt, args);
177 va_end(args); 177 va_end(args);
178
179 if (len && buf[len - 1] == '\n')
180 buf[len - 1] = '\0';
181
178 pr_emerg("Kernel panic - not syncing: %s\n", buf); 182 pr_emerg("Kernel panic - not syncing: %s\n", buf);
179#ifdef CONFIG_DEBUG_BUGVERBOSE 183#ifdef CONFIG_DEBUG_BUGVERBOSE
180 /* 184 /*
@@ -631,7 +635,7 @@ device_initcall(register_warn_debugfs);
631 */ 635 */
632__visible void __stack_chk_fail(void) 636__visible void __stack_chk_fail(void)
633{ 637{
634 panic("stack-protector: Kernel stack is corrupted in: %pB\n", 638 panic("stack-protector: Kernel stack is corrupted in: %pB",
635 __builtin_return_address(0)); 639 __builtin_return_address(0));
636} 640}
637EXPORT_SYMBOL(__stack_chk_fail); 641EXPORT_SYMBOL(__stack_chk_fail);
diff --git a/kernel/pid.c b/kernel/pid.c
index cdf63e53a014..b2f6c506035d 100644
--- a/kernel/pid.c
+++ b/kernel/pid.c
@@ -31,7 +31,7 @@
31#include <linux/slab.h> 31#include <linux/slab.h>
32#include <linux/init.h> 32#include <linux/init.h>
33#include <linux/rculist.h> 33#include <linux/rculist.h>
34#include <linux/bootmem.h> 34#include <linux/memblock.h>
35#include <linux/hash.h> 35#include <linux/hash.h>
36#include <linux/pid_namespace.h> 36#include <linux/pid_namespace.h>
37#include <linux/init_task.h> 37#include <linux/init_task.h>
diff --git a/kernel/power/snapshot.c b/kernel/power/snapshot.c
index 3d37c279c090..b0308a2c6000 100644
--- a/kernel/power/snapshot.c
+++ b/kernel/power/snapshot.c
@@ -23,7 +23,7 @@
23#include <linux/pm.h> 23#include <linux/pm.h>
24#include <linux/device.h> 24#include <linux/device.h>
25#include <linux/init.h> 25#include <linux/init.h>
26#include <linux/bootmem.h> 26#include <linux/memblock.h>
27#include <linux/nmi.h> 27#include <linux/nmi.h>
28#include <linux/syscalls.h> 28#include <linux/syscalls.h>
29#include <linux/console.h> 29#include <linux/console.h>
@@ -963,7 +963,8 @@ void __init __register_nosave_region(unsigned long start_pfn,
963 BUG_ON(!region); 963 BUG_ON(!region);
964 } else { 964 } else {
965 /* This allocation cannot fail */ 965 /* This allocation cannot fail */
966 region = memblock_virt_alloc(sizeof(struct nosave_region), 0); 966 region = memblock_alloc(sizeof(struct nosave_region),
967 SMP_CACHE_BYTES);
967 } 968 }
968 region->start_pfn = start_pfn; 969 region->start_pfn = start_pfn;
969 region->end_pfn = end_pfn; 970 region->end_pfn = end_pfn;
diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c
index b77150ad1965..1b2a029360b7 100644
--- a/kernel/printk/printk.c
+++ b/kernel/printk/printk.c
@@ -31,7 +31,6 @@
31#include <linux/delay.h> 31#include <linux/delay.h>
32#include <linux/smp.h> 32#include <linux/smp.h>
33#include <linux/security.h> 33#include <linux/security.h>
34#include <linux/bootmem.h>
35#include <linux/memblock.h> 34#include <linux/memblock.h>
36#include <linux/syscalls.h> 35#include <linux/syscalls.h>
37#include <linux/crash_core.h> 36#include <linux/crash_core.h>
@@ -1111,9 +1110,9 @@ void __init setup_log_buf(int early)
1111 1110
1112 if (early) { 1111 if (early) {
1113 new_log_buf = 1112 new_log_buf =
1114 memblock_virt_alloc(new_log_buf_len, LOG_ALIGN); 1113 memblock_alloc(new_log_buf_len, LOG_ALIGN);
1115 } else { 1114 } else {
1116 new_log_buf = memblock_virt_alloc_nopanic(new_log_buf_len, 1115 new_log_buf = memblock_alloc_nopanic(new_log_buf_len,
1117 LOG_ALIGN); 1116 LOG_ALIGN);
1118 } 1117 }
1119 1118
diff --git a/kernel/profile.c b/kernel/profile.c
index 9aa2a4445b0d..9c08a2c7cb1d 100644
--- a/kernel/profile.c
+++ b/kernel/profile.c
@@ -16,7 +16,7 @@
16 16
17#include <linux/export.h> 17#include <linux/export.h>
18#include <linux/profile.h> 18#include <linux/profile.h>
19#include <linux/bootmem.h> 19#include <linux/memblock.h>
20#include <linux/notifier.h> 20#include <linux/notifier.h>
21#include <linux/mm.h> 21#include <linux/mm.h>
22#include <linux/cpumask.h> 22#include <linux/cpumask.h>
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index fd2fce8a001b..f12225f26b70 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -2881,6 +2881,18 @@ unsigned long long nr_context_switches(void)
2881} 2881}
2882 2882
2883/* 2883/*
2884 * Consumers of these two interfaces, like for example the cpuidle menu
2885 * governor, are using nonsensical data. Preferring shallow idle state selection
2886 * for a CPU that has IO-wait which might not even end up running the task when
2887 * it does become runnable.
2888 */
2889
2890unsigned long nr_iowait_cpu(int cpu)
2891{
2892 return atomic_read(&cpu_rq(cpu)->nr_iowait);
2893}
2894
2895/*
2884 * IO-wait accounting, and how its mostly bollocks (on SMP). 2896 * IO-wait accounting, and how its mostly bollocks (on SMP).
2885 * 2897 *
2886 * The idea behind IO-wait account is to account the idle time that we could 2898 * The idea behind IO-wait account is to account the idle time that we could
@@ -2915,31 +2927,11 @@ unsigned long nr_iowait(void)
2915 unsigned long i, sum = 0; 2927 unsigned long i, sum = 0;
2916 2928
2917 for_each_possible_cpu(i) 2929 for_each_possible_cpu(i)
2918 sum += atomic_read(&cpu_rq(i)->nr_iowait); 2930 sum += nr_iowait_cpu(i);
2919 2931
2920 return sum; 2932 return sum;
2921} 2933}
2922 2934
2923/*
2924 * Consumers of these two interfaces, like for example the cpuidle menu
2925 * governor, are using nonsensical data. Preferring shallow idle state selection
2926 * for a CPU that has IO-wait which might not even end up running the task when
2927 * it does become runnable.
2928 */
2929
2930unsigned long nr_iowait_cpu(int cpu)
2931{
2932 struct rq *this = cpu_rq(cpu);
2933 return atomic_read(&this->nr_iowait);
2934}
2935
2936void get_iowait_load(unsigned long *nr_waiters, unsigned long *load)
2937{
2938 struct rq *rq = this_rq();
2939 *nr_waiters = atomic_read(&rq->nr_iowait);
2940 *load = rq->load.weight;
2941}
2942
2943#ifdef CONFIG_SMP 2935#ifdef CONFIG_SMP
2944 2936
2945/* 2937/*
diff --git a/kernel/signal.c b/kernel/signal.c
index 17565240b1c6..9a32bc2088c9 100644
--- a/kernel/signal.c
+++ b/kernel/signal.c
@@ -892,7 +892,7 @@ static bool prepare_signal(int sig, struct task_struct *p, bool force)
892 /* 892 /*
893 * The first thread which returns from do_signal_stop() 893 * The first thread which returns from do_signal_stop()
894 * will take ->siglock, notice SIGNAL_CLD_MASK, and 894 * will take ->siglock, notice SIGNAL_CLD_MASK, and
895 * notify its parent. See get_signal_to_deliver(). 895 * notify its parent. See get_signal().
896 */ 896 */
897 signal_set_stop_flags(signal, why | SIGNAL_STOP_CONTINUED); 897 signal_set_stop_flags(signal, why | SIGNAL_STOP_CONTINUED);
898 signal->group_stop_count = 0; 898 signal->group_stop_count = 0;
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index bf6f1d70484d..ff1c4b20cd0a 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -2727,6 +2727,7 @@ void trace_dump_stack(int skip)
2727 __ftrace_trace_stack(global_trace.trace_buffer.buffer, 2727 __ftrace_trace_stack(global_trace.trace_buffer.buffer,
2728 flags, skip, preempt_count(), NULL); 2728 flags, skip, preempt_count(), NULL);
2729} 2729}
2730EXPORT_SYMBOL_GPL(trace_dump_stack);
2730 2731
2731static DEFINE_PER_CPU(int, user_stack_count); 2732static DEFINE_PER_CPU(int, user_stack_count);
2732 2733
@@ -4621,13 +4622,18 @@ static const char readme_msg[] =
4621 "place (kretprobe): [<module>:]<symbol>[+<offset>]|<memaddr>\n" 4622 "place (kretprobe): [<module>:]<symbol>[+<offset>]|<memaddr>\n"
4622#endif 4623#endif
4623#ifdef CONFIG_UPROBE_EVENTS 4624#ifdef CONFIG_UPROBE_EVENTS
4624 "\t place: <path>:<offset>\n" 4625 " place (uprobe): <path>:<offset>[(ref_ctr_offset)]\n"
4625#endif 4626#endif
4626 "\t args: <name>=fetcharg[:type]\n" 4627 "\t args: <name>=fetcharg[:type]\n"
4627 "\t fetcharg: %<register>, @<address>, @<symbol>[+|-<offset>],\n" 4628 "\t fetcharg: %<register>, @<address>, @<symbol>[+|-<offset>],\n"
4629#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
4630 "\t $stack<index>, $stack, $retval, $comm, $arg<N>\n"
4631#else
4628 "\t $stack<index>, $stack, $retval, $comm\n" 4632 "\t $stack<index>, $stack, $retval, $comm\n"
4629 "\t type: s8/16/32/64, u8/16/32/64, x8/16/32/64, string,\n" 4633#endif
4630 "\t b<bit-width>@<bit-offset>/<container-size>\n" 4634 "\t type: s8/16/32/64, u8/16/32/64, x8/16/32/64, string, symbol,\n"
4635 "\t b<bit-width>@<bit-offset>/<container-size>,\n"
4636 "\t <type>\\[<array-size>\\]\n"
4631#endif 4637#endif
4632 " events/\t\t- Directory containing all trace event subsystems:\n" 4638 " events/\t\t- Directory containing all trace event subsystems:\n"
4633 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n" 4639 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
diff --git a/kernel/trace/trace_event_perf.c b/kernel/trace/trace_event_perf.c
index 69a3fe926e8c..76217bbef815 100644
--- a/kernel/trace/trace_event_perf.c
+++ b/kernel/trace/trace_event_perf.c
@@ -290,7 +290,8 @@ void perf_kprobe_destroy(struct perf_event *p_event)
290#endif /* CONFIG_KPROBE_EVENTS */ 290#endif /* CONFIG_KPROBE_EVENTS */
291 291
292#ifdef CONFIG_UPROBE_EVENTS 292#ifdef CONFIG_UPROBE_EVENTS
293int perf_uprobe_init(struct perf_event *p_event, bool is_retprobe) 293int perf_uprobe_init(struct perf_event *p_event,
294 unsigned long ref_ctr_offset, bool is_retprobe)
294{ 295{
295 int ret; 296 int ret;
296 char *path = NULL; 297 char *path = NULL;
@@ -312,8 +313,8 @@ int perf_uprobe_init(struct perf_event *p_event, bool is_retprobe)
312 goto out; 313 goto out;
313 } 314 }
314 315
315 tp_event = create_local_trace_uprobe( 316 tp_event = create_local_trace_uprobe(path, p_event->attr.probe_offset,
316 path, p_event->attr.probe_offset, is_retprobe); 317 ref_ctr_offset, is_retprobe);
317 if (IS_ERR(tp_event)) { 318 if (IS_ERR(tp_event)) {
318 ret = PTR_ERR(tp_event); 319 ret = PTR_ERR(tp_event);
319 goto out; 320 goto out;
diff --git a/kernel/trace/trace_events_hist.c b/kernel/trace/trace_events_hist.c
index d239004aaf29..eb908ef2ecec 100644
--- a/kernel/trace/trace_events_hist.c
+++ b/kernel/trace/trace_events_hist.c
@@ -1063,8 +1063,10 @@ static int create_synth_event(int argc, char **argv)
1063 event = NULL; 1063 event = NULL;
1064 ret = -EEXIST; 1064 ret = -EEXIST;
1065 goto out; 1065 goto out;
1066 } else if (delete_event) 1066 } else if (delete_event) {
1067 ret = -ENOENT;
1067 goto out; 1068 goto out;
1069 }
1068 1070
1069 if (argc < 2) { 1071 if (argc < 2) {
1070 ret = -EINVAL; 1072 ret = -EINVAL;
diff --git a/kernel/trace/trace_kprobe.c b/kernel/trace/trace_kprobe.c
index c30032367aab..fec67188c4d2 100644
--- a/kernel/trace/trace_kprobe.c
+++ b/kernel/trace/trace_kprobe.c
@@ -14,6 +14,7 @@
14 14
15#include "trace_kprobe_selftest.h" 15#include "trace_kprobe_selftest.h"
16#include "trace_probe.h" 16#include "trace_probe.h"
17#include "trace_probe_tmpl.h"
17 18
18#define KPROBE_EVENT_SYSTEM "kprobes" 19#define KPROBE_EVENT_SYSTEM "kprobes"
19#define KRETPROBE_MAXACTIVE_MAX 4096 20#define KRETPROBE_MAXACTIVE_MAX 4096
@@ -61,9 +62,23 @@ static nokprobe_inline bool trace_kprobe_within_module(struct trace_kprobe *tk,
61 return strncmp(mod->name, name, len) == 0 && name[len] == ':'; 62 return strncmp(mod->name, name, len) == 0 && name[len] == ':';
62} 63}
63 64
64static nokprobe_inline bool trace_kprobe_is_on_module(struct trace_kprobe *tk) 65static nokprobe_inline bool trace_kprobe_module_exist(struct trace_kprobe *tk)
65{ 66{
66 return !!strchr(trace_kprobe_symbol(tk), ':'); 67 char *p;
68 bool ret;
69
70 if (!tk->symbol)
71 return false;
72 p = strchr(tk->symbol, ':');
73 if (!p)
74 return true;
75 *p = '\0';
76 mutex_lock(&module_mutex);
77 ret = !!find_module(tk->symbol);
78 mutex_unlock(&module_mutex);
79 *p = ':';
80
81 return ret;
67} 82}
68 83
69static nokprobe_inline unsigned long trace_kprobe_nhit(struct trace_kprobe *tk) 84static nokprobe_inline unsigned long trace_kprobe_nhit(struct trace_kprobe *tk)
@@ -120,184 +135,6 @@ static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs);
120static int kretprobe_dispatcher(struct kretprobe_instance *ri, 135static int kretprobe_dispatcher(struct kretprobe_instance *ri,
121 struct pt_regs *regs); 136 struct pt_regs *regs);
122 137
123/* Memory fetching by symbol */
124struct symbol_cache {
125 char *symbol;
126 long offset;
127 unsigned long addr;
128};
129
130unsigned long update_symbol_cache(struct symbol_cache *sc)
131{
132 sc->addr = (unsigned long)kallsyms_lookup_name(sc->symbol);
133
134 if (sc->addr)
135 sc->addr += sc->offset;
136
137 return sc->addr;
138}
139
140void free_symbol_cache(struct symbol_cache *sc)
141{
142 kfree(sc->symbol);
143 kfree(sc);
144}
145
146struct symbol_cache *alloc_symbol_cache(const char *sym, long offset)
147{
148 struct symbol_cache *sc;
149
150 if (!sym || strlen(sym) == 0)
151 return NULL;
152
153 sc = kzalloc(sizeof(struct symbol_cache), GFP_KERNEL);
154 if (!sc)
155 return NULL;
156
157 sc->symbol = kstrdup(sym, GFP_KERNEL);
158 if (!sc->symbol) {
159 kfree(sc);
160 return NULL;
161 }
162 sc->offset = offset;
163 update_symbol_cache(sc);
164
165 return sc;
166}
167
168/*
169 * Kprobes-specific fetch functions
170 */
171#define DEFINE_FETCH_stack(type) \
172static void FETCH_FUNC_NAME(stack, type)(struct pt_regs *regs, \
173 void *offset, void *dest) \
174{ \
175 *(type *)dest = (type)regs_get_kernel_stack_nth(regs, \
176 (unsigned int)((unsigned long)offset)); \
177} \
178NOKPROBE_SYMBOL(FETCH_FUNC_NAME(stack, type));
179
180DEFINE_BASIC_FETCH_FUNCS(stack)
181/* No string on the stack entry */
182#define fetch_stack_string NULL
183#define fetch_stack_string_size NULL
184
185#define DEFINE_FETCH_memory(type) \
186static void FETCH_FUNC_NAME(memory, type)(struct pt_regs *regs, \
187 void *addr, void *dest) \
188{ \
189 type retval; \
190 if (probe_kernel_address(addr, retval)) \
191 *(type *)dest = 0; \
192 else \
193 *(type *)dest = retval; \
194} \
195NOKPROBE_SYMBOL(FETCH_FUNC_NAME(memory, type));
196
197DEFINE_BASIC_FETCH_FUNCS(memory)
198/*
199 * Fetch a null-terminated string. Caller MUST set *(u32 *)dest with max
200 * length and relative data location.
201 */
202static void FETCH_FUNC_NAME(memory, string)(struct pt_regs *regs,
203 void *addr, void *dest)
204{
205 int maxlen = get_rloc_len(*(u32 *)dest);
206 u8 *dst = get_rloc_data(dest);
207 long ret;
208
209 if (!maxlen)
210 return;
211
212 /*
213 * Try to get string again, since the string can be changed while
214 * probing.
215 */
216 ret = strncpy_from_unsafe(dst, addr, maxlen);
217
218 if (ret < 0) { /* Failed to fetch string */
219 dst[0] = '\0';
220 *(u32 *)dest = make_data_rloc(0, get_rloc_offs(*(u32 *)dest));
221 } else {
222 *(u32 *)dest = make_data_rloc(ret, get_rloc_offs(*(u32 *)dest));
223 }
224}
225NOKPROBE_SYMBOL(FETCH_FUNC_NAME(memory, string));
226
227/* Return the length of string -- including null terminal byte */
228static void FETCH_FUNC_NAME(memory, string_size)(struct pt_regs *regs,
229 void *addr, void *dest)
230{
231 mm_segment_t old_fs;
232 int ret, len = 0;
233 u8 c;
234
235 old_fs = get_fs();
236 set_fs(KERNEL_DS);
237 pagefault_disable();
238
239 do {
240 ret = __copy_from_user_inatomic(&c, (u8 *)addr + len, 1);
241 len++;
242 } while (c && ret == 0 && len < MAX_STRING_SIZE);
243
244 pagefault_enable();
245 set_fs(old_fs);
246
247 if (ret < 0) /* Failed to check the length */
248 *(u32 *)dest = 0;
249 else
250 *(u32 *)dest = len;
251}
252NOKPROBE_SYMBOL(FETCH_FUNC_NAME(memory, string_size));
253
254#define DEFINE_FETCH_symbol(type) \
255void FETCH_FUNC_NAME(symbol, type)(struct pt_regs *regs, void *data, void *dest)\
256{ \
257 struct symbol_cache *sc = data; \
258 if (sc->addr) \
259 fetch_memory_##type(regs, (void *)sc->addr, dest); \
260 else \
261 *(type *)dest = 0; \
262} \
263NOKPROBE_SYMBOL(FETCH_FUNC_NAME(symbol, type));
264
265DEFINE_BASIC_FETCH_FUNCS(symbol)
266DEFINE_FETCH_symbol(string)
267DEFINE_FETCH_symbol(string_size)
268
269/* kprobes don't support file_offset fetch methods */
270#define fetch_file_offset_u8 NULL
271#define fetch_file_offset_u16 NULL
272#define fetch_file_offset_u32 NULL
273#define fetch_file_offset_u64 NULL
274#define fetch_file_offset_string NULL
275#define fetch_file_offset_string_size NULL
276
277/* Fetch type information table */
278static const struct fetch_type kprobes_fetch_type_table[] = {
279 /* Special types */
280 [FETCH_TYPE_STRING] = __ASSIGN_FETCH_TYPE("string", string, string,
281 sizeof(u32), 1, "__data_loc char[]"),
282 [FETCH_TYPE_STRSIZE] = __ASSIGN_FETCH_TYPE("string_size", u32,
283 string_size, sizeof(u32), 0, "u32"),
284 /* Basic types */
285 ASSIGN_FETCH_TYPE(u8, u8, 0),
286 ASSIGN_FETCH_TYPE(u16, u16, 0),
287 ASSIGN_FETCH_TYPE(u32, u32, 0),
288 ASSIGN_FETCH_TYPE(u64, u64, 0),
289 ASSIGN_FETCH_TYPE(s8, u8, 1),
290 ASSIGN_FETCH_TYPE(s16, u16, 1),
291 ASSIGN_FETCH_TYPE(s32, u32, 1),
292 ASSIGN_FETCH_TYPE(s64, u64, 1),
293 ASSIGN_FETCH_TYPE_ALIAS(x8, u8, u8, 0),
294 ASSIGN_FETCH_TYPE_ALIAS(x16, u16, u16, 0),
295 ASSIGN_FETCH_TYPE_ALIAS(x32, u32, u32, 0),
296 ASSIGN_FETCH_TYPE_ALIAS(x64, u64, u64, 0),
297
298 ASSIGN_FETCH_TYPE_END
299};
300
301/* 138/*
302 * Allocate new trace_probe and initialize it (including kprobes). 139 * Allocate new trace_probe and initialize it (including kprobes).
303 */ 140 */
@@ -540,8 +377,11 @@ static int __register_trace_kprobe(struct trace_kprobe *tk)
540 return -EINVAL; 377 return -EINVAL;
541 } 378 }
542 379
543 for (i = 0; i < tk->tp.nr_args; i++) 380 for (i = 0; i < tk->tp.nr_args; i++) {
544 traceprobe_update_arg(&tk->tp.args[i]); 381 ret = traceprobe_update_arg(&tk->tp.args[i]);
382 if (ret)
383 return ret;
384 }
545 385
546 /* Set/clear disabled flag according to tp->flag */ 386 /* Set/clear disabled flag according to tp->flag */
547 if (trace_probe_is_enabled(&tk->tp)) 387 if (trace_probe_is_enabled(&tk->tp))
@@ -554,19 +394,13 @@ static int __register_trace_kprobe(struct trace_kprobe *tk)
554 else 394 else
555 ret = register_kprobe(&tk->rp.kp); 395 ret = register_kprobe(&tk->rp.kp);
556 396
557 if (ret == 0) 397 if (ret == 0) {
558 tk->tp.flags |= TP_FLAG_REGISTERED; 398 tk->tp.flags |= TP_FLAG_REGISTERED;
559 else { 399 } else if (ret == -EILSEQ) {
560 if (ret == -ENOENT && trace_kprobe_is_on_module(tk)) { 400 pr_warn("Probing address(0x%p) is not an instruction boundary.\n",
561 pr_warn("This probe might be able to register after target module is loaded. Continue.\n"); 401 tk->rp.kp.addr);
562 ret = 0; 402 ret = -EINVAL;
563 } else if (ret == -EILSEQ) {
564 pr_warn("Probing address(0x%p) is not an instruction boundary.\n",
565 tk->rp.kp.addr);
566 ret = -EINVAL;
567 }
568 } 403 }
569
570 return ret; 404 return ret;
571} 405}
572 406
@@ -629,6 +463,11 @@ static int register_trace_kprobe(struct trace_kprobe *tk)
629 463
630 /* Register k*probe */ 464 /* Register k*probe */
631 ret = __register_trace_kprobe(tk); 465 ret = __register_trace_kprobe(tk);
466 if (ret == -ENOENT && !trace_kprobe_module_exist(tk)) {
467 pr_warn("This probe might be able to register after target module is loaded. Continue.\n");
468 ret = 0;
469 }
470
632 if (ret < 0) 471 if (ret < 0)
633 unregister_kprobe_event(tk); 472 unregister_kprobe_event(tk);
634 else 473 else
@@ -713,13 +552,15 @@ static int create_trace_kprobe(int argc, char **argv)
713 long offset = 0; 552 long offset = 0;
714 void *addr = NULL; 553 void *addr = NULL;
715 char buf[MAX_EVENT_NAME_LEN]; 554 char buf[MAX_EVENT_NAME_LEN];
555 unsigned int flags = TPARG_FL_KERNEL;
716 556
717 /* argc must be >= 1 */ 557 /* argc must be >= 1 */
718 if (argv[0][0] == 'p') 558 if (argv[0][0] == 'p')
719 is_return = false; 559 is_return = false;
720 else if (argv[0][0] == 'r') 560 else if (argv[0][0] == 'r') {
721 is_return = true; 561 is_return = true;
722 else if (argv[0][0] == '-') 562 flags |= TPARG_FL_RETURN;
563 } else if (argv[0][0] == '-')
723 is_delete = true; 564 is_delete = true;
724 else { 565 else {
725 pr_info("Probe definition must be started with 'p', 'r' or" 566 pr_info("Probe definition must be started with 'p', 'r' or"
@@ -749,10 +590,13 @@ static int create_trace_kprobe(int argc, char **argv)
749 } 590 }
750 591
751 if (event) { 592 if (event) {
752 if (strchr(event, '/')) { 593 char *slash;
594
595 slash = strchr(event, '/');
596 if (slash) {
753 group = event; 597 group = event;
754 event = strchr(group, '/') + 1; 598 event = slash + 1;
755 event[-1] = '\0'; 599 slash[0] = '\0';
756 if (strlen(group) == 0) { 600 if (strlen(group) == 0) {
757 pr_info("Group name is not specified\n"); 601 pr_info("Group name is not specified\n");
758 return -EINVAL; 602 return -EINVAL;
@@ -802,8 +646,9 @@ static int create_trace_kprobe(int argc, char **argv)
802 pr_info("Failed to parse either an address or a symbol.\n"); 646 pr_info("Failed to parse either an address or a symbol.\n");
803 return ret; 647 return ret;
804 } 648 }
805 if (offset && is_return && 649 if (kprobe_on_func_entry(NULL, symbol, offset))
806 !kprobe_on_func_entry(NULL, symbol, offset)) { 650 flags |= TPARG_FL_FENTRY;
651 if (offset && is_return && !(flags & TPARG_FL_FENTRY)) {
807 pr_info("Given offset is not valid for return probe.\n"); 652 pr_info("Given offset is not valid for return probe.\n");
808 return -EINVAL; 653 return -EINVAL;
809 } 654 }
@@ -873,8 +718,7 @@ static int create_trace_kprobe(int argc, char **argv)
873 718
874 /* Parse fetch argument */ 719 /* Parse fetch argument */
875 ret = traceprobe_parse_probe_arg(arg, &tk->tp.size, parg, 720 ret = traceprobe_parse_probe_arg(arg, &tk->tp.size, parg,
876 is_return, true, 721 flags);
877 kprobes_fetch_type_table);
878 if (ret) { 722 if (ret) {
879 pr_info("Parse error at argument[%d]. (%d)\n", i, ret); 723 pr_info("Parse error at argument[%d]. (%d)\n", i, ret);
880 goto error; 724 goto error;
@@ -1028,6 +872,106 @@ static const struct file_operations kprobe_profile_ops = {
1028 .release = seq_release, 872 .release = seq_release,
1029}; 873};
1030 874
875/* Kprobe specific fetch functions */
876
877/* Return the length of string -- including null terminal byte */
878static nokprobe_inline int
879fetch_store_strlen(unsigned long addr)
880{
881 mm_segment_t old_fs;
882 int ret, len = 0;
883 u8 c;
884
885 old_fs = get_fs();
886 set_fs(KERNEL_DS);
887 pagefault_disable();
888
889 do {
890 ret = __copy_from_user_inatomic(&c, (u8 *)addr + len, 1);
891 len++;
892 } while (c && ret == 0 && len < MAX_STRING_SIZE);
893
894 pagefault_enable();
895 set_fs(old_fs);
896
897 return (ret < 0) ? ret : len;
898}
899
900/*
901 * Fetch a null-terminated string. Caller MUST set *(u32 *)buf with max
902 * length and relative data location.
903 */
904static nokprobe_inline int
905fetch_store_string(unsigned long addr, void *dest, void *base)
906{
907 int maxlen = get_loc_len(*(u32 *)dest);
908 u8 *dst = get_loc_data(dest, base);
909 long ret;
910
911 if (unlikely(!maxlen))
912 return -ENOMEM;
913 /*
914 * Try to get string again, since the string can be changed while
915 * probing.
916 */
917 ret = strncpy_from_unsafe(dst, (void *)addr, maxlen);
918
919 if (ret >= 0)
920 *(u32 *)dest = make_data_loc(ret, (void *)dst - base);
921 return ret;
922}
923
924static nokprobe_inline int
925probe_mem_read(void *dest, void *src, size_t size)
926{
927 return probe_kernel_read(dest, src, size);
928}
929
930/* Note that we don't verify it, since the code does not come from user space */
931static int
932process_fetch_insn(struct fetch_insn *code, struct pt_regs *regs, void *dest,
933 void *base)
934{
935 unsigned long val;
936
937retry:
938 /* 1st stage: get value from context */
939 switch (code->op) {
940 case FETCH_OP_REG:
941 val = regs_get_register(regs, code->param);
942 break;
943 case FETCH_OP_STACK:
944 val = regs_get_kernel_stack_nth(regs, code->param);
945 break;
946 case FETCH_OP_STACKP:
947 val = kernel_stack_pointer(regs);
948 break;
949 case FETCH_OP_RETVAL:
950 val = regs_return_value(regs);
951 break;
952 case FETCH_OP_IMM:
953 val = code->immediate;
954 break;
955 case FETCH_OP_COMM:
956 val = (unsigned long)current->comm;
957 break;
958#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
959 case FETCH_OP_ARG:
960 val = regs_get_kernel_argument(regs, code->param);
961 break;
962#endif
963 case FETCH_NOP_SYMBOL: /* Ignore a place holder */
964 code++;
965 goto retry;
966 default:
967 return -EILSEQ;
968 }
969 code++;
970
971 return process_fetch_insn_bottom(code, val, dest, base);
972}
973NOKPROBE_SYMBOL(process_fetch_insn)
974
1031/* Kprobe handler */ 975/* Kprobe handler */
1032static nokprobe_inline void 976static nokprobe_inline void
1033__kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs, 977__kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs,
@@ -1059,7 +1003,7 @@ __kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs,
1059 1003
1060 entry = ring_buffer_event_data(event); 1004 entry = ring_buffer_event_data(event);
1061 entry->ip = (unsigned long)tk->rp.kp.addr; 1005 entry->ip = (unsigned long)tk->rp.kp.addr;
1062 store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize); 1006 store_trace_args(&entry[1], &tk->tp, regs, sizeof(*entry), dsize);
1063 1007
1064 event_trigger_unlock_commit_regs(trace_file, buffer, event, 1008 event_trigger_unlock_commit_regs(trace_file, buffer, event,
1065 entry, irq_flags, pc, regs); 1009 entry, irq_flags, pc, regs);
@@ -1108,7 +1052,7 @@ __kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
1108 entry = ring_buffer_event_data(event); 1052 entry = ring_buffer_event_data(event);
1109 entry->func = (unsigned long)tk->rp.kp.addr; 1053 entry->func = (unsigned long)tk->rp.kp.addr;
1110 entry->ret_ip = (unsigned long)ri->ret_addr; 1054 entry->ret_ip = (unsigned long)ri->ret_addr;
1111 store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize); 1055 store_trace_args(&entry[1], &tk->tp, regs, sizeof(*entry), dsize);
1112 1056
1113 event_trigger_unlock_commit_regs(trace_file, buffer, event, 1057 event_trigger_unlock_commit_regs(trace_file, buffer, event,
1114 entry, irq_flags, pc, regs); 1058 entry, irq_flags, pc, regs);
@@ -1133,8 +1077,6 @@ print_kprobe_event(struct trace_iterator *iter, int flags,
1133 struct kprobe_trace_entry_head *field; 1077 struct kprobe_trace_entry_head *field;
1134 struct trace_seq *s = &iter->seq; 1078 struct trace_seq *s = &iter->seq;
1135 struct trace_probe *tp; 1079 struct trace_probe *tp;
1136 u8 *data;
1137 int i;
1138 1080
1139 field = (struct kprobe_trace_entry_head *)iter->ent; 1081 field = (struct kprobe_trace_entry_head *)iter->ent;
1140 tp = container_of(event, struct trace_probe, call.event); 1082 tp = container_of(event, struct trace_probe, call.event);
@@ -1146,11 +1088,9 @@ print_kprobe_event(struct trace_iterator *iter, int flags,
1146 1088
1147 trace_seq_putc(s, ')'); 1089 trace_seq_putc(s, ')');
1148 1090
1149 data = (u8 *)&field[1]; 1091 if (print_probe_args(s, tp->args, tp->nr_args,
1150 for (i = 0; i < tp->nr_args; i++) 1092 (u8 *)&field[1], field) < 0)
1151 if (!tp->args[i].type->print(s, tp->args[i].name, 1093 goto out;
1152 data + tp->args[i].offset, field))
1153 goto out;
1154 1094
1155 trace_seq_putc(s, '\n'); 1095 trace_seq_putc(s, '\n');
1156 out: 1096 out:
@@ -1164,8 +1104,6 @@ print_kretprobe_event(struct trace_iterator *iter, int flags,
1164 struct kretprobe_trace_entry_head *field; 1104 struct kretprobe_trace_entry_head *field;
1165 struct trace_seq *s = &iter->seq; 1105 struct trace_seq *s = &iter->seq;
1166 struct trace_probe *tp; 1106 struct trace_probe *tp;
1167 u8 *data;
1168 int i;
1169 1107
1170 field = (struct kretprobe_trace_entry_head *)iter->ent; 1108 field = (struct kretprobe_trace_entry_head *)iter->ent;
1171 tp = container_of(event, struct trace_probe, call.event); 1109 tp = container_of(event, struct trace_probe, call.event);
@@ -1182,11 +1120,9 @@ print_kretprobe_event(struct trace_iterator *iter, int flags,
1182 1120
1183 trace_seq_putc(s, ')'); 1121 trace_seq_putc(s, ')');
1184 1122
1185 data = (u8 *)&field[1]; 1123 if (print_probe_args(s, tp->args, tp->nr_args,
1186 for (i = 0; i < tp->nr_args; i++) 1124 (u8 *)&field[1], field) < 0)
1187 if (!tp->args[i].type->print(s, tp->args[i].name, 1125 goto out;
1188 data + tp->args[i].offset, field))
1189 goto out;
1190 1126
1191 trace_seq_putc(s, '\n'); 1127 trace_seq_putc(s, '\n');
1192 1128
@@ -1197,49 +1133,25 @@ print_kretprobe_event(struct trace_iterator *iter, int flags,
1197 1133
1198static int kprobe_event_define_fields(struct trace_event_call *event_call) 1134static int kprobe_event_define_fields(struct trace_event_call *event_call)
1199{ 1135{
1200 int ret, i; 1136 int ret;
1201 struct kprobe_trace_entry_head field; 1137 struct kprobe_trace_entry_head field;
1202 struct trace_kprobe *tk = (struct trace_kprobe *)event_call->data; 1138 struct trace_kprobe *tk = (struct trace_kprobe *)event_call->data;
1203 1139
1204 DEFINE_FIELD(unsigned long, ip, FIELD_STRING_IP, 0); 1140 DEFINE_FIELD(unsigned long, ip, FIELD_STRING_IP, 0);
1205 /* Set argument names as fields */
1206 for (i = 0; i < tk->tp.nr_args; i++) {
1207 struct probe_arg *parg = &tk->tp.args[i];
1208 1141
1209 ret = trace_define_field(event_call, parg->type->fmttype, 1142 return traceprobe_define_arg_fields(event_call, sizeof(field), &tk->tp);
1210 parg->name,
1211 sizeof(field) + parg->offset,
1212 parg->type->size,
1213 parg->type->is_signed,
1214 FILTER_OTHER);
1215 if (ret)
1216 return ret;
1217 }
1218 return 0;
1219} 1143}
1220 1144
1221static int kretprobe_event_define_fields(struct trace_event_call *event_call) 1145static int kretprobe_event_define_fields(struct trace_event_call *event_call)
1222{ 1146{
1223 int ret, i; 1147 int ret;
1224 struct kretprobe_trace_entry_head field; 1148 struct kretprobe_trace_entry_head field;
1225 struct trace_kprobe *tk = (struct trace_kprobe *)event_call->data; 1149 struct trace_kprobe *tk = (struct trace_kprobe *)event_call->data;
1226 1150
1227 DEFINE_FIELD(unsigned long, func, FIELD_STRING_FUNC, 0); 1151 DEFINE_FIELD(unsigned long, func, FIELD_STRING_FUNC, 0);
1228 DEFINE_FIELD(unsigned long, ret_ip, FIELD_STRING_RETIP, 0); 1152 DEFINE_FIELD(unsigned long, ret_ip, FIELD_STRING_RETIP, 0);
1229 /* Set argument names as fields */
1230 for (i = 0; i < tk->tp.nr_args; i++) {
1231 struct probe_arg *parg = &tk->tp.args[i];
1232 1153
1233 ret = trace_define_field(event_call, parg->type->fmttype, 1154 return traceprobe_define_arg_fields(event_call, sizeof(field), &tk->tp);
1234 parg->name,
1235 sizeof(field) + parg->offset,
1236 parg->type->size,
1237 parg->type->is_signed,
1238 FILTER_OTHER);
1239 if (ret)
1240 return ret;
1241 }
1242 return 0;
1243} 1155}
1244 1156
1245#ifdef CONFIG_PERF_EVENTS 1157#ifdef CONFIG_PERF_EVENTS
@@ -1286,7 +1198,7 @@ kprobe_perf_func(struct trace_kprobe *tk, struct pt_regs *regs)
1286 1198
1287 entry->ip = (unsigned long)tk->rp.kp.addr; 1199 entry->ip = (unsigned long)tk->rp.kp.addr;
1288 memset(&entry[1], 0, dsize); 1200 memset(&entry[1], 0, dsize);
1289 store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize); 1201 store_trace_args(&entry[1], &tk->tp, regs, sizeof(*entry), dsize);
1290 perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs, 1202 perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
1291 head, NULL); 1203 head, NULL);
1292 return 0; 1204 return 0;
@@ -1322,7 +1234,7 @@ kretprobe_perf_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
1322 1234
1323 entry->func = (unsigned long)tk->rp.kp.addr; 1235 entry->func = (unsigned long)tk->rp.kp.addr;
1324 entry->ret_ip = (unsigned long)ri->ret_addr; 1236 entry->ret_ip = (unsigned long)ri->ret_addr;
1325 store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize); 1237 store_trace_args(&entry[1], &tk->tp, regs, sizeof(*entry), dsize);
1326 perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs, 1238 perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
1327 head, NULL); 1239 head, NULL);
1328} 1240}
@@ -1457,7 +1369,7 @@ static int register_kprobe_event(struct trace_kprobe *tk)
1457 1369
1458 init_trace_event_call(tk, call); 1370 init_trace_event_call(tk, call);
1459 1371
1460 if (set_print_fmt(&tk->tp, trace_kprobe_is_return(tk)) < 0) 1372 if (traceprobe_set_print_fmt(&tk->tp, trace_kprobe_is_return(tk)) < 0)
1461 return -ENOMEM; 1373 return -ENOMEM;
1462 ret = register_trace_event(&call->event); 1374 ret = register_trace_event(&call->event);
1463 if (!ret) { 1375 if (!ret) {
@@ -1514,7 +1426,7 @@ create_local_trace_kprobe(char *func, void *addr, unsigned long offs,
1514 1426
1515 init_trace_event_call(tk, &tk->tp.call); 1427 init_trace_event_call(tk, &tk->tp.call);
1516 1428
1517 if (set_print_fmt(&tk->tp, trace_kprobe_is_return(tk)) < 0) { 1429 if (traceprobe_set_print_fmt(&tk->tp, trace_kprobe_is_return(tk)) < 0) {
1518 ret = -ENOMEM; 1430 ret = -ENOMEM;
1519 goto error; 1431 goto error;
1520 } 1432 }
diff --git a/kernel/trace/trace_probe.c b/kernel/trace/trace_probe.c
index e99c3ce7aa65..3ef15a6683c0 100644
--- a/kernel/trace/trace_probe.c
+++ b/kernel/trace/trace_probe.c
@@ -26,14 +26,12 @@ const char *reserved_field_names[] = {
26 26
27/* Printing in basic type function template */ 27/* Printing in basic type function template */
28#define DEFINE_BASIC_PRINT_TYPE_FUNC(tname, type, fmt) \ 28#define DEFINE_BASIC_PRINT_TYPE_FUNC(tname, type, fmt) \
29int PRINT_TYPE_FUNC_NAME(tname)(struct trace_seq *s, const char *name, \ 29int PRINT_TYPE_FUNC_NAME(tname)(struct trace_seq *s, void *data, void *ent)\
30 void *data, void *ent) \
31{ \ 30{ \
32 trace_seq_printf(s, " %s=" fmt, name, *(type *)data); \ 31 trace_seq_printf(s, fmt, *(type *)data); \
33 return !trace_seq_has_overflowed(s); \ 32 return !trace_seq_has_overflowed(s); \
34} \ 33} \
35const char PRINT_TYPE_FMT_NAME(tname)[] = fmt; \ 34const char PRINT_TYPE_FMT_NAME(tname)[] = fmt;
36NOKPROBE_SYMBOL(PRINT_TYPE_FUNC_NAME(tname));
37 35
38DEFINE_BASIC_PRINT_TYPE_FUNC(u8, u8, "%u") 36DEFINE_BASIC_PRINT_TYPE_FUNC(u8, u8, "%u")
39DEFINE_BASIC_PRINT_TYPE_FUNC(u16, u16, "%u") 37DEFINE_BASIC_PRINT_TYPE_FUNC(u16, u16, "%u")
@@ -48,193 +46,52 @@ DEFINE_BASIC_PRINT_TYPE_FUNC(x16, u16, "0x%x")
48DEFINE_BASIC_PRINT_TYPE_FUNC(x32, u32, "0x%x") 46DEFINE_BASIC_PRINT_TYPE_FUNC(x32, u32, "0x%x")
49DEFINE_BASIC_PRINT_TYPE_FUNC(x64, u64, "0x%Lx") 47DEFINE_BASIC_PRINT_TYPE_FUNC(x64, u64, "0x%Lx")
50 48
49int PRINT_TYPE_FUNC_NAME(symbol)(struct trace_seq *s, void *data, void *ent)
50{
51 trace_seq_printf(s, "%pS", (void *)*(unsigned long *)data);
52 return !trace_seq_has_overflowed(s);
53}
54const char PRINT_TYPE_FMT_NAME(symbol)[] = "%pS";
55
51/* Print type function for string type */ 56/* Print type function for string type */
52int PRINT_TYPE_FUNC_NAME(string)(struct trace_seq *s, const char *name, 57int PRINT_TYPE_FUNC_NAME(string)(struct trace_seq *s, void *data, void *ent)
53 void *data, void *ent)
54{ 58{
55 int len = *(u32 *)data >> 16; 59 int len = *(u32 *)data >> 16;
56 60
57 if (!len) 61 if (!len)
58 trace_seq_printf(s, " %s=(fault)", name); 62 trace_seq_puts(s, "(fault)");
59 else 63 else
60 trace_seq_printf(s, " %s=\"%s\"", name, 64 trace_seq_printf(s, "\"%s\"",
61 (const char *)get_loc_data(data, ent)); 65 (const char *)get_loc_data(data, ent));
62 return !trace_seq_has_overflowed(s); 66 return !trace_seq_has_overflowed(s);
63} 67}
64NOKPROBE_SYMBOL(PRINT_TYPE_FUNC_NAME(string));
65 68
66const char PRINT_TYPE_FMT_NAME(string)[] = "\\\"%s\\\""; 69const char PRINT_TYPE_FMT_NAME(string)[] = "\\\"%s\\\"";
67 70
68#define CHECK_FETCH_FUNCS(method, fn) \ 71/* Fetch type information table */
69 (((FETCH_FUNC_NAME(method, u8) == fn) || \ 72static const struct fetch_type probe_fetch_types[] = {
70 (FETCH_FUNC_NAME(method, u16) == fn) || \ 73 /* Special types */
71 (FETCH_FUNC_NAME(method, u32) == fn) || \ 74 __ASSIGN_FETCH_TYPE("string", string, string, sizeof(u32), 1,
72 (FETCH_FUNC_NAME(method, u64) == fn) || \ 75 "__data_loc char[]"),
73 (FETCH_FUNC_NAME(method, string) == fn) || \ 76 /* Basic types */
74 (FETCH_FUNC_NAME(method, string_size) == fn)) \ 77 ASSIGN_FETCH_TYPE(u8, u8, 0),
75 && (fn != NULL)) 78 ASSIGN_FETCH_TYPE(u16, u16, 0),
76 79 ASSIGN_FETCH_TYPE(u32, u32, 0),
77/* Data fetch function templates */ 80 ASSIGN_FETCH_TYPE(u64, u64, 0),
78#define DEFINE_FETCH_reg(type) \ 81 ASSIGN_FETCH_TYPE(s8, u8, 1),
79void FETCH_FUNC_NAME(reg, type)(struct pt_regs *regs, void *offset, void *dest) \ 82 ASSIGN_FETCH_TYPE(s16, u16, 1),
80{ \ 83 ASSIGN_FETCH_TYPE(s32, u32, 1),
81 *(type *)dest = (type)regs_get_register(regs, \ 84 ASSIGN_FETCH_TYPE(s64, u64, 1),
82 (unsigned int)((unsigned long)offset)); \ 85 ASSIGN_FETCH_TYPE_ALIAS(x8, u8, u8, 0),
83} \ 86 ASSIGN_FETCH_TYPE_ALIAS(x16, u16, u16, 0),
84NOKPROBE_SYMBOL(FETCH_FUNC_NAME(reg, type)); 87 ASSIGN_FETCH_TYPE_ALIAS(x32, u32, u32, 0),
85DEFINE_BASIC_FETCH_FUNCS(reg) 88 ASSIGN_FETCH_TYPE_ALIAS(x64, u64, u64, 0),
86/* No string on the register */ 89 ASSIGN_FETCH_TYPE_ALIAS(symbol, ADDR_FETCH_TYPE, ADDR_FETCH_TYPE, 0),
87#define fetch_reg_string NULL 90
88#define fetch_reg_string_size NULL 91 ASSIGN_FETCH_TYPE_END
89
90#define DEFINE_FETCH_retval(type) \
91void FETCH_FUNC_NAME(retval, type)(struct pt_regs *regs, \
92 void *dummy, void *dest) \
93{ \
94 *(type *)dest = (type)regs_return_value(regs); \
95} \
96NOKPROBE_SYMBOL(FETCH_FUNC_NAME(retval, type));
97DEFINE_BASIC_FETCH_FUNCS(retval)
98/* No string on the retval */
99#define fetch_retval_string NULL
100#define fetch_retval_string_size NULL
101
102/* Dereference memory access function */
103struct deref_fetch_param {
104 struct fetch_param orig;
105 long offset;
106 fetch_func_t fetch;
107 fetch_func_t fetch_size;
108}; 92};
109 93
110#define DEFINE_FETCH_deref(type) \ 94static const struct fetch_type *find_fetch_type(const char *type)
111void FETCH_FUNC_NAME(deref, type)(struct pt_regs *regs, \
112 void *data, void *dest) \
113{ \
114 struct deref_fetch_param *dprm = data; \
115 unsigned long addr; \
116 call_fetch(&dprm->orig, regs, &addr); \
117 if (addr) { \
118 addr += dprm->offset; \
119 dprm->fetch(regs, (void *)addr, dest); \
120 } else \
121 *(type *)dest = 0; \
122} \
123NOKPROBE_SYMBOL(FETCH_FUNC_NAME(deref, type));
124DEFINE_BASIC_FETCH_FUNCS(deref)
125DEFINE_FETCH_deref(string)
126
127void FETCH_FUNC_NAME(deref, string_size)(struct pt_regs *regs,
128 void *data, void *dest)
129{
130 struct deref_fetch_param *dprm = data;
131 unsigned long addr;
132
133 call_fetch(&dprm->orig, regs, &addr);
134 if (addr && dprm->fetch_size) {
135 addr += dprm->offset;
136 dprm->fetch_size(regs, (void *)addr, dest);
137 } else
138 *(string_size *)dest = 0;
139}
140NOKPROBE_SYMBOL(FETCH_FUNC_NAME(deref, string_size));
141
142static void update_deref_fetch_param(struct deref_fetch_param *data)
143{
144 if (CHECK_FETCH_FUNCS(deref, data->orig.fn))
145 update_deref_fetch_param(data->orig.data);
146 else if (CHECK_FETCH_FUNCS(symbol, data->orig.fn))
147 update_symbol_cache(data->orig.data);
148}
149NOKPROBE_SYMBOL(update_deref_fetch_param);
150
151static void free_deref_fetch_param(struct deref_fetch_param *data)
152{
153 if (CHECK_FETCH_FUNCS(deref, data->orig.fn))
154 free_deref_fetch_param(data->orig.data);
155 else if (CHECK_FETCH_FUNCS(symbol, data->orig.fn))
156 free_symbol_cache(data->orig.data);
157 kfree(data);
158}
159NOKPROBE_SYMBOL(free_deref_fetch_param);
160
161/* Bitfield fetch function */
162struct bitfield_fetch_param {
163 struct fetch_param orig;
164 unsigned char hi_shift;
165 unsigned char low_shift;
166};
167
168#define DEFINE_FETCH_bitfield(type) \
169void FETCH_FUNC_NAME(bitfield, type)(struct pt_regs *regs, \
170 void *data, void *dest) \
171{ \
172 struct bitfield_fetch_param *bprm = data; \
173 type buf = 0; \
174 call_fetch(&bprm->orig, regs, &buf); \
175 if (buf) { \
176 buf <<= bprm->hi_shift; \
177 buf >>= bprm->low_shift; \
178 } \
179 *(type *)dest = buf; \
180} \
181NOKPROBE_SYMBOL(FETCH_FUNC_NAME(bitfield, type));
182DEFINE_BASIC_FETCH_FUNCS(bitfield)
183#define fetch_bitfield_string NULL
184#define fetch_bitfield_string_size NULL
185
186static void
187update_bitfield_fetch_param(struct bitfield_fetch_param *data)
188{
189 /*
190 * Don't check the bitfield itself, because this must be the
191 * last fetch function.
192 */
193 if (CHECK_FETCH_FUNCS(deref, data->orig.fn))
194 update_deref_fetch_param(data->orig.data);
195 else if (CHECK_FETCH_FUNCS(symbol, data->orig.fn))
196 update_symbol_cache(data->orig.data);
197}
198
199static void
200free_bitfield_fetch_param(struct bitfield_fetch_param *data)
201{
202 /*
203 * Don't check the bitfield itself, because this must be the
204 * last fetch function.
205 */
206 if (CHECK_FETCH_FUNCS(deref, data->orig.fn))
207 free_deref_fetch_param(data->orig.data);
208 else if (CHECK_FETCH_FUNCS(symbol, data->orig.fn))
209 free_symbol_cache(data->orig.data);
210
211 kfree(data);
212}
213
214void FETCH_FUNC_NAME(comm, string)(struct pt_regs *regs,
215 void *data, void *dest)
216{
217 int maxlen = get_rloc_len(*(u32 *)dest);
218 u8 *dst = get_rloc_data(dest);
219 long ret;
220
221 if (!maxlen)
222 return;
223
224 ret = strlcpy(dst, current->comm, maxlen);
225 *(u32 *)dest = make_data_rloc(ret, get_rloc_offs(*(u32 *)dest));
226}
227NOKPROBE_SYMBOL(FETCH_FUNC_NAME(comm, string));
228
229void FETCH_FUNC_NAME(comm, string_size)(struct pt_regs *regs,
230 void *data, void *dest)
231{
232 *(u32 *)dest = strlen(current->comm) + 1;
233}
234NOKPROBE_SYMBOL(FETCH_FUNC_NAME(comm, string_size));
235
236static const struct fetch_type *find_fetch_type(const char *type,
237 const struct fetch_type *ftbl)
238{ 95{
239 int i; 96 int i;
240 97
@@ -255,58 +112,27 @@ static const struct fetch_type *find_fetch_type(const char *type,
255 112
256 switch (bs) { 113 switch (bs) {
257 case 8: 114 case 8:
258 return find_fetch_type("u8", ftbl); 115 return find_fetch_type("u8");
259 case 16: 116 case 16:
260 return find_fetch_type("u16", ftbl); 117 return find_fetch_type("u16");
261 case 32: 118 case 32:
262 return find_fetch_type("u32", ftbl); 119 return find_fetch_type("u32");
263 case 64: 120 case 64:
264 return find_fetch_type("u64", ftbl); 121 return find_fetch_type("u64");
265 default: 122 default:
266 goto fail; 123 goto fail;
267 } 124 }
268 } 125 }
269 126
270 for (i = 0; ftbl[i].name; i++) { 127 for (i = 0; probe_fetch_types[i].name; i++) {
271 if (strcmp(type, ftbl[i].name) == 0) 128 if (strcmp(type, probe_fetch_types[i].name) == 0)
272 return &ftbl[i]; 129 return &probe_fetch_types[i];
273 } 130 }
274 131
275fail: 132fail:
276 return NULL; 133 return NULL;
277} 134}
278 135
279/* Special function : only accept unsigned long */
280static void fetch_kernel_stack_address(struct pt_regs *regs, void *dummy, void *dest)
281{
282 *(unsigned long *)dest = kernel_stack_pointer(regs);
283}
284NOKPROBE_SYMBOL(fetch_kernel_stack_address);
285
286static void fetch_user_stack_address(struct pt_regs *regs, void *dummy, void *dest)
287{
288 *(unsigned long *)dest = user_stack_pointer(regs);
289}
290NOKPROBE_SYMBOL(fetch_user_stack_address);
291
292static fetch_func_t get_fetch_size_function(const struct fetch_type *type,
293 fetch_func_t orig_fn,
294 const struct fetch_type *ftbl)
295{
296 int i;
297
298 if (type != &ftbl[FETCH_TYPE_STRING])
299 return NULL; /* Only string type needs size function */
300
301 for (i = 0; i < FETCH_MTD_END; i++)
302 if (type->fetch[i] == orig_fn)
303 return ftbl[FETCH_TYPE_STRSIZE].fetch[i];
304
305 WARN_ON(1); /* This should not happen */
306
307 return NULL;
308}
309
310/* Split symbol and offset. */ 136/* Split symbol and offset. */
311int traceprobe_split_symbol_offset(char *symbol, long *offset) 137int traceprobe_split_symbol_offset(char *symbol, long *offset)
312{ 138{
@@ -331,41 +157,44 @@ int traceprobe_split_symbol_offset(char *symbol, long *offset)
331#define PARAM_MAX_STACK (THREAD_SIZE / sizeof(unsigned long)) 157#define PARAM_MAX_STACK (THREAD_SIZE / sizeof(unsigned long))
332 158
333static int parse_probe_vars(char *arg, const struct fetch_type *t, 159static int parse_probe_vars(char *arg, const struct fetch_type *t,
334 struct fetch_param *f, bool is_return, 160 struct fetch_insn *code, unsigned int flags)
335 bool is_kprobe)
336{ 161{
337 int ret = 0; 162 int ret = 0;
338 unsigned long param; 163 unsigned long param;
339 164
340 if (strcmp(arg, "retval") == 0) { 165 if (strcmp(arg, "retval") == 0) {
341 if (is_return) 166 if (flags & TPARG_FL_RETURN)
342 f->fn = t->fetch[FETCH_MTD_retval]; 167 code->op = FETCH_OP_RETVAL;
343 else 168 else
344 ret = -EINVAL; 169 ret = -EINVAL;
345 } else if (strncmp(arg, "stack", 5) == 0) { 170 } else if (strncmp(arg, "stack", 5) == 0) {
346 if (arg[5] == '\0') { 171 if (arg[5] == '\0') {
347 if (strcmp(t->name, DEFAULT_FETCH_TYPE_STR)) 172 code->op = FETCH_OP_STACKP;
348 return -EINVAL;
349
350 if (is_kprobe)
351 f->fn = fetch_kernel_stack_address;
352 else
353 f->fn = fetch_user_stack_address;
354 } else if (isdigit(arg[5])) { 173 } else if (isdigit(arg[5])) {
355 ret = kstrtoul(arg + 5, 10, &param); 174 ret = kstrtoul(arg + 5, 10, &param);
356 if (ret || (is_kprobe && param > PARAM_MAX_STACK)) 175 if (ret || ((flags & TPARG_FL_KERNEL) &&
176 param > PARAM_MAX_STACK))
357 ret = -EINVAL; 177 ret = -EINVAL;
358 else { 178 else {
359 f->fn = t->fetch[FETCH_MTD_stack]; 179 code->op = FETCH_OP_STACK;
360 f->data = (void *)param; 180 code->param = (unsigned int)param;
361 } 181 }
362 } else 182 } else
363 ret = -EINVAL; 183 ret = -EINVAL;
364 } else if (strcmp(arg, "comm") == 0) { 184 } else if (strcmp(arg, "comm") == 0) {
365 if (strcmp(t->name, "string") != 0 && 185 code->op = FETCH_OP_COMM;
366 strcmp(t->name, "string_size") != 0) 186#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
187 } else if (((flags & TPARG_FL_MASK) ==
188 (TPARG_FL_KERNEL | TPARG_FL_FENTRY)) &&
189 strncmp(arg, "arg", 3) == 0) {
190 if (!isdigit(arg[3]))
191 return -EINVAL;
192 ret = kstrtoul(arg + 3, 10, &param);
193 if (ret || !param || param > PARAM_MAX_STACK)
367 return -EINVAL; 194 return -EINVAL;
368 f->fn = t->fetch[FETCH_MTD_comm]; 195 code->op = FETCH_OP_ARG;
196 code->param = (unsigned int)param - 1;
197#endif
369 } else 198 } else
370 ret = -EINVAL; 199 ret = -EINVAL;
371 200
@@ -373,25 +202,27 @@ static int parse_probe_vars(char *arg, const struct fetch_type *t,
373} 202}
374 203
375/* Recursive argument parser */ 204/* Recursive argument parser */
376static int parse_probe_arg(char *arg, const struct fetch_type *t, 205static int
377 struct fetch_param *f, bool is_return, bool is_kprobe, 206parse_probe_arg(char *arg, const struct fetch_type *type,
378 const struct fetch_type *ftbl) 207 struct fetch_insn **pcode, struct fetch_insn *end,
208 unsigned int flags)
379{ 209{
210 struct fetch_insn *code = *pcode;
380 unsigned long param; 211 unsigned long param;
381 long offset; 212 long offset = 0;
382 char *tmp; 213 char *tmp;
383 int ret = 0; 214 int ret = 0;
384 215
385 switch (arg[0]) { 216 switch (arg[0]) {
386 case '$': 217 case '$':
387 ret = parse_probe_vars(arg + 1, t, f, is_return, is_kprobe); 218 ret = parse_probe_vars(arg + 1, type, code, flags);
388 break; 219 break;
389 220
390 case '%': /* named register */ 221 case '%': /* named register */
391 ret = regs_query_register_offset(arg + 1); 222 ret = regs_query_register_offset(arg + 1);
392 if (ret >= 0) { 223 if (ret >= 0) {
393 f->fn = t->fetch[FETCH_MTD_reg]; 224 code->op = FETCH_OP_REG;
394 f->data = (void *)(unsigned long)ret; 225 code->param = (unsigned int)ret;
395 ret = 0; 226 ret = 0;
396 } 227 }
397 break; 228 break;
@@ -401,33 +232,42 @@ static int parse_probe_arg(char *arg, const struct fetch_type *t,
401 ret = kstrtoul(arg + 1, 0, &param); 232 ret = kstrtoul(arg + 1, 0, &param);
402 if (ret) 233 if (ret)
403 break; 234 break;
404 235 /* load address */
405 f->fn = t->fetch[FETCH_MTD_memory]; 236 code->op = FETCH_OP_IMM;
406 f->data = (void *)param; 237 code->immediate = param;
407 } else if (arg[1] == '+') { 238 } else if (arg[1] == '+') {
408 /* kprobes don't support file offsets */ 239 /* kprobes don't support file offsets */
409 if (is_kprobe) 240 if (flags & TPARG_FL_KERNEL)
410 return -EINVAL; 241 return -EINVAL;
411 242
412 ret = kstrtol(arg + 2, 0, &offset); 243 ret = kstrtol(arg + 2, 0, &offset);
413 if (ret) 244 if (ret)
414 break; 245 break;
415 246
416 f->fn = t->fetch[FETCH_MTD_file_offset]; 247 code->op = FETCH_OP_FOFFS;
417 f->data = (void *)offset; 248 code->immediate = (unsigned long)offset; // imm64?
418 } else { 249 } else {
419 /* uprobes don't support symbols */ 250 /* uprobes don't support symbols */
420 if (!is_kprobe) 251 if (!(flags & TPARG_FL_KERNEL))
421 return -EINVAL; 252 return -EINVAL;
422 253
423 ret = traceprobe_split_symbol_offset(arg + 1, &offset); 254 /* Preserve symbol for updating */
424 if (ret) 255 code->op = FETCH_NOP_SYMBOL;
425 break; 256 code->data = kstrdup(arg + 1, GFP_KERNEL);
257 if (!code->data)
258 return -ENOMEM;
259 if (++code == end)
260 return -E2BIG;
426 261
427 f->data = alloc_symbol_cache(arg + 1, offset); 262 code->op = FETCH_OP_IMM;
428 if (f->data) 263 code->immediate = 0;
429 f->fn = t->fetch[FETCH_MTD_symbol];
430 } 264 }
265 /* These are fetching from memory */
266 if (++code == end)
267 return -E2BIG;
268 *pcode = code;
269 code->op = FETCH_OP_DEREF;
270 code->offset = offset;
431 break; 271 break;
432 272
433 case '+': /* deref memory */ 273 case '+': /* deref memory */
@@ -435,11 +275,10 @@ static int parse_probe_arg(char *arg, const struct fetch_type *t,
435 case '-': 275 case '-':
436 tmp = strchr(arg, '('); 276 tmp = strchr(arg, '(');
437 if (!tmp) 277 if (!tmp)
438 break; 278 return -EINVAL;
439 279
440 *tmp = '\0'; 280 *tmp = '\0';
441 ret = kstrtol(arg, 0, &offset); 281 ret = kstrtol(arg, 0, &offset);
442
443 if (ret) 282 if (ret)
444 break; 283 break;
445 284
@@ -447,36 +286,27 @@ static int parse_probe_arg(char *arg, const struct fetch_type *t,
447 tmp = strrchr(arg, ')'); 286 tmp = strrchr(arg, ')');
448 287
449 if (tmp) { 288 if (tmp) {
450 struct deref_fetch_param *dprm; 289 const struct fetch_type *t2 = find_fetch_type(NULL);
451 const struct fetch_type *t2;
452 290
453 t2 = find_fetch_type(NULL, ftbl);
454 *tmp = '\0'; 291 *tmp = '\0';
455 dprm = kzalloc(sizeof(struct deref_fetch_param), GFP_KERNEL); 292 ret = parse_probe_arg(arg, t2, &code, end, flags);
456
457 if (!dprm)
458 return -ENOMEM;
459
460 dprm->offset = offset;
461 dprm->fetch = t->fetch[FETCH_MTD_memory];
462 dprm->fetch_size = get_fetch_size_function(t,
463 dprm->fetch, ftbl);
464 ret = parse_probe_arg(arg, t2, &dprm->orig, is_return,
465 is_kprobe, ftbl);
466 if (ret) 293 if (ret)
467 kfree(dprm); 294 break;
468 else { 295 if (code->op == FETCH_OP_COMM)
469 f->fn = t->fetch[FETCH_MTD_deref]; 296 return -EINVAL;
470 f->data = (void *)dprm; 297 if (++code == end)
471 } 298 return -E2BIG;
299 *pcode = code;
300
301 code->op = FETCH_OP_DEREF;
302 code->offset = offset;
472 } 303 }
473 break; 304 break;
474 } 305 }
475 if (!ret && !f->fn) { /* Parsed, but do not find fetch method */ 306 if (!ret && code->op == FETCH_OP_NOP) {
476 pr_info("%s type has no corresponding fetch method.\n", t->name); 307 /* Parsed, but do not find fetch method */
477 ret = -EINVAL; 308 ret = -EINVAL;
478 } 309 }
479
480 return ret; 310 return ret;
481} 311}
482 312
@@ -485,22 +315,15 @@ static int parse_probe_arg(char *arg, const struct fetch_type *t,
485/* Bitfield type needs to be parsed into a fetch function */ 315/* Bitfield type needs to be parsed into a fetch function */
486static int __parse_bitfield_probe_arg(const char *bf, 316static int __parse_bitfield_probe_arg(const char *bf,
487 const struct fetch_type *t, 317 const struct fetch_type *t,
488 struct fetch_param *f) 318 struct fetch_insn **pcode)
489{ 319{
490 struct bitfield_fetch_param *bprm; 320 struct fetch_insn *code = *pcode;
491 unsigned long bw, bo; 321 unsigned long bw, bo;
492 char *tail; 322 char *tail;
493 323
494 if (*bf != 'b') 324 if (*bf != 'b')
495 return 0; 325 return 0;
496 326
497 bprm = kzalloc(sizeof(*bprm), GFP_KERNEL);
498 if (!bprm)
499 return -ENOMEM;
500
501 bprm->orig = *f;
502 f->fn = t->fetch[FETCH_MTD_bitfield];
503 f->data = (void *)bprm;
504 bw = simple_strtoul(bf + 1, &tail, 0); /* Use simple one */ 327 bw = simple_strtoul(bf + 1, &tail, 0); /* Use simple one */
505 328
506 if (bw == 0 || *tail != '@') 329 if (bw == 0 || *tail != '@')
@@ -511,20 +334,26 @@ static int __parse_bitfield_probe_arg(const char *bf,
511 334
512 if (tail == bf || *tail != '/') 335 if (tail == bf || *tail != '/')
513 return -EINVAL; 336 return -EINVAL;
337 code++;
338 if (code->op != FETCH_OP_NOP)
339 return -E2BIG;
340 *pcode = code;
514 341
515 bprm->hi_shift = BYTES_TO_BITS(t->size) - (bw + bo); 342 code->op = FETCH_OP_MOD_BF;
516 bprm->low_shift = bprm->hi_shift + bo; 343 code->lshift = BYTES_TO_BITS(t->size) - (bw + bo);
344 code->rshift = BYTES_TO_BITS(t->size) - bw;
345 code->basesize = t->size;
517 346
518 return (BYTES_TO_BITS(t->size) < (bw + bo)) ? -EINVAL : 0; 347 return (BYTES_TO_BITS(t->size) < (bw + bo)) ? -EINVAL : 0;
519} 348}
520 349
521/* String length checking wrapper */ 350/* String length checking wrapper */
522int traceprobe_parse_probe_arg(char *arg, ssize_t *size, 351int traceprobe_parse_probe_arg(char *arg, ssize_t *size,
523 struct probe_arg *parg, bool is_return, bool is_kprobe, 352 struct probe_arg *parg, unsigned int flags)
524 const struct fetch_type *ftbl)
525{ 353{
526 const char *t; 354 struct fetch_insn *code, *scode, *tmp = NULL;
527 int ret; 355 char *t, *t2;
356 int ret, len;
528 357
529 if (strlen(arg) > MAX_ARGSTR_LEN) { 358 if (strlen(arg) > MAX_ARGSTR_LEN) {
530 pr_info("Argument is too long.: %s\n", arg); 359 pr_info("Argument is too long.: %s\n", arg);
@@ -535,36 +364,128 @@ int traceprobe_parse_probe_arg(char *arg, ssize_t *size,
535 pr_info("Failed to allocate memory for command '%s'.\n", arg); 364 pr_info("Failed to allocate memory for command '%s'.\n", arg);
536 return -ENOMEM; 365 return -ENOMEM;
537 } 366 }
538 t = strchr(parg->comm, ':'); 367 t = strchr(arg, ':');
539 if (t) { 368 if (t) {
540 arg[t - parg->comm] = '\0'; 369 *t = '\0';
541 t++; 370 t2 = strchr(++t, '[');
371 if (t2) {
372 *t2 = '\0';
373 parg->count = simple_strtoul(t2 + 1, &t2, 0);
374 if (strcmp(t2, "]") || parg->count == 0)
375 return -EINVAL;
376 if (parg->count > MAX_ARRAY_LEN)
377 return -E2BIG;
378 }
542 } 379 }
543 /* 380 /*
544 * The default type of $comm should be "string", and it can't be 381 * The default type of $comm should be "string", and it can't be
545 * dereferenced. 382 * dereferenced.
546 */ 383 */
547 if (!t && strcmp(arg, "$comm") == 0) 384 if (!t && strcmp(arg, "$comm") == 0)
548 t = "string"; 385 parg->type = find_fetch_type("string");
549 parg->type = find_fetch_type(t, ftbl); 386 else
387 parg->type = find_fetch_type(t);
550 if (!parg->type) { 388 if (!parg->type) {
551 pr_info("Unsupported type: %s\n", t); 389 pr_info("Unsupported type: %s\n", t);
552 return -EINVAL; 390 return -EINVAL;
553 } 391 }
554 parg->offset = *size; 392 parg->offset = *size;
555 *size += parg->type->size; 393 *size += parg->type->size * (parg->count ?: 1);
556 ret = parse_probe_arg(arg, parg->type, &parg->fetch, is_return, 394
557 is_kprobe, ftbl); 395 if (parg->count) {
558 396 len = strlen(parg->type->fmttype) + 6;
559 if (ret >= 0 && t != NULL) 397 parg->fmt = kmalloc(len, GFP_KERNEL);
560 ret = __parse_bitfield_probe_arg(t, parg->type, &parg->fetch); 398 if (!parg->fmt)
561 399 return -ENOMEM;
562 if (ret >= 0) { 400 snprintf(parg->fmt, len, "%s[%d]", parg->type->fmttype,
563 parg->fetch_size.fn = get_fetch_size_function(parg->type, 401 parg->count);
564 parg->fetch.fn, 402 }
565 ftbl); 403
566 parg->fetch_size.data = parg->fetch.data; 404 code = tmp = kzalloc(sizeof(*code) * FETCH_INSN_MAX, GFP_KERNEL);
405 if (!code)
406 return -ENOMEM;
407 code[FETCH_INSN_MAX - 1].op = FETCH_OP_END;
408
409 ret = parse_probe_arg(arg, parg->type, &code, &code[FETCH_INSN_MAX - 1],
410 flags);
411 if (ret)
412 goto fail;
413
414 /* Store operation */
415 if (!strcmp(parg->type->name, "string")) {
416 if (code->op != FETCH_OP_DEREF && code->op != FETCH_OP_IMM &&
417 code->op != FETCH_OP_COMM) {
418 pr_info("string only accepts memory or address.\n");
419 ret = -EINVAL;
420 goto fail;
421 }
422 if (code->op != FETCH_OP_DEREF || parg->count) {
423 /*
424 * IMM and COMM is pointing actual address, those must
425 * be kept, and if parg->count != 0, this is an array
426 * of string pointers instead of string address itself.
427 */
428 code++;
429 if (code->op != FETCH_OP_NOP) {
430 ret = -E2BIG;
431 goto fail;
432 }
433 }
434 code->op = FETCH_OP_ST_STRING; /* In DEREF case, replace it */
435 code->size = parg->type->size;
436 parg->dynamic = true;
437 } else if (code->op == FETCH_OP_DEREF) {
438 code->op = FETCH_OP_ST_MEM;
439 code->size = parg->type->size;
440 } else {
441 code++;
442 if (code->op != FETCH_OP_NOP) {
443 ret = -E2BIG;
444 goto fail;
445 }
446 code->op = FETCH_OP_ST_RAW;
447 code->size = parg->type->size;
448 }
449 scode = code;
450 /* Modify operation */
451 if (t != NULL) {
452 ret = __parse_bitfield_probe_arg(t, parg->type, &code);
453 if (ret)
454 goto fail;
567 } 455 }
456 /* Loop(Array) operation */
457 if (parg->count) {
458 if (scode->op != FETCH_OP_ST_MEM &&
459 scode->op != FETCH_OP_ST_STRING) {
460 pr_info("array only accepts memory or address\n");
461 ret = -EINVAL;
462 goto fail;
463 }
464 code++;
465 if (code->op != FETCH_OP_NOP) {
466 ret = -E2BIG;
467 goto fail;
468 }
469 code->op = FETCH_OP_LP_ARRAY;
470 code->param = parg->count;
471 }
472 code++;
473 code->op = FETCH_OP_END;
474
475 /* Shrink down the code buffer */
476 parg->code = kzalloc(sizeof(*code) * (code - tmp + 1), GFP_KERNEL);
477 if (!parg->code)
478 ret = -ENOMEM;
479 else
480 memcpy(parg->code, tmp, sizeof(*code) * (code - tmp + 1));
481
482fail:
483 if (ret) {
484 for (code = tmp; code < tmp + FETCH_INSN_MAX; code++)
485 if (code->op == FETCH_NOP_SYMBOL)
486 kfree(code->data);
487 }
488 kfree(tmp);
568 489
569 return ret; 490 return ret;
570} 491}
@@ -586,35 +507,63 @@ int traceprobe_conflict_field_name(const char *name,
586 return 0; 507 return 0;
587} 508}
588 509
589void traceprobe_update_arg(struct probe_arg *arg)
590{
591 if (CHECK_FETCH_FUNCS(bitfield, arg->fetch.fn))
592 update_bitfield_fetch_param(arg->fetch.data);
593 else if (CHECK_FETCH_FUNCS(deref, arg->fetch.fn))
594 update_deref_fetch_param(arg->fetch.data);
595 else if (CHECK_FETCH_FUNCS(symbol, arg->fetch.fn))
596 update_symbol_cache(arg->fetch.data);
597}
598
599void traceprobe_free_probe_arg(struct probe_arg *arg) 510void traceprobe_free_probe_arg(struct probe_arg *arg)
600{ 511{
601 if (CHECK_FETCH_FUNCS(bitfield, arg->fetch.fn)) 512 struct fetch_insn *code = arg->code;
602 free_bitfield_fetch_param(arg->fetch.data);
603 else if (CHECK_FETCH_FUNCS(deref, arg->fetch.fn))
604 free_deref_fetch_param(arg->fetch.data);
605 else if (CHECK_FETCH_FUNCS(symbol, arg->fetch.fn))
606 free_symbol_cache(arg->fetch.data);
607 513
514 while (code && code->op != FETCH_OP_END) {
515 if (code->op == FETCH_NOP_SYMBOL)
516 kfree(code->data);
517 code++;
518 }
519 kfree(arg->code);
608 kfree(arg->name); 520 kfree(arg->name);
609 kfree(arg->comm); 521 kfree(arg->comm);
522 kfree(arg->fmt);
610} 523}
611 524
525int traceprobe_update_arg(struct probe_arg *arg)
526{
527 struct fetch_insn *code = arg->code;
528 long offset;
529 char *tmp;
530 char c;
531 int ret = 0;
532
533 while (code && code->op != FETCH_OP_END) {
534 if (code->op == FETCH_NOP_SYMBOL) {
535 if (code[1].op != FETCH_OP_IMM)
536 return -EINVAL;
537
538 tmp = strpbrk("+-", code->data);
539 if (tmp)
540 c = *tmp;
541 ret = traceprobe_split_symbol_offset(code->data,
542 &offset);
543 if (ret)
544 return ret;
545
546 code[1].immediate =
547 (unsigned long)kallsyms_lookup_name(code->data);
548 if (tmp)
549 *tmp = c;
550 if (!code[1].immediate)
551 return -ENOENT;
552 code[1].immediate += offset;
553 }
554 code++;
555 }
556 return 0;
557}
558
559/* When len=0, we just calculate the needed length */
560#define LEN_OR_ZERO (len ? len - pos : 0)
612static int __set_print_fmt(struct trace_probe *tp, char *buf, int len, 561static int __set_print_fmt(struct trace_probe *tp, char *buf, int len,
613 bool is_return) 562 bool is_return)
614{ 563{
615 int i; 564 struct probe_arg *parg;
565 int i, j;
616 int pos = 0; 566 int pos = 0;
617
618 const char *fmt, *arg; 567 const char *fmt, *arg;
619 568
620 if (!is_return) { 569 if (!is_return) {
@@ -625,35 +574,51 @@ static int __set_print_fmt(struct trace_probe *tp, char *buf, int len,
625 arg = "REC->" FIELD_STRING_FUNC ", REC->" FIELD_STRING_RETIP; 574 arg = "REC->" FIELD_STRING_FUNC ", REC->" FIELD_STRING_RETIP;
626 } 575 }
627 576
628 /* When len=0, we just calculate the needed length */
629#define LEN_OR_ZERO (len ? len - pos : 0)
630
631 pos += snprintf(buf + pos, LEN_OR_ZERO, "\"%s", fmt); 577 pos += snprintf(buf + pos, LEN_OR_ZERO, "\"%s", fmt);
632 578
633 for (i = 0; i < tp->nr_args; i++) { 579 for (i = 0; i < tp->nr_args; i++) {
634 pos += snprintf(buf + pos, LEN_OR_ZERO, " %s=%s", 580 parg = tp->args + i;
635 tp->args[i].name, tp->args[i].type->fmt); 581 pos += snprintf(buf + pos, LEN_OR_ZERO, " %s=", parg->name);
582 if (parg->count) {
583 pos += snprintf(buf + pos, LEN_OR_ZERO, "{%s",
584 parg->type->fmt);
585 for (j = 1; j < parg->count; j++)
586 pos += snprintf(buf + pos, LEN_OR_ZERO, ",%s",
587 parg->type->fmt);
588 pos += snprintf(buf + pos, LEN_OR_ZERO, "}");
589 } else
590 pos += snprintf(buf + pos, LEN_OR_ZERO, "%s",
591 parg->type->fmt);
636 } 592 }
637 593
638 pos += snprintf(buf + pos, LEN_OR_ZERO, "\", %s", arg); 594 pos += snprintf(buf + pos, LEN_OR_ZERO, "\", %s", arg);
639 595
640 for (i = 0; i < tp->nr_args; i++) { 596 for (i = 0; i < tp->nr_args; i++) {
641 if (strcmp(tp->args[i].type->name, "string") == 0) 597 parg = tp->args + i;
598 if (parg->count) {
599 if (strcmp(parg->type->name, "string") == 0)
600 fmt = ", __get_str(%s[%d])";
601 else
602 fmt = ", REC->%s[%d]";
603 for (j = 0; j < parg->count; j++)
604 pos += snprintf(buf + pos, LEN_OR_ZERO,
605 fmt, parg->name, j);
606 } else {
607 if (strcmp(parg->type->name, "string") == 0)
608 fmt = ", __get_str(%s)";
609 else
610 fmt = ", REC->%s";
642 pos += snprintf(buf + pos, LEN_OR_ZERO, 611 pos += snprintf(buf + pos, LEN_OR_ZERO,
643 ", __get_str(%s)", 612 fmt, parg->name);
644 tp->args[i].name); 613 }
645 else
646 pos += snprintf(buf + pos, LEN_OR_ZERO, ", REC->%s",
647 tp->args[i].name);
648 } 614 }
649 615
650#undef LEN_OR_ZERO
651
652 /* return the length of print_fmt */ 616 /* return the length of print_fmt */
653 return pos; 617 return pos;
654} 618}
619#undef LEN_OR_ZERO
655 620
656int set_print_fmt(struct trace_probe *tp, bool is_return) 621int traceprobe_set_print_fmt(struct trace_probe *tp, bool is_return)
657{ 622{
658 int len; 623 int len;
659 char *print_fmt; 624 char *print_fmt;
@@ -670,3 +635,28 @@ int set_print_fmt(struct trace_probe *tp, bool is_return)
670 635
671 return 0; 636 return 0;
672} 637}
638
639int traceprobe_define_arg_fields(struct trace_event_call *event_call,
640 size_t offset, struct trace_probe *tp)
641{
642 int ret, i;
643
644 /* Set argument names as fields */
645 for (i = 0; i < tp->nr_args; i++) {
646 struct probe_arg *parg = &tp->args[i];
647 const char *fmt = parg->type->fmttype;
648 int size = parg->type->size;
649
650 if (parg->fmt)
651 fmt = parg->fmt;
652 if (parg->count)
653 size *= parg->count;
654 ret = trace_define_field(event_call, fmt, parg->name,
655 offset + parg->offset, size,
656 parg->type->is_signed,
657 FILTER_OTHER);
658 if (ret)
659 return ret;
660 }
661 return 0;
662}
diff --git a/kernel/trace/trace_probe.h b/kernel/trace/trace_probe.h
index 5f52668e165d..974afc1a3e73 100644
--- a/kernel/trace/trace_probe.h
+++ b/kernel/trace/trace_probe.h
@@ -23,6 +23,7 @@
23#include <linux/stringify.h> 23#include <linux/stringify.h>
24#include <linux/limits.h> 24#include <linux/limits.h>
25#include <linux/uaccess.h> 25#include <linux/uaccess.h>
26#include <linux/bitops.h>
26#include <asm/bitsperlong.h> 27#include <asm/bitsperlong.h>
27 28
28#include "trace.h" 29#include "trace.h"
@@ -30,6 +31,7 @@
30 31
31#define MAX_TRACE_ARGS 128 32#define MAX_TRACE_ARGS 128
32#define MAX_ARGSTR_LEN 63 33#define MAX_ARGSTR_LEN 63
34#define MAX_ARRAY_LEN 64
33#define MAX_STRING_SIZE PATH_MAX 35#define MAX_STRING_SIZE PATH_MAX
34 36
35/* Reserved field names */ 37/* Reserved field names */
@@ -54,50 +56,74 @@
54#define TP_FLAG_PROFILE 2 56#define TP_FLAG_PROFILE 2
55#define TP_FLAG_REGISTERED 4 57#define TP_FLAG_REGISTERED 4
56 58
59/* data_loc: data location, compatible with u32 */
60#define make_data_loc(len, offs) \
61 (((u32)(len) << 16) | ((u32)(offs) & 0xffff))
62#define get_loc_len(dl) ((u32)(dl) >> 16)
63#define get_loc_offs(dl) ((u32)(dl) & 0xffff)
57 64
58/* data_rloc: data relative location, compatible with u32 */ 65static nokprobe_inline void *get_loc_data(u32 *dl, void *ent)
59#define make_data_rloc(len, roffs) \
60 (((u32)(len) << 16) | ((u32)(roffs) & 0xffff))
61#define get_rloc_len(dl) ((u32)(dl) >> 16)
62#define get_rloc_offs(dl) ((u32)(dl) & 0xffff)
63
64/*
65 * Convert data_rloc to data_loc:
66 * data_rloc stores the offset from data_rloc itself, but data_loc
67 * stores the offset from event entry.
68 */
69#define convert_rloc_to_loc(dl, offs) ((u32)(dl) + (offs))
70
71static nokprobe_inline void *get_rloc_data(u32 *dl)
72{ 66{
73 return (u8 *)dl + get_rloc_offs(*dl); 67 return (u8 *)ent + get_loc_offs(*dl);
74} 68}
75 69
76/* For data_loc conversion */ 70static nokprobe_inline u32 update_data_loc(u32 loc, int consumed)
77static nokprobe_inline void *get_loc_data(u32 *dl, void *ent)
78{ 71{
79 return (u8 *)ent + get_rloc_offs(*dl); 72 u32 maxlen = get_loc_len(loc);
73 u32 offset = get_loc_offs(loc);
74
75 return make_data_loc(maxlen - consumed, offset + consumed);
80} 76}
81 77
82/* Data fetch function type */
83typedef void (*fetch_func_t)(struct pt_regs *, void *, void *);
84/* Printing function type */ 78/* Printing function type */
85typedef int (*print_type_func_t)(struct trace_seq *, const char *, void *, void *); 79typedef int (*print_type_func_t)(struct trace_seq *, void *, void *);
86 80
87/* Fetch types */ 81enum fetch_op {
88enum { 82 FETCH_OP_NOP = 0,
89 FETCH_MTD_reg = 0, 83 // Stage 1 (load) ops
90 FETCH_MTD_stack, 84 FETCH_OP_REG, /* Register : .param = offset */
91 FETCH_MTD_retval, 85 FETCH_OP_STACK, /* Stack : .param = index */
92 FETCH_MTD_comm, 86 FETCH_OP_STACKP, /* Stack pointer */
93 FETCH_MTD_memory, 87 FETCH_OP_RETVAL, /* Return value */
94 FETCH_MTD_symbol, 88 FETCH_OP_IMM, /* Immediate : .immediate */
95 FETCH_MTD_deref, 89 FETCH_OP_COMM, /* Current comm */
96 FETCH_MTD_bitfield, 90 FETCH_OP_ARG, /* Function argument : .param */
97 FETCH_MTD_file_offset, 91 FETCH_OP_FOFFS, /* File offset: .immediate */
98 FETCH_MTD_END, 92 // Stage 2 (dereference) op
93 FETCH_OP_DEREF, /* Dereference: .offset */
94 // Stage 3 (store) ops
95 FETCH_OP_ST_RAW, /* Raw: .size */
96 FETCH_OP_ST_MEM, /* Mem: .offset, .size */
97 FETCH_OP_ST_STRING, /* String: .offset, .size */
98 // Stage 4 (modify) op
99 FETCH_OP_MOD_BF, /* Bitfield: .basesize, .lshift, .rshift */
100 // Stage 5 (loop) op
101 FETCH_OP_LP_ARRAY, /* Array: .param = loop count */
102 FETCH_OP_END,
103 FETCH_NOP_SYMBOL, /* Unresolved Symbol holder */
99}; 104};
100 105
106struct fetch_insn {
107 enum fetch_op op;
108 union {
109 unsigned int param;
110 struct {
111 unsigned int size;
112 int offset;
113 };
114 struct {
115 unsigned char basesize;
116 unsigned char lshift;
117 unsigned char rshift;
118 };
119 unsigned long immediate;
120 void *data;
121 };
122};
123
124/* fetch + deref*N + store + mod + end <= 16, this allows N=12, enough */
125#define FETCH_INSN_MAX 16
126
101/* Fetch type information table */ 127/* Fetch type information table */
102struct fetch_type { 128struct fetch_type {
103 const char *name; /* Name of type */ 129 const char *name; /* Name of type */
@@ -106,13 +132,6 @@ struct fetch_type {
106 print_type_func_t print; /* Print functions */ 132 print_type_func_t print; /* Print functions */
107 const char *fmt; /* Fromat string */ 133 const char *fmt; /* Fromat string */
108 const char *fmttype; /* Name in format file */ 134 const char *fmttype; /* Name in format file */
109 /* Fetch functions */
110 fetch_func_t fetch[FETCH_MTD_END];
111};
112
113struct fetch_param {
114 fetch_func_t fn;
115 void *data;
116}; 135};
117 136
118/* For defining macros, define string/string_size types */ 137/* For defining macros, define string/string_size types */
@@ -124,8 +143,7 @@ typedef u32 string_size;
124 143
125/* Printing in basic type function template */ 144/* Printing in basic type function template */
126#define DECLARE_BASIC_PRINT_TYPE_FUNC(type) \ 145#define DECLARE_BASIC_PRINT_TYPE_FUNC(type) \
127int PRINT_TYPE_FUNC_NAME(type)(struct trace_seq *s, const char *name, \ 146int PRINT_TYPE_FUNC_NAME(type)(struct trace_seq *s, void *data, void *ent);\
128 void *data, void *ent); \
129extern const char PRINT_TYPE_FMT_NAME(type)[] 147extern const char PRINT_TYPE_FMT_NAME(type)[]
130 148
131DECLARE_BASIC_PRINT_TYPE_FUNC(u8); 149DECLARE_BASIC_PRINT_TYPE_FUNC(u8);
@@ -142,57 +160,7 @@ DECLARE_BASIC_PRINT_TYPE_FUNC(x32);
142DECLARE_BASIC_PRINT_TYPE_FUNC(x64); 160DECLARE_BASIC_PRINT_TYPE_FUNC(x64);
143 161
144DECLARE_BASIC_PRINT_TYPE_FUNC(string); 162DECLARE_BASIC_PRINT_TYPE_FUNC(string);
145 163DECLARE_BASIC_PRINT_TYPE_FUNC(symbol);
146#define FETCH_FUNC_NAME(method, type) fetch_##method##_##type
147
148/* Declare macro for basic types */
149#define DECLARE_FETCH_FUNC(method, type) \
150extern void FETCH_FUNC_NAME(method, type)(struct pt_regs *regs, \
151 void *data, void *dest)
152
153#define DECLARE_BASIC_FETCH_FUNCS(method) \
154DECLARE_FETCH_FUNC(method, u8); \
155DECLARE_FETCH_FUNC(method, u16); \
156DECLARE_FETCH_FUNC(method, u32); \
157DECLARE_FETCH_FUNC(method, u64)
158
159DECLARE_BASIC_FETCH_FUNCS(reg);
160#define fetch_reg_string NULL
161#define fetch_reg_string_size NULL
162
163DECLARE_BASIC_FETCH_FUNCS(retval);
164#define fetch_retval_string NULL
165#define fetch_retval_string_size NULL
166
167DECLARE_BASIC_FETCH_FUNCS(symbol);
168DECLARE_FETCH_FUNC(symbol, string);
169DECLARE_FETCH_FUNC(symbol, string_size);
170
171DECLARE_BASIC_FETCH_FUNCS(deref);
172DECLARE_FETCH_FUNC(deref, string);
173DECLARE_FETCH_FUNC(deref, string_size);
174
175DECLARE_BASIC_FETCH_FUNCS(bitfield);
176#define fetch_bitfield_string NULL
177#define fetch_bitfield_string_size NULL
178
179/* comm only makes sense as a string */
180#define fetch_comm_u8 NULL
181#define fetch_comm_u16 NULL
182#define fetch_comm_u32 NULL
183#define fetch_comm_u64 NULL
184DECLARE_FETCH_FUNC(comm, string);
185DECLARE_FETCH_FUNC(comm, string_size);
186
187/*
188 * Define macro for basic types - we don't need to define s* types, because
189 * we have to care only about bitwidth at recording time.
190 */
191#define DEFINE_BASIC_FETCH_FUNCS(method) \
192DEFINE_FETCH_##method(u8) \
193DEFINE_FETCH_##method(u16) \
194DEFINE_FETCH_##method(u32) \
195DEFINE_FETCH_##method(u64)
196 164
197/* Default (unsigned long) fetch type */ 165/* Default (unsigned long) fetch type */
198#define __DEFAULT_FETCH_TYPE(t) x##t 166#define __DEFAULT_FETCH_TYPE(t) x##t
@@ -200,8 +168,9 @@ DEFINE_FETCH_##method(u64)
200#define DEFAULT_FETCH_TYPE _DEFAULT_FETCH_TYPE(BITS_PER_LONG) 168#define DEFAULT_FETCH_TYPE _DEFAULT_FETCH_TYPE(BITS_PER_LONG)
201#define DEFAULT_FETCH_TYPE_STR __stringify(DEFAULT_FETCH_TYPE) 169#define DEFAULT_FETCH_TYPE_STR __stringify(DEFAULT_FETCH_TYPE)
202 170
203#define ASSIGN_FETCH_FUNC(method, type) \ 171#define __ADDR_FETCH_TYPE(t) u##t
204 [FETCH_MTD_##method] = FETCH_FUNC_NAME(method, type) 172#define _ADDR_FETCH_TYPE(t) __ADDR_FETCH_TYPE(t)
173#define ADDR_FETCH_TYPE _ADDR_FETCH_TYPE(BITS_PER_LONG)
205 174
206#define __ASSIGN_FETCH_TYPE(_name, ptype, ftype, _size, sign, _fmttype) \ 175#define __ASSIGN_FETCH_TYPE(_name, ptype, ftype, _size, sign, _fmttype) \
207 {.name = _name, \ 176 {.name = _name, \
@@ -210,64 +179,23 @@ DEFINE_FETCH_##method(u64)
210 .print = PRINT_TYPE_FUNC_NAME(ptype), \ 179 .print = PRINT_TYPE_FUNC_NAME(ptype), \
211 .fmt = PRINT_TYPE_FMT_NAME(ptype), \ 180 .fmt = PRINT_TYPE_FMT_NAME(ptype), \
212 .fmttype = _fmttype, \ 181 .fmttype = _fmttype, \
213 .fetch = { \
214ASSIGN_FETCH_FUNC(reg, ftype), \
215ASSIGN_FETCH_FUNC(stack, ftype), \
216ASSIGN_FETCH_FUNC(retval, ftype), \
217ASSIGN_FETCH_FUNC(comm, ftype), \
218ASSIGN_FETCH_FUNC(memory, ftype), \
219ASSIGN_FETCH_FUNC(symbol, ftype), \
220ASSIGN_FETCH_FUNC(deref, ftype), \
221ASSIGN_FETCH_FUNC(bitfield, ftype), \
222ASSIGN_FETCH_FUNC(file_offset, ftype), \
223 } \
224 } 182 }
225 183#define _ASSIGN_FETCH_TYPE(_name, ptype, ftype, _size, sign, _fmttype) \
184 __ASSIGN_FETCH_TYPE(_name, ptype, ftype, _size, sign, #_fmttype)
226#define ASSIGN_FETCH_TYPE(ptype, ftype, sign) \ 185#define ASSIGN_FETCH_TYPE(ptype, ftype, sign) \
227 __ASSIGN_FETCH_TYPE(#ptype, ptype, ftype, sizeof(ftype), sign, #ptype) 186 _ASSIGN_FETCH_TYPE(#ptype, ptype, ftype, sizeof(ftype), sign, ptype)
228 187
229/* If ptype is an alias of atype, use this macro (show atype in format) */ 188/* If ptype is an alias of atype, use this macro (show atype in format) */
230#define ASSIGN_FETCH_TYPE_ALIAS(ptype, atype, ftype, sign) \ 189#define ASSIGN_FETCH_TYPE_ALIAS(ptype, atype, ftype, sign) \
231 __ASSIGN_FETCH_TYPE(#ptype, ptype, ftype, sizeof(ftype), sign, #atype) 190 _ASSIGN_FETCH_TYPE(#ptype, ptype, ftype, sizeof(ftype), sign, atype)
232 191
233#define ASSIGN_FETCH_TYPE_END {} 192#define ASSIGN_FETCH_TYPE_END {}
234 193#define MAX_ARRAY_LEN 64
235#define FETCH_TYPE_STRING 0
236#define FETCH_TYPE_STRSIZE 1
237 194
238#ifdef CONFIG_KPROBE_EVENTS 195#ifdef CONFIG_KPROBE_EVENTS
239struct symbol_cache;
240unsigned long update_symbol_cache(struct symbol_cache *sc);
241void free_symbol_cache(struct symbol_cache *sc);
242struct symbol_cache *alloc_symbol_cache(const char *sym, long offset);
243bool trace_kprobe_on_func_entry(struct trace_event_call *call); 196bool trace_kprobe_on_func_entry(struct trace_event_call *call);
244bool trace_kprobe_error_injectable(struct trace_event_call *call); 197bool trace_kprobe_error_injectable(struct trace_event_call *call);
245#else 198#else
246/* uprobes do not support symbol fetch methods */
247#define fetch_symbol_u8 NULL
248#define fetch_symbol_u16 NULL
249#define fetch_symbol_u32 NULL
250#define fetch_symbol_u64 NULL
251#define fetch_symbol_string NULL
252#define fetch_symbol_string_size NULL
253
254struct symbol_cache {
255};
256static inline unsigned long __used update_symbol_cache(struct symbol_cache *sc)
257{
258 return 0;
259}
260
261static inline void __used free_symbol_cache(struct symbol_cache *sc)
262{
263}
264
265static inline struct symbol_cache * __used
266alloc_symbol_cache(const char *sym, long offset)
267{
268 return NULL;
269}
270
271static inline bool trace_kprobe_on_func_entry(struct trace_event_call *call) 199static inline bool trace_kprobe_on_func_entry(struct trace_event_call *call)
272{ 200{
273 return false; 201 return false;
@@ -280,11 +208,13 @@ static inline bool trace_kprobe_error_injectable(struct trace_event_call *call)
280#endif /* CONFIG_KPROBE_EVENTS */ 208#endif /* CONFIG_KPROBE_EVENTS */
281 209
282struct probe_arg { 210struct probe_arg {
283 struct fetch_param fetch; 211 struct fetch_insn *code;
284 struct fetch_param fetch_size; 212 bool dynamic;/* Dynamic array (string) is used */
285 unsigned int offset; /* Offset from argument entry */ 213 unsigned int offset; /* Offset from argument entry */
214 unsigned int count; /* Array count */
286 const char *name; /* Name of this argument */ 215 const char *name; /* Name of this argument */
287 const char *comm; /* Command of this argument */ 216 const char *comm; /* Command of this argument */
217 char *fmt; /* Format string if needed */
288 const struct fetch_type *type; /* Type of this argument */ 218 const struct fetch_type *type; /* Type of this argument */
289}; 219};
290 220
@@ -313,12 +243,6 @@ static inline bool trace_probe_is_registered(struct trace_probe *tp)
313 return !!(tp->flags & TP_FLAG_REGISTERED); 243 return !!(tp->flags & TP_FLAG_REGISTERED);
314} 244}
315 245
316static nokprobe_inline void call_fetch(struct fetch_param *fprm,
317 struct pt_regs *regs, void *dest)
318{
319 return fprm->fn(regs, fprm->data, dest);
320}
321
322/* Check the name is good for event/group/fields */ 246/* Check the name is good for event/group/fields */
323static inline bool is_good_name(const char *name) 247static inline bool is_good_name(const char *name)
324{ 248{
@@ -343,67 +267,23 @@ find_event_file_link(struct trace_probe *tp, struct trace_event_file *file)
343 return NULL; 267 return NULL;
344} 268}
345 269
270#define TPARG_FL_RETURN BIT(0)
271#define TPARG_FL_KERNEL BIT(1)
272#define TPARG_FL_FENTRY BIT(2)
273#define TPARG_FL_MASK GENMASK(2, 0)
274
346extern int traceprobe_parse_probe_arg(char *arg, ssize_t *size, 275extern int traceprobe_parse_probe_arg(char *arg, ssize_t *size,
347 struct probe_arg *parg, bool is_return, bool is_kprobe, 276 struct probe_arg *parg, unsigned int flags);
348 const struct fetch_type *ftbl);
349 277
350extern int traceprobe_conflict_field_name(const char *name, 278extern int traceprobe_conflict_field_name(const char *name,
351 struct probe_arg *args, int narg); 279 struct probe_arg *args, int narg);
352 280
353extern void traceprobe_update_arg(struct probe_arg *arg); 281extern int traceprobe_update_arg(struct probe_arg *arg);
354extern void traceprobe_free_probe_arg(struct probe_arg *arg); 282extern void traceprobe_free_probe_arg(struct probe_arg *arg);
355 283
356extern int traceprobe_split_symbol_offset(char *symbol, long *offset); 284extern int traceprobe_split_symbol_offset(char *symbol, long *offset);
357 285
358/* Sum up total data length for dynamic arraies (strings) */ 286extern int traceprobe_set_print_fmt(struct trace_probe *tp, bool is_return);
359static nokprobe_inline int
360__get_data_size(struct trace_probe *tp, struct pt_regs *regs)
361{
362 int i, ret = 0;
363 u32 len;
364
365 for (i = 0; i < tp->nr_args; i++)
366 if (unlikely(tp->args[i].fetch_size.fn)) {
367 call_fetch(&tp->args[i].fetch_size, regs, &len);
368 ret += len;
369 }
370
371 return ret;
372}
373
374/* Store the value of each argument */
375static nokprobe_inline void
376store_trace_args(int ent_size, struct trace_probe *tp, struct pt_regs *regs,
377 u8 *data, int maxlen)
378{
379 int i;
380 u32 end = tp->size;
381 u32 *dl; /* Data (relative) location */
382
383 for (i = 0; i < tp->nr_args; i++) {
384 if (unlikely(tp->args[i].fetch_size.fn)) {
385 /*
386 * First, we set the relative location and
387 * maximum data length to *dl
388 */
389 dl = (u32 *)(data + tp->args[i].offset);
390 *dl = make_data_rloc(maxlen, end - tp->args[i].offset);
391 /* Then try to fetch string or dynamic array data */
392 call_fetch(&tp->args[i].fetch, regs, dl);
393 /* Reduce maximum length */
394 end += get_rloc_len(*dl);
395 maxlen -= get_rloc_len(*dl);
396 /* Trick here, convert data_rloc to data_loc */
397 *dl = convert_rloc_to_loc(*dl,
398 ent_size + tp->args[i].offset);
399 } else
400 /* Just fetching data normally */
401 call_fetch(&tp->args[i].fetch, regs,
402 data + tp->args[i].offset);
403 }
404}
405
406extern int set_print_fmt(struct trace_probe *tp, bool is_return);
407 287
408#ifdef CONFIG_PERF_EVENTS 288#ifdef CONFIG_PERF_EVENTS
409extern struct trace_event_call * 289extern struct trace_event_call *
@@ -412,6 +292,9 @@ create_local_trace_kprobe(char *func, void *addr, unsigned long offs,
412extern void destroy_local_trace_kprobe(struct trace_event_call *event_call); 292extern void destroy_local_trace_kprobe(struct trace_event_call *event_call);
413 293
414extern struct trace_event_call * 294extern struct trace_event_call *
415create_local_trace_uprobe(char *name, unsigned long offs, bool is_return); 295create_local_trace_uprobe(char *name, unsigned long offs,
296 unsigned long ref_ctr_offset, bool is_return);
416extern void destroy_local_trace_uprobe(struct trace_event_call *event_call); 297extern void destroy_local_trace_uprobe(struct trace_event_call *event_call);
417#endif 298#endif
299extern int traceprobe_define_arg_fields(struct trace_event_call *event_call,
300 size_t offset, struct trace_probe *tp);
diff --git a/kernel/trace/trace_probe_tmpl.h b/kernel/trace/trace_probe_tmpl.h
new file mode 100644
index 000000000000..5c56afc17cf8
--- /dev/null
+++ b/kernel/trace/trace_probe_tmpl.h
@@ -0,0 +1,216 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Traceprobe fetch helper inlines
4 */
5
6static nokprobe_inline void
7fetch_store_raw(unsigned long val, struct fetch_insn *code, void *buf)
8{
9 switch (code->size) {
10 case 1:
11 *(u8 *)buf = (u8)val;
12 break;
13 case 2:
14 *(u16 *)buf = (u16)val;
15 break;
16 case 4:
17 *(u32 *)buf = (u32)val;
18 break;
19 case 8:
20 //TBD: 32bit signed
21 *(u64 *)buf = (u64)val;
22 break;
23 default:
24 *(unsigned long *)buf = val;
25 }
26}
27
28static nokprobe_inline void
29fetch_apply_bitfield(struct fetch_insn *code, void *buf)
30{
31 switch (code->basesize) {
32 case 1:
33 *(u8 *)buf <<= code->lshift;
34 *(u8 *)buf >>= code->rshift;
35 break;
36 case 2:
37 *(u16 *)buf <<= code->lshift;
38 *(u16 *)buf >>= code->rshift;
39 break;
40 case 4:
41 *(u32 *)buf <<= code->lshift;
42 *(u32 *)buf >>= code->rshift;
43 break;
44 case 8:
45 *(u64 *)buf <<= code->lshift;
46 *(u64 *)buf >>= code->rshift;
47 break;
48 }
49}
50
51/*
52 * These functions must be defined for each callsite.
53 * Return consumed dynamic data size (>= 0), or error (< 0).
54 * If dest is NULL, don't store result and return required dynamic data size.
55 */
56static int
57process_fetch_insn(struct fetch_insn *code, struct pt_regs *regs,
58 void *dest, void *base);
59static nokprobe_inline int fetch_store_strlen(unsigned long addr);
60static nokprobe_inline int
61fetch_store_string(unsigned long addr, void *dest, void *base);
62static nokprobe_inline int
63probe_mem_read(void *dest, void *src, size_t size);
64
65/* From the 2nd stage, routine is same */
66static nokprobe_inline int
67process_fetch_insn_bottom(struct fetch_insn *code, unsigned long val,
68 void *dest, void *base)
69{
70 struct fetch_insn *s3 = NULL;
71 int total = 0, ret = 0, i = 0;
72 u32 loc = 0;
73 unsigned long lval = val;
74
75stage2:
76 /* 2nd stage: dereference memory if needed */
77 while (code->op == FETCH_OP_DEREF) {
78 lval = val;
79 ret = probe_mem_read(&val, (void *)val + code->offset,
80 sizeof(val));
81 if (ret)
82 return ret;
83 code++;
84 }
85
86 s3 = code;
87stage3:
88 /* 3rd stage: store value to buffer */
89 if (unlikely(!dest)) {
90 if (code->op == FETCH_OP_ST_STRING) {
91 ret += fetch_store_strlen(val + code->offset);
92 code++;
93 goto array;
94 } else
95 return -EILSEQ;
96 }
97
98 switch (code->op) {
99 case FETCH_OP_ST_RAW:
100 fetch_store_raw(val, code, dest);
101 break;
102 case FETCH_OP_ST_MEM:
103 probe_mem_read(dest, (void *)val + code->offset, code->size);
104 break;
105 case FETCH_OP_ST_STRING:
106 loc = *(u32 *)dest;
107 ret = fetch_store_string(val + code->offset, dest, base);
108 break;
109 default:
110 return -EILSEQ;
111 }
112 code++;
113
114 /* 4th stage: modify stored value if needed */
115 if (code->op == FETCH_OP_MOD_BF) {
116 fetch_apply_bitfield(code, dest);
117 code++;
118 }
119
120array:
121 /* the last stage: Loop on array */
122 if (code->op == FETCH_OP_LP_ARRAY) {
123 total += ret;
124 if (++i < code->param) {
125 code = s3;
126 if (s3->op != FETCH_OP_ST_STRING) {
127 dest += s3->size;
128 val += s3->size;
129 goto stage3;
130 }
131 code--;
132 val = lval + sizeof(char *);
133 if (dest) {
134 dest += sizeof(u32);
135 *(u32 *)dest = update_data_loc(loc, ret);
136 }
137 goto stage2;
138 }
139 code++;
140 ret = total;
141 }
142
143 return code->op == FETCH_OP_END ? ret : -EILSEQ;
144}
145
146/* Sum up total data length for dynamic arraies (strings) */
147static nokprobe_inline int
148__get_data_size(struct trace_probe *tp, struct pt_regs *regs)
149{
150 struct probe_arg *arg;
151 int i, len, ret = 0;
152
153 for (i = 0; i < tp->nr_args; i++) {
154 arg = tp->args + i;
155 if (unlikely(arg->dynamic)) {
156 len = process_fetch_insn(arg->code, regs, NULL, NULL);
157 if (len > 0)
158 ret += len;
159 }
160 }
161
162 return ret;
163}
164
165/* Store the value of each argument */
166static nokprobe_inline void
167store_trace_args(void *data, struct trace_probe *tp, struct pt_regs *regs,
168 int header_size, int maxlen)
169{
170 struct probe_arg *arg;
171 void *base = data - header_size;
172 void *dyndata = data + tp->size;
173 u32 *dl; /* Data location */
174 int ret, i;
175
176 for (i = 0; i < tp->nr_args; i++) {
177 arg = tp->args + i;
178 dl = data + arg->offset;
179 /* Point the dynamic data area if needed */
180 if (unlikely(arg->dynamic))
181 *dl = make_data_loc(maxlen, dyndata - base);
182 ret = process_fetch_insn(arg->code, regs, dl, base);
183 if (unlikely(ret < 0 && arg->dynamic))
184 *dl = make_data_loc(0, dyndata - base);
185 else
186 dyndata += ret;
187 }
188}
189
190static inline int
191print_probe_args(struct trace_seq *s, struct probe_arg *args, int nr_args,
192 u8 *data, void *field)
193{
194 void *p;
195 int i, j;
196
197 for (i = 0; i < nr_args; i++) {
198 struct probe_arg *a = args + i;
199
200 trace_seq_printf(s, " %s=", a->name);
201 if (likely(!a->count)) {
202 if (!a->type->print(s, data + a->offset, field))
203 return -ENOMEM;
204 continue;
205 }
206 trace_seq_putc(s, '{');
207 p = data + a->offset;
208 for (j = 0; j < a->count; j++) {
209 if (!a->type->print(s, p, field))
210 return -ENOMEM;
211 trace_seq_putc(s, j == a->count - 1 ? '}' : ',');
212 p += a->type->size;
213 }
214 }
215 return 0;
216}
diff --git a/kernel/trace/trace_stack.c b/kernel/trace/trace_stack.c
index 4237eba4ef20..2b0d1ee3241c 100644
--- a/kernel/trace/trace_stack.c
+++ b/kernel/trace/trace_stack.c
@@ -111,7 +111,7 @@ check_stack(unsigned long ip, unsigned long *stack)
111 stack_trace_max_size = this_size; 111 stack_trace_max_size = this_size;
112 112
113 stack_trace_max.nr_entries = 0; 113 stack_trace_max.nr_entries = 0;
114 stack_trace_max.skip = 3; 114 stack_trace_max.skip = 0;
115 115
116 save_stack_trace(&stack_trace_max); 116 save_stack_trace(&stack_trace_max);
117 117
diff --git a/kernel/trace/trace_uprobe.c b/kernel/trace/trace_uprobe.c
index e696667da29a..31ea48eceda1 100644
--- a/kernel/trace/trace_uprobe.c
+++ b/kernel/trace/trace_uprobe.c
@@ -15,6 +15,7 @@
15#include <linux/rculist.h> 15#include <linux/rculist.h>
16 16
17#include "trace_probe.h" 17#include "trace_probe.h"
18#include "trace_probe_tmpl.h"
18 19
19#define UPROBE_EVENT_SYSTEM "uprobes" 20#define UPROBE_EVENT_SYSTEM "uprobes"
20 21
@@ -47,6 +48,7 @@ struct trace_uprobe {
47 struct inode *inode; 48 struct inode *inode;
48 char *filename; 49 char *filename;
49 unsigned long offset; 50 unsigned long offset;
51 unsigned long ref_ctr_offset;
50 unsigned long nhit; 52 unsigned long nhit;
51 struct trace_probe tp; 53 struct trace_probe tp;
52}; 54};
@@ -98,74 +100,52 @@ static unsigned long get_user_stack_nth(struct pt_regs *regs, unsigned int n)
98/* 100/*
99 * Uprobes-specific fetch functions 101 * Uprobes-specific fetch functions
100 */ 102 */
101#define DEFINE_FETCH_stack(type) \ 103static nokprobe_inline int
102static void FETCH_FUNC_NAME(stack, type)(struct pt_regs *regs, \ 104probe_mem_read(void *dest, void *src, size_t size)
103 void *offset, void *dest) \ 105{
104{ \ 106 void __user *vaddr = (void __force __user *)src;
105 *(type *)dest = (type)get_user_stack_nth(regs, \ 107
106 ((unsigned long)offset)); \ 108 return copy_from_user(dest, vaddr, size) ? -EFAULT : 0;
107}
108DEFINE_BASIC_FETCH_FUNCS(stack)
109/* No string on the stack entry */
110#define fetch_stack_string NULL
111#define fetch_stack_string_size NULL
112
113#define DEFINE_FETCH_memory(type) \
114static void FETCH_FUNC_NAME(memory, type)(struct pt_regs *regs, \
115 void *addr, void *dest) \
116{ \
117 type retval; \
118 void __user *vaddr = (void __force __user *) addr; \
119 \
120 if (copy_from_user(&retval, vaddr, sizeof(type))) \
121 *(type *)dest = 0; \
122 else \
123 *(type *) dest = retval; \
124} 109}
125DEFINE_BASIC_FETCH_FUNCS(memory)
126/* 110/*
127 * Fetch a null-terminated string. Caller MUST set *(u32 *)dest with max 111 * Fetch a null-terminated string. Caller MUST set *(u32 *)dest with max
128 * length and relative data location. 112 * length and relative data location.
129 */ 113 */
130static void FETCH_FUNC_NAME(memory, string)(struct pt_regs *regs, 114static nokprobe_inline int
131 void *addr, void *dest) 115fetch_store_string(unsigned long addr, void *dest, void *base)
132{ 116{
133 long ret; 117 long ret;
134 u32 rloc = *(u32 *)dest; 118 u32 loc = *(u32 *)dest;
135 int maxlen = get_rloc_len(rloc); 119 int maxlen = get_loc_len(loc);
136 u8 *dst = get_rloc_data(dest); 120 u8 *dst = get_loc_data(dest, base);
137 void __user *src = (void __force __user *) addr; 121 void __user *src = (void __force __user *) addr;
138 122
139 if (!maxlen) 123 if (unlikely(!maxlen))
140 return; 124 return -ENOMEM;
141 125
142 ret = strncpy_from_user(dst, src, maxlen); 126 ret = strncpy_from_user(dst, src, maxlen);
143 if (ret == maxlen) 127 if (ret >= 0) {
144 dst[--ret] = '\0'; 128 if (ret == maxlen)
145 129 dst[ret - 1] = '\0';
146 if (ret < 0) { /* Failed to fetch string */ 130 *(u32 *)dest = make_data_loc(ret, (void *)dst - base);
147 ((u8 *)get_rloc_data(dest))[0] = '\0';
148 *(u32 *)dest = make_data_rloc(0, get_rloc_offs(rloc));
149 } else {
150 *(u32 *)dest = make_data_rloc(ret, get_rloc_offs(rloc));
151 } 131 }
132
133 return ret;
152} 134}
153 135
154static void FETCH_FUNC_NAME(memory, string_size)(struct pt_regs *regs, 136/* Return the length of string -- including null terminal byte */
155 void *addr, void *dest) 137static nokprobe_inline int
138fetch_store_strlen(unsigned long addr)
156{ 139{
157 int len; 140 int len;
158 void __user *vaddr = (void __force __user *) addr; 141 void __user *vaddr = (void __force __user *) addr;
159 142
160 len = strnlen_user(vaddr, MAX_STRING_SIZE); 143 len = strnlen_user(vaddr, MAX_STRING_SIZE);
161 144
162 if (len == 0 || len > MAX_STRING_SIZE) /* Failed to check length */ 145 return (len > MAX_STRING_SIZE) ? 0 : len;
163 *(u32 *)dest = 0;
164 else
165 *(u32 *)dest = len;
166} 146}
167 147
168static unsigned long translate_user_vaddr(void *file_offset) 148static unsigned long translate_user_vaddr(unsigned long file_offset)
169{ 149{
170 unsigned long base_addr; 150 unsigned long base_addr;
171 struct uprobe_dispatch_data *udd; 151 struct uprobe_dispatch_data *udd;
@@ -173,44 +153,44 @@ static unsigned long translate_user_vaddr(void *file_offset)
173 udd = (void *) current->utask->vaddr; 153 udd = (void *) current->utask->vaddr;
174 154
175 base_addr = udd->bp_addr - udd->tu->offset; 155 base_addr = udd->bp_addr - udd->tu->offset;
176 return base_addr + (unsigned long)file_offset; 156 return base_addr + file_offset;
177} 157}
178 158
179#define DEFINE_FETCH_file_offset(type) \ 159/* Note that we don't verify it, since the code does not come from user space */
180static void FETCH_FUNC_NAME(file_offset, type)(struct pt_regs *regs, \ 160static int
181 void *offset, void *dest)\ 161process_fetch_insn(struct fetch_insn *code, struct pt_regs *regs, void *dest,
182{ \ 162 void *base)
183 void *vaddr = (void *)translate_user_vaddr(offset); \ 163{
184 \ 164 unsigned long val;
185 FETCH_FUNC_NAME(memory, type)(regs, vaddr, dest); \ 165
166 /* 1st stage: get value from context */
167 switch (code->op) {
168 case FETCH_OP_REG:
169 val = regs_get_register(regs, code->param);
170 break;
171 case FETCH_OP_STACK:
172 val = get_user_stack_nth(regs, code->param);
173 break;
174 case FETCH_OP_STACKP:
175 val = user_stack_pointer(regs);
176 break;
177 case FETCH_OP_RETVAL:
178 val = regs_return_value(regs);
179 break;
180 case FETCH_OP_IMM:
181 val = code->immediate;
182 break;
183 case FETCH_OP_FOFFS:
184 val = translate_user_vaddr(code->immediate);
185 break;
186 default:
187 return -EILSEQ;
188 }
189 code++;
190
191 return process_fetch_insn_bottom(code, val, dest, base);
186} 192}
187DEFINE_BASIC_FETCH_FUNCS(file_offset) 193NOKPROBE_SYMBOL(process_fetch_insn)
188DEFINE_FETCH_file_offset(string)
189DEFINE_FETCH_file_offset(string_size)
190
191/* Fetch type information table */
192static const struct fetch_type uprobes_fetch_type_table[] = {
193 /* Special types */
194 [FETCH_TYPE_STRING] = __ASSIGN_FETCH_TYPE("string", string, string,
195 sizeof(u32), 1, "__data_loc char[]"),
196 [FETCH_TYPE_STRSIZE] = __ASSIGN_FETCH_TYPE("string_size", u32,
197 string_size, sizeof(u32), 0, "u32"),
198 /* Basic types */
199 ASSIGN_FETCH_TYPE(u8, u8, 0),
200 ASSIGN_FETCH_TYPE(u16, u16, 0),
201 ASSIGN_FETCH_TYPE(u32, u32, 0),
202 ASSIGN_FETCH_TYPE(u64, u64, 0),
203 ASSIGN_FETCH_TYPE(s8, u8, 1),
204 ASSIGN_FETCH_TYPE(s16, u16, 1),
205 ASSIGN_FETCH_TYPE(s32, u32, 1),
206 ASSIGN_FETCH_TYPE(s64, u64, 1),
207 ASSIGN_FETCH_TYPE_ALIAS(x8, u8, u8, 0),
208 ASSIGN_FETCH_TYPE_ALIAS(x16, u16, u16, 0),
209 ASSIGN_FETCH_TYPE_ALIAS(x32, u32, u32, 0),
210 ASSIGN_FETCH_TYPE_ALIAS(x64, u64, u64, 0),
211
212 ASSIGN_FETCH_TYPE_END
213};
214 194
215static inline void init_trace_uprobe_filter(struct trace_uprobe_filter *filter) 195static inline void init_trace_uprobe_filter(struct trace_uprobe_filter *filter)
216{ 196{
@@ -311,6 +291,35 @@ static int unregister_trace_uprobe(struct trace_uprobe *tu)
311 return 0; 291 return 0;
312} 292}
313 293
294/*
295 * Uprobe with multiple reference counter is not allowed. i.e.
296 * If inode and offset matches, reference counter offset *must*
297 * match as well. Though, there is one exception: If user is
298 * replacing old trace_uprobe with new one(same group/event),
299 * then we allow same uprobe with new reference counter as far
300 * as the new one does not conflict with any other existing
301 * ones.
302 */
303static struct trace_uprobe *find_old_trace_uprobe(struct trace_uprobe *new)
304{
305 struct trace_uprobe *tmp, *old = NULL;
306 struct inode *new_inode = d_real_inode(new->path.dentry);
307
308 old = find_probe_event(trace_event_name(&new->tp.call),
309 new->tp.call.class->system);
310
311 list_for_each_entry(tmp, &uprobe_list, list) {
312 if ((old ? old != tmp : true) &&
313 new_inode == d_real_inode(tmp->path.dentry) &&
314 new->offset == tmp->offset &&
315 new->ref_ctr_offset != tmp->ref_ctr_offset) {
316 pr_warn("Reference counter offset mismatch.");
317 return ERR_PTR(-EINVAL);
318 }
319 }
320 return old;
321}
322
314/* Register a trace_uprobe and probe_event */ 323/* Register a trace_uprobe and probe_event */
315static int register_trace_uprobe(struct trace_uprobe *tu) 324static int register_trace_uprobe(struct trace_uprobe *tu)
316{ 325{
@@ -320,8 +329,12 @@ static int register_trace_uprobe(struct trace_uprobe *tu)
320 mutex_lock(&uprobe_lock); 329 mutex_lock(&uprobe_lock);
321 330
322 /* register as an event */ 331 /* register as an event */
323 old_tu = find_probe_event(trace_event_name(&tu->tp.call), 332 old_tu = find_old_trace_uprobe(tu);
324 tu->tp.call.class->system); 333 if (IS_ERR(old_tu)) {
334 ret = PTR_ERR(old_tu);
335 goto end;
336 }
337
325 if (old_tu) { 338 if (old_tu) {
326 /* delete old event */ 339 /* delete old event */
327 ret = unregister_trace_uprobe(old_tu); 340 ret = unregister_trace_uprobe(old_tu);
@@ -352,10 +365,10 @@ end:
352static int create_trace_uprobe(int argc, char **argv) 365static int create_trace_uprobe(int argc, char **argv)
353{ 366{
354 struct trace_uprobe *tu; 367 struct trace_uprobe *tu;
355 char *arg, *event, *group, *filename; 368 char *arg, *event, *group, *filename, *rctr, *rctr_end;
356 char buf[MAX_EVENT_NAME_LEN]; 369 char buf[MAX_EVENT_NAME_LEN];
357 struct path path; 370 struct path path;
358 unsigned long offset; 371 unsigned long offset, ref_ctr_offset;
359 bool is_delete, is_return; 372 bool is_delete, is_return;
360 int i, ret; 373 int i, ret;
361 374
@@ -364,6 +377,7 @@ static int create_trace_uprobe(int argc, char **argv)
364 is_return = false; 377 is_return = false;
365 event = NULL; 378 event = NULL;
366 group = NULL; 379 group = NULL;
380 ref_ctr_offset = 0;
367 381
368 /* argc must be >= 1 */ 382 /* argc must be >= 1 */
369 if (argv[0][0] == '-') 383 if (argv[0][0] == '-')
@@ -438,6 +452,26 @@ static int create_trace_uprobe(int argc, char **argv)
438 goto fail_address_parse; 452 goto fail_address_parse;
439 } 453 }
440 454
455 /* Parse reference counter offset if specified. */
456 rctr = strchr(arg, '(');
457 if (rctr) {
458 rctr_end = strchr(rctr, ')');
459 if (rctr > rctr_end || *(rctr_end + 1) != 0) {
460 ret = -EINVAL;
461 pr_info("Invalid reference counter offset.\n");
462 goto fail_address_parse;
463 }
464
465 *rctr++ = '\0';
466 *rctr_end = '\0';
467 ret = kstrtoul(rctr, 0, &ref_ctr_offset);
468 if (ret) {
469 pr_info("Invalid reference counter offset.\n");
470 goto fail_address_parse;
471 }
472 }
473
474 /* Parse uprobe offset. */
441 ret = kstrtoul(arg, 0, &offset); 475 ret = kstrtoul(arg, 0, &offset);
442 if (ret) 476 if (ret)
443 goto fail_address_parse; 477 goto fail_address_parse;
@@ -472,6 +506,7 @@ static int create_trace_uprobe(int argc, char **argv)
472 goto fail_address_parse; 506 goto fail_address_parse;
473 } 507 }
474 tu->offset = offset; 508 tu->offset = offset;
509 tu->ref_ctr_offset = ref_ctr_offset;
475 tu->path = path; 510 tu->path = path;
476 tu->filename = kstrdup(filename, GFP_KERNEL); 511 tu->filename = kstrdup(filename, GFP_KERNEL);
477 512
@@ -522,8 +557,7 @@ static int create_trace_uprobe(int argc, char **argv)
522 557
523 /* Parse fetch argument */ 558 /* Parse fetch argument */
524 ret = traceprobe_parse_probe_arg(arg, &tu->tp.size, parg, 559 ret = traceprobe_parse_probe_arg(arg, &tu->tp.size, parg,
525 is_return, false, 560 is_return ? TPARG_FL_RETURN : 0);
526 uprobes_fetch_type_table);
527 if (ret) { 561 if (ret) {
528 pr_info("Parse error at argument[%d]. (%d)\n", i, ret); 562 pr_info("Parse error at argument[%d]. (%d)\n", i, ret);
529 goto error; 563 goto error;
@@ -590,6 +624,9 @@ static int probes_seq_show(struct seq_file *m, void *v)
590 trace_event_name(&tu->tp.call), tu->filename, 624 trace_event_name(&tu->tp.call), tu->filename,
591 (int)(sizeof(void *) * 2), tu->offset); 625 (int)(sizeof(void *) * 2), tu->offset);
592 626
627 if (tu->ref_ctr_offset)
628 seq_printf(m, "(0x%lx)", tu->ref_ctr_offset);
629
593 for (i = 0; i < tu->tp.nr_args; i++) 630 for (i = 0; i < tu->tp.nr_args; i++)
594 seq_printf(m, " %s=%s", tu->tp.args[i].name, tu->tp.args[i].comm); 631 seq_printf(m, " %s=%s", tu->tp.args[i].name, tu->tp.args[i].comm);
595 632
@@ -833,7 +870,6 @@ print_uprobe_event(struct trace_iterator *iter, int flags, struct trace_event *e
833 struct trace_seq *s = &iter->seq; 870 struct trace_seq *s = &iter->seq;
834 struct trace_uprobe *tu; 871 struct trace_uprobe *tu;
835 u8 *data; 872 u8 *data;
836 int i;
837 873
838 entry = (struct uprobe_trace_entry_head *)iter->ent; 874 entry = (struct uprobe_trace_entry_head *)iter->ent;
839 tu = container_of(event, struct trace_uprobe, tp.call.event); 875 tu = container_of(event, struct trace_uprobe, tp.call.event);
@@ -850,12 +886,8 @@ print_uprobe_event(struct trace_iterator *iter, int flags, struct trace_event *e
850 data = DATAOF_TRACE_ENTRY(entry, false); 886 data = DATAOF_TRACE_ENTRY(entry, false);
851 } 887 }
852 888
853 for (i = 0; i < tu->tp.nr_args; i++) { 889 if (print_probe_args(s, tu->tp.args, tu->tp.nr_args, data, entry) < 0)
854 struct probe_arg *parg = &tu->tp.args[i]; 890 goto out;
855
856 if (!parg->type->print(s, parg->name, data + parg->offset, entry))
857 goto out;
858 }
859 891
860 trace_seq_putc(s, '\n'); 892 trace_seq_putc(s, '\n');
861 893
@@ -905,7 +937,13 @@ probe_event_enable(struct trace_uprobe *tu, struct trace_event_file *file,
905 937
906 tu->consumer.filter = filter; 938 tu->consumer.filter = filter;
907 tu->inode = d_real_inode(tu->path.dentry); 939 tu->inode = d_real_inode(tu->path.dentry);
908 ret = uprobe_register(tu->inode, tu->offset, &tu->consumer); 940 if (tu->ref_ctr_offset) {
941 ret = uprobe_register_refctr(tu->inode, tu->offset,
942 tu->ref_ctr_offset, &tu->consumer);
943 } else {
944 ret = uprobe_register(tu->inode, tu->offset, &tu->consumer);
945 }
946
909 if (ret) 947 if (ret)
910 goto err_buffer; 948 goto err_buffer;
911 949
@@ -958,7 +996,7 @@ probe_event_disable(struct trace_uprobe *tu, struct trace_event_file *file)
958 996
959static int uprobe_event_define_fields(struct trace_event_call *event_call) 997static int uprobe_event_define_fields(struct trace_event_call *event_call)
960{ 998{
961 int ret, i, size; 999 int ret, size;
962 struct uprobe_trace_entry_head field; 1000 struct uprobe_trace_entry_head field;
963 struct trace_uprobe *tu = event_call->data; 1001 struct trace_uprobe *tu = event_call->data;
964 1002
@@ -970,19 +1008,8 @@ static int uprobe_event_define_fields(struct trace_event_call *event_call)
970 DEFINE_FIELD(unsigned long, vaddr[0], FIELD_STRING_IP, 0); 1008 DEFINE_FIELD(unsigned long, vaddr[0], FIELD_STRING_IP, 0);
971 size = SIZEOF_TRACE_ENTRY(false); 1009 size = SIZEOF_TRACE_ENTRY(false);
972 } 1010 }
973 /* Set argument names as fields */
974 for (i = 0; i < tu->tp.nr_args; i++) {
975 struct probe_arg *parg = &tu->tp.args[i];
976
977 ret = trace_define_field(event_call, parg->type->fmttype,
978 parg->name, size + parg->offset,
979 parg->type->size, parg->type->is_signed,
980 FILTER_OTHER);
981 1011
982 if (ret) 1012 return traceprobe_define_arg_fields(event_call, size, &tu->tp);
983 return ret;
984 }
985 return 0;
986} 1013}
987 1014
988#ifdef CONFIG_PERF_EVENTS 1015#ifdef CONFIG_PERF_EVENTS
@@ -1233,7 +1260,7 @@ static int uprobe_dispatcher(struct uprobe_consumer *con, struct pt_regs *regs)
1233 esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu)); 1260 esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));
1234 1261
1235 ucb = uprobe_buffer_get(); 1262 ucb = uprobe_buffer_get();
1236 store_trace_args(esize, &tu->tp, regs, ucb->buf, dsize); 1263 store_trace_args(ucb->buf, &tu->tp, regs, esize, dsize);
1237 1264
1238 if (tu->tp.flags & TP_FLAG_TRACE) 1265 if (tu->tp.flags & TP_FLAG_TRACE)
1239 ret |= uprobe_trace_func(tu, regs, ucb, dsize); 1266 ret |= uprobe_trace_func(tu, regs, ucb, dsize);
@@ -1268,7 +1295,7 @@ static int uretprobe_dispatcher(struct uprobe_consumer *con,
1268 esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu)); 1295 esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));
1269 1296
1270 ucb = uprobe_buffer_get(); 1297 ucb = uprobe_buffer_get();
1271 store_trace_args(esize, &tu->tp, regs, ucb->buf, dsize); 1298 store_trace_args(ucb->buf, &tu->tp, regs, esize, dsize);
1272 1299
1273 if (tu->tp.flags & TP_FLAG_TRACE) 1300 if (tu->tp.flags & TP_FLAG_TRACE)
1274 uretprobe_trace_func(tu, func, regs, ucb, dsize); 1301 uretprobe_trace_func(tu, func, regs, ucb, dsize);
@@ -1304,7 +1331,7 @@ static int register_uprobe_event(struct trace_uprobe *tu)
1304 1331
1305 init_trace_event_call(tu, call); 1332 init_trace_event_call(tu, call);
1306 1333
1307 if (set_print_fmt(&tu->tp, is_ret_probe(tu)) < 0) 1334 if (traceprobe_set_print_fmt(&tu->tp, is_ret_probe(tu)) < 0)
1308 return -ENOMEM; 1335 return -ENOMEM;
1309 1336
1310 ret = register_trace_event(&call->event); 1337 ret = register_trace_event(&call->event);
@@ -1340,7 +1367,8 @@ static int unregister_uprobe_event(struct trace_uprobe *tu)
1340 1367
1341#ifdef CONFIG_PERF_EVENTS 1368#ifdef CONFIG_PERF_EVENTS
1342struct trace_event_call * 1369struct trace_event_call *
1343create_local_trace_uprobe(char *name, unsigned long offs, bool is_return) 1370create_local_trace_uprobe(char *name, unsigned long offs,
1371 unsigned long ref_ctr_offset, bool is_return)
1344{ 1372{
1345 struct trace_uprobe *tu; 1373 struct trace_uprobe *tu;
1346 struct path path; 1374 struct path path;
@@ -1372,10 +1400,11 @@ create_local_trace_uprobe(char *name, unsigned long offs, bool is_return)
1372 1400
1373 tu->offset = offs; 1401 tu->offset = offs;
1374 tu->path = path; 1402 tu->path = path;
1403 tu->ref_ctr_offset = ref_ctr_offset;
1375 tu->filename = kstrdup(name, GFP_KERNEL); 1404 tu->filename = kstrdup(name, GFP_KERNEL);
1376 init_trace_event_call(tu, &tu->tp.call); 1405 init_trace_event_call(tu, &tu->tp.call);
1377 1406
1378 if (set_print_fmt(&tu->tp, is_ret_probe(tu)) < 0) { 1407 if (traceprobe_set_print_fmt(&tu->tp, is_ret_probe(tu)) < 0) {
1379 ret = -ENOMEM; 1408 ret = -ENOMEM;
1380 goto error; 1409 goto error;
1381 } 1410 }
generated by cgit v1.2.2 (git 2.25.0) at 2026-04-05 23:30:34 -0400