diff options
author | Daniel Borkmann <daniel@iogearbox.net> | 2018-03-28 16:55:20 -0400 |
---|---|---|
committer | Daniel Borkmann <daniel@iogearbox.net> | 2018-03-28 16:55:21 -0400 |
commit | f6ef56589374670b7c1939720dfa00212bd80a5b (patch) | |
tree | f8f5e66c8fba220b34783b0d38518192bc53bf39 | |
parent | 6f5c39fa5cd4a78c5432021e981aa8f79437a32c (diff) | |
parent | 3bbe0869884ceebffd59d5519c1d560207c6e116 (diff) |
Merge branch 'bpf-raw-tracepoints'
Alexei Starovoitov says:
====================
v7->v8:
- moved 'u32 num_args' from 'struct tracepoint' into 'struct bpf_raw_event_map'
that increases memory overhead, but can be optimized/compressed later.
Now it's zero changes in tracepoint.[ch]
v6->v7:
- adopted Steven's bpf_raw_tp_map section approach to find tracepoint
and corresponding bpf probe function instead of kallsyms approach.
dropped kernel_tracepoint_find_by_name() patch
v5->v6:
- avoid changing semantics of for_each_kernel_tracepoint() function, instead
introduce kernel_tracepoint_find_by_name() helper
v4->v5:
- adopted Daniel's fancy REPEAT macro in bpf_trace.c in patch 6
v3->v4:
- adopted Linus's CAST_TO_U64 macro to cast any integer, pointer, or small
struct to u64. That nicely reduced the size of patch 1
v2->v3:
- with Linus's suggestion introduced generic COUNT_ARGS and CONCATENATE macros
(or rather moved them from apparmor)
that cleaned up patch 6
- added patch 4 to refactor trace_iwlwifi_dev_ucode_error() from 17 args to 4
Now any tracepoint with >12 args will have build error
v1->v2:
- simplified api by combing bpf_raw_tp_open(name) + bpf_attach(prog_fd) into
bpf_raw_tp_open(name, prog_fd) as suggested by Daniel.
That simplifies bpf_detach as well which is now simple close() of fd.
- fixed memory leak in error path which was spotted by Daniel.
- fixed bpf_get_stackid(), bpf_perf_event_output() called from raw tracepoints
- added more tests
- fixed allyesconfig build caught by buildbot
v1:
This patch set is a different way to address the pressing need to access
task_struct pointers in sched tracepoints from bpf programs.
The first approach simply added these pointers to sched tracepoints:
https://lkml.org/lkml/2017/12/14/753
which Peter nacked.
Few options were discussed and eventually the discussion converged on
doing bpf specific tracepoint_probe_register() probe functions.
Details here:
https://lkml.org/lkml/2017/12/20/929
Patch 1 is kernel wide cleanup of pass-struct-by-value into
pass-struct-by-reference into tracepoints.
Patches 2 and 3 are minor cleanups to address allyesconfig build
Patch 4 refactor trace_iwlwifi_dev_ucode_error from 17 to 4 args
Patch 5 introduces COUNT_ARGS macro
Patch 6 introduces BPF_RAW_TRACEPOINT api.
the auto-cleanup and multiple concurrent users are must have
features of tracing api. For bpf raw tracepoints it looks like:
// load bpf prog with BPF_PROG_TYPE_RAW_TRACEPOINT type
prog_fd = bpf_prog_load(...);
// receive anon_inode fd for given bpf_raw_tracepoint
// and attach bpf program to it
raw_tp_fd = bpf_raw_tracepoint_open("xdp_exception", prog_fd);
Ctrl-C of tracing daemon or cmdline tool will automatically
detach bpf program, unload it and unregister tracepoint probe.
More details in patch 6.
Patch 7 - trivial support in libbpf
Patches 8, 9 - user space tests
samples/bpf/test_overhead performance on 1 cpu:
tracepoint base kprobe+bpf tracepoint+bpf raw_tracepoint+bpf
task_rename 1.1M 769K 947K 1.0M
urandom_read 789K 697K 750K 755K
====================
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
30 files changed, 607 insertions, 77 deletions
diff --git a/drivers/infiniband/hw/hfi1/file_ops.c b/drivers/infiniband/hw/hfi1/file_ops.c index 41fafebe3b0d..da4aa1a95b11 100644 --- a/drivers/infiniband/hw/hfi1/file_ops.c +++ b/drivers/infiniband/hw/hfi1/file_ops.c | |||
@@ -1153,7 +1153,7 @@ static int get_ctxt_info(struct hfi1_filedata *fd, unsigned long arg, u32 len) | |||
1153 | cinfo.sdma_ring_size = fd->cq->nentries; | 1153 | cinfo.sdma_ring_size = fd->cq->nentries; |
1154 | cinfo.rcvegr_size = uctxt->egrbufs.rcvtid_size; | 1154 | cinfo.rcvegr_size = uctxt->egrbufs.rcvtid_size; |
1155 | 1155 | ||
1156 | trace_hfi1_ctxt_info(uctxt->dd, uctxt->ctxt, fd->subctxt, cinfo); | 1156 | trace_hfi1_ctxt_info(uctxt->dd, uctxt->ctxt, fd->subctxt, &cinfo); |
1157 | if (copy_to_user((void __user *)arg, &cinfo, len)) | 1157 | if (copy_to_user((void __user *)arg, &cinfo, len)) |
1158 | return -EFAULT; | 1158 | return -EFAULT; |
1159 | 1159 | ||
diff --git a/drivers/infiniband/hw/hfi1/trace_ctxts.h b/drivers/infiniband/hw/hfi1/trace_ctxts.h index 4eb4cc798035..e00c8a7d559c 100644 --- a/drivers/infiniband/hw/hfi1/trace_ctxts.h +++ b/drivers/infiniband/hw/hfi1/trace_ctxts.h | |||
@@ -106,7 +106,7 @@ TRACE_EVENT(hfi1_uctxtdata, | |||
106 | TRACE_EVENT(hfi1_ctxt_info, | 106 | TRACE_EVENT(hfi1_ctxt_info, |
107 | TP_PROTO(struct hfi1_devdata *dd, unsigned int ctxt, | 107 | TP_PROTO(struct hfi1_devdata *dd, unsigned int ctxt, |
108 | unsigned int subctxt, | 108 | unsigned int subctxt, |
109 | struct hfi1_ctxt_info cinfo), | 109 | struct hfi1_ctxt_info *cinfo), |
110 | TP_ARGS(dd, ctxt, subctxt, cinfo), | 110 | TP_ARGS(dd, ctxt, subctxt, cinfo), |
111 | TP_STRUCT__entry(DD_DEV_ENTRY(dd) | 111 | TP_STRUCT__entry(DD_DEV_ENTRY(dd) |
112 | __field(unsigned int, ctxt) | 112 | __field(unsigned int, ctxt) |
@@ -120,11 +120,11 @@ TRACE_EVENT(hfi1_ctxt_info, | |||
120 | TP_fast_assign(DD_DEV_ASSIGN(dd); | 120 | TP_fast_assign(DD_DEV_ASSIGN(dd); |
121 | __entry->ctxt = ctxt; | 121 | __entry->ctxt = ctxt; |
122 | __entry->subctxt = subctxt; | 122 | __entry->subctxt = subctxt; |
123 | __entry->egrtids = cinfo.egrtids; | 123 | __entry->egrtids = cinfo->egrtids; |
124 | __entry->rcvhdrq_cnt = cinfo.rcvhdrq_cnt; | 124 | __entry->rcvhdrq_cnt = cinfo->rcvhdrq_cnt; |
125 | __entry->rcvhdrq_size = cinfo.rcvhdrq_entsize; | 125 | __entry->rcvhdrq_size = cinfo->rcvhdrq_entsize; |
126 | __entry->sdma_ring_size = cinfo.sdma_ring_size; | 126 | __entry->sdma_ring_size = cinfo->sdma_ring_size; |
127 | __entry->rcvegr_size = cinfo.rcvegr_size; | 127 | __entry->rcvegr_size = cinfo->rcvegr_size; |
128 | ), | 128 | ), |
129 | TP_printk("[%s] ctxt %u:%u " CINFO_FMT, | 129 | TP_printk("[%s] ctxt %u:%u " CINFO_FMT, |
130 | __get_str(dev), | 130 | __get_str(dev), |
diff --git a/drivers/net/wireless/intel/iwlwifi/dvm/main.c b/drivers/net/wireless/intel/iwlwifi/dvm/main.c index d11d72615de2..e68254e12764 100644 --- a/drivers/net/wireless/intel/iwlwifi/dvm/main.c +++ b/drivers/net/wireless/intel/iwlwifi/dvm/main.c | |||
@@ -1651,12 +1651,7 @@ static void iwl_dump_nic_error_log(struct iwl_priv *priv) | |||
1651 | priv->status, table.valid); | 1651 | priv->status, table.valid); |
1652 | } | 1652 | } |
1653 | 1653 | ||
1654 | trace_iwlwifi_dev_ucode_error(trans->dev, table.error_id, table.tsf_low, | 1654 | trace_iwlwifi_dev_ucode_error(trans->dev, &table, 0, table.brd_ver); |
1655 | table.data1, table.data2, table.line, | ||
1656 | table.blink2, table.ilink1, table.ilink2, | ||
1657 | table.bcon_time, table.gp1, table.gp2, | ||
1658 | table.gp3, table.ucode_ver, table.hw_ver, | ||
1659 | 0, table.brd_ver); | ||
1660 | IWL_ERR(priv, "0x%08X | %-28s\n", table.error_id, | 1655 | IWL_ERR(priv, "0x%08X | %-28s\n", table.error_id, |
1661 | desc_lookup(table.error_id)); | 1656 | desc_lookup(table.error_id)); |
1662 | IWL_ERR(priv, "0x%08X | uPc\n", table.pc); | 1657 | IWL_ERR(priv, "0x%08X | uPc\n", table.pc); |
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-devtrace-iwlwifi.h b/drivers/net/wireless/intel/iwlwifi/iwl-devtrace-iwlwifi.h index 9518a82f44c2..27e3e4e96aa2 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-devtrace-iwlwifi.h +++ b/drivers/net/wireless/intel/iwlwifi/iwl-devtrace-iwlwifi.h | |||
@@ -126,14 +126,11 @@ TRACE_EVENT(iwlwifi_dev_tx, | |||
126 | __entry->framelen, __entry->skbaddr) | 126 | __entry->framelen, __entry->skbaddr) |
127 | ); | 127 | ); |
128 | 128 | ||
129 | struct iwl_error_event_table; | ||
129 | TRACE_EVENT(iwlwifi_dev_ucode_error, | 130 | TRACE_EVENT(iwlwifi_dev_ucode_error, |
130 | TP_PROTO(const struct device *dev, u32 desc, u32 tsf_low, | 131 | TP_PROTO(const struct device *dev, const struct iwl_error_event_table *table, |
131 | u32 data1, u32 data2, u32 line, u32 blink2, u32 ilink1, | 132 | u32 hw_ver, u32 brd_ver), |
132 | u32 ilink2, u32 bcon_time, u32 gp1, u32 gp2, u32 rev_type, | 133 | TP_ARGS(dev, table, hw_ver, brd_ver), |
133 | u32 major, u32 minor, u32 hw_ver, u32 brd_ver), | ||
134 | TP_ARGS(dev, desc, tsf_low, data1, data2, line, | ||
135 | blink2, ilink1, ilink2, bcon_time, gp1, gp2, | ||
136 | rev_type, major, minor, hw_ver, brd_ver), | ||
137 | TP_STRUCT__entry( | 134 | TP_STRUCT__entry( |
138 | DEV_ENTRY | 135 | DEV_ENTRY |
139 | __field(u32, desc) | 136 | __field(u32, desc) |
@@ -155,20 +152,20 @@ TRACE_EVENT(iwlwifi_dev_ucode_error, | |||
155 | ), | 152 | ), |
156 | TP_fast_assign( | 153 | TP_fast_assign( |
157 | DEV_ASSIGN; | 154 | DEV_ASSIGN; |
158 | __entry->desc = desc; | 155 | __entry->desc = table->error_id; |
159 | __entry->tsf_low = tsf_low; | 156 | __entry->tsf_low = table->tsf_low; |
160 | __entry->data1 = data1; | 157 | __entry->data1 = table->data1; |
161 | __entry->data2 = data2; | 158 | __entry->data2 = table->data2; |
162 | __entry->line = line; | 159 | __entry->line = table->line; |
163 | __entry->blink2 = blink2; | 160 | __entry->blink2 = table->blink2; |
164 | __entry->ilink1 = ilink1; | 161 | __entry->ilink1 = table->ilink1; |
165 | __entry->ilink2 = ilink2; | 162 | __entry->ilink2 = table->ilink2; |
166 | __entry->bcon_time = bcon_time; | 163 | __entry->bcon_time = table->bcon_time; |
167 | __entry->gp1 = gp1; | 164 | __entry->gp1 = table->gp1; |
168 | __entry->gp2 = gp2; | 165 | __entry->gp2 = table->gp2; |
169 | __entry->rev_type = rev_type; | 166 | __entry->rev_type = table->gp3; |
170 | __entry->major = major; | 167 | __entry->major = table->ucode_ver; |
171 | __entry->minor = minor; | 168 | __entry->minor = table->hw_ver; |
172 | __entry->hw_ver = hw_ver; | 169 | __entry->hw_ver = hw_ver; |
173 | __entry->brd_ver = brd_ver; | 170 | __entry->brd_ver = brd_ver; |
174 | ), | 171 | ), |
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-devtrace.c b/drivers/net/wireless/intel/iwlwifi/iwl-devtrace.c index 50510fb6ab8c..6aa719865a58 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-devtrace.c +++ b/drivers/net/wireless/intel/iwlwifi/iwl-devtrace.c | |||
@@ -30,6 +30,7 @@ | |||
30 | #ifndef __CHECKER__ | 30 | #ifndef __CHECKER__ |
31 | #include "iwl-trans.h" | 31 | #include "iwl-trans.h" |
32 | 32 | ||
33 | #include "dvm/commands.h" | ||
33 | #define CREATE_TRACE_POINTS | 34 | #define CREATE_TRACE_POINTS |
34 | #include "iwl-devtrace.h" | 35 | #include "iwl-devtrace.h" |
35 | 36 | ||
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/utils.c b/drivers/net/wireless/intel/iwlwifi/mvm/utils.c index d65e1db7c097..5442ead876eb 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/utils.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/utils.c | |||
@@ -549,12 +549,7 @@ static void iwl_mvm_dump_lmac_error_log(struct iwl_mvm *mvm, u32 base) | |||
549 | 549 | ||
550 | IWL_ERR(mvm, "Loaded firmware version: %s\n", mvm->fw->fw_version); | 550 | IWL_ERR(mvm, "Loaded firmware version: %s\n", mvm->fw->fw_version); |
551 | 551 | ||
552 | trace_iwlwifi_dev_ucode_error(trans->dev, table.error_id, table.tsf_low, | 552 | trace_iwlwifi_dev_ucode_error(trans->dev, &table, table.hw_ver, table.brd_ver); |
553 | table.data1, table.data2, table.data3, | ||
554 | table.blink2, table.ilink1, | ||
555 | table.ilink2, table.bcon_time, table.gp1, | ||
556 | table.gp2, table.fw_rev_type, table.major, | ||
557 | table.minor, table.hw_ver, table.brd_ver); | ||
558 | IWL_ERR(mvm, "0x%08X | %-28s\n", table.error_id, | 553 | IWL_ERR(mvm, "0x%08X | %-28s\n", table.error_id, |
559 | desc_lookup(table.error_id)); | 554 | desc_lookup(table.error_id)); |
560 | IWL_ERR(mvm, "0x%08X | trm_hw_status0\n", table.trm_hw_status0); | 555 | IWL_ERR(mvm, "0x%08X | trm_hw_status0\n", table.trm_hw_status0); |
diff --git a/drivers/net/wireless/mediatek/mt7601u/trace.h b/drivers/net/wireless/mediatek/mt7601u/trace.h index 289897300ef0..82c8898b9076 100644 --- a/drivers/net/wireless/mediatek/mt7601u/trace.h +++ b/drivers/net/wireless/mediatek/mt7601u/trace.h | |||
@@ -34,7 +34,7 @@ | |||
34 | #define REG_PR_FMT "%04x=%08x" | 34 | #define REG_PR_FMT "%04x=%08x" |
35 | #define REG_PR_ARG __entry->reg, __entry->val | 35 | #define REG_PR_ARG __entry->reg, __entry->val |
36 | 36 | ||
37 | DECLARE_EVENT_CLASS(dev_reg_evt, | 37 | DECLARE_EVENT_CLASS(dev_reg_evtu, |
38 | TP_PROTO(struct mt7601u_dev *dev, u32 reg, u32 val), | 38 | TP_PROTO(struct mt7601u_dev *dev, u32 reg, u32 val), |
39 | TP_ARGS(dev, reg, val), | 39 | TP_ARGS(dev, reg, val), |
40 | TP_STRUCT__entry( | 40 | TP_STRUCT__entry( |
@@ -51,12 +51,12 @@ DECLARE_EVENT_CLASS(dev_reg_evt, | |||
51 | ) | 51 | ) |
52 | ); | 52 | ); |
53 | 53 | ||
54 | DEFINE_EVENT(dev_reg_evt, reg_read, | 54 | DEFINE_EVENT(dev_reg_evtu, reg_read, |
55 | TP_PROTO(struct mt7601u_dev *dev, u32 reg, u32 val), | 55 | TP_PROTO(struct mt7601u_dev *dev, u32 reg, u32 val), |
56 | TP_ARGS(dev, reg, val) | 56 | TP_ARGS(dev, reg, val) |
57 | ); | 57 | ); |
58 | 58 | ||
59 | DEFINE_EVENT(dev_reg_evt, reg_write, | 59 | DEFINE_EVENT(dev_reg_evtu, reg_write, |
60 | TP_PROTO(struct mt7601u_dev *dev, u32 reg, u32 val), | 60 | TP_PROTO(struct mt7601u_dev *dev, u32 reg, u32 val), |
61 | TP_ARGS(dev, reg, val) | 61 | TP_ARGS(dev, reg, val) |
62 | ); | 62 | ); |
diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h index 1ab0e520d6fc..8add3493a202 100644 --- a/include/asm-generic/vmlinux.lds.h +++ b/include/asm-generic/vmlinux.lds.h | |||
@@ -178,6 +178,15 @@ | |||
178 | #define TRACE_SYSCALLS() | 178 | #define TRACE_SYSCALLS() |
179 | #endif | 179 | #endif |
180 | 180 | ||
181 | #ifdef CONFIG_BPF_EVENTS | ||
182 | #define BPF_RAW_TP() STRUCT_ALIGN(); \ | ||
183 | VMLINUX_SYMBOL(__start__bpf_raw_tp) = .; \ | ||
184 | KEEP(*(__bpf_raw_tp_map)) \ | ||
185 | VMLINUX_SYMBOL(__stop__bpf_raw_tp) = .; | ||
186 | #else | ||
187 | #define BPF_RAW_TP() | ||
188 | #endif | ||
189 | |||
181 | #ifdef CONFIG_SERIAL_EARLYCON | 190 | #ifdef CONFIG_SERIAL_EARLYCON |
182 | #define EARLYCON_TABLE() STRUCT_ALIGN(); \ | 191 | #define EARLYCON_TABLE() STRUCT_ALIGN(); \ |
183 | VMLINUX_SYMBOL(__earlycon_table) = .; \ | 192 | VMLINUX_SYMBOL(__earlycon_table) = .; \ |
@@ -249,6 +258,7 @@ | |||
249 | LIKELY_PROFILE() \ | 258 | LIKELY_PROFILE() \ |
250 | BRANCH_PROFILE() \ | 259 | BRANCH_PROFILE() \ |
251 | TRACE_PRINTKS() \ | 260 | TRACE_PRINTKS() \ |
261 | BPF_RAW_TP() \ | ||
252 | TRACEPOINT_STR() | 262 | TRACEPOINT_STR() |
253 | 263 | ||
254 | /* | 264 | /* |
diff --git a/include/linux/bpf_types.h b/include/linux/bpf_types.h index 5e2e8a49fb21..6d7243bfb0ff 100644 --- a/include/linux/bpf_types.h +++ b/include/linux/bpf_types.h | |||
@@ -19,6 +19,7 @@ BPF_PROG_TYPE(BPF_PROG_TYPE_SK_MSG, sk_msg) | |||
19 | BPF_PROG_TYPE(BPF_PROG_TYPE_KPROBE, kprobe) | 19 | BPF_PROG_TYPE(BPF_PROG_TYPE_KPROBE, kprobe) |
20 | BPF_PROG_TYPE(BPF_PROG_TYPE_TRACEPOINT, tracepoint) | 20 | BPF_PROG_TYPE(BPF_PROG_TYPE_TRACEPOINT, tracepoint) |
21 | BPF_PROG_TYPE(BPF_PROG_TYPE_PERF_EVENT, perf_event) | 21 | BPF_PROG_TYPE(BPF_PROG_TYPE_PERF_EVENT, perf_event) |
22 | BPF_PROG_TYPE(BPF_PROG_TYPE_RAW_TRACEPOINT, raw_tracepoint) | ||
22 | #endif | 23 | #endif |
23 | #ifdef CONFIG_CGROUP_BPF | 24 | #ifdef CONFIG_CGROUP_BPF |
24 | BPF_PROG_TYPE(BPF_PROG_TYPE_CGROUP_DEVICE, cg_dev) | 25 | BPF_PROG_TYPE(BPF_PROG_TYPE_CGROUP_DEVICE, cg_dev) |
diff --git a/include/linux/kernel.h b/include/linux/kernel.h index 3fd291503576..293fa0677fba 100644 --- a/include/linux/kernel.h +++ b/include/linux/kernel.h | |||
@@ -919,6 +919,13 @@ static inline void ftrace_dump(enum ftrace_dump_mode oops_dump_mode) { } | |||
919 | #define swap(a, b) \ | 919 | #define swap(a, b) \ |
920 | do { typeof(a) __tmp = (a); (a) = (b); (b) = __tmp; } while (0) | 920 | do { typeof(a) __tmp = (a); (a) = (b); (b) = __tmp; } while (0) |
921 | 921 | ||
922 | /* This counts to 12. Any more, it will return 13th argument. */ | ||
923 | #define __COUNT_ARGS(_0, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _n, X...) _n | ||
924 | #define COUNT_ARGS(X...) __COUNT_ARGS(, ##X, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0) | ||
925 | |||
926 | #define __CONCAT(a, b) a ## b | ||
927 | #define CONCATENATE(a, b) __CONCAT(a, b) | ||
928 | |||
922 | /** | 929 | /** |
923 | * container_of - cast a member of a structure out to the containing structure | 930 | * container_of - cast a member of a structure out to the containing structure |
924 | * @ptr: the pointer to the member. | 931 | * @ptr: the pointer to the member. |
diff --git a/include/linux/trace_events.h b/include/linux/trace_events.h index 8a1442c4e513..b0357cd198b0 100644 --- a/include/linux/trace_events.h +++ b/include/linux/trace_events.h | |||
@@ -468,6 +468,9 @@ unsigned int trace_call_bpf(struct trace_event_call *call, void *ctx); | |||
468 | int perf_event_attach_bpf_prog(struct perf_event *event, struct bpf_prog *prog); | 468 | int perf_event_attach_bpf_prog(struct perf_event *event, struct bpf_prog *prog); |
469 | void perf_event_detach_bpf_prog(struct perf_event *event); | 469 | void perf_event_detach_bpf_prog(struct perf_event *event); |
470 | int perf_event_query_prog_array(struct perf_event *event, void __user *info); | 470 | int perf_event_query_prog_array(struct perf_event *event, void __user *info); |
471 | int bpf_probe_register(struct bpf_raw_event_map *btp, struct bpf_prog *prog); | ||
472 | int bpf_probe_unregister(struct bpf_raw_event_map *btp, struct bpf_prog *prog); | ||
473 | struct bpf_raw_event_map *bpf_find_raw_tracepoint(const char *name); | ||
471 | #else | 474 | #else |
472 | static inline unsigned int trace_call_bpf(struct trace_event_call *call, void *ctx) | 475 | static inline unsigned int trace_call_bpf(struct trace_event_call *call, void *ctx) |
473 | { | 476 | { |
@@ -487,6 +490,18 @@ perf_event_query_prog_array(struct perf_event *event, void __user *info) | |||
487 | { | 490 | { |
488 | return -EOPNOTSUPP; | 491 | return -EOPNOTSUPP; |
489 | } | 492 | } |
493 | static inline int bpf_probe_register(struct bpf_raw_event_map *btp, struct bpf_prog *p) | ||
494 | { | ||
495 | return -EOPNOTSUPP; | ||
496 | } | ||
497 | static inline int bpf_probe_unregister(struct bpf_raw_event_map *btp, struct bpf_prog *p) | ||
498 | { | ||
499 | return -EOPNOTSUPP; | ||
500 | } | ||
501 | static inline struct bpf_raw_event_map *bpf_find_raw_tracepoint(const char *name) | ||
502 | { | ||
503 | return NULL; | ||
504 | } | ||
490 | #endif | 505 | #endif |
491 | 506 | ||
492 | enum { | 507 | enum { |
@@ -546,6 +561,33 @@ extern void ftrace_profile_free_filter(struct perf_event *event); | |||
546 | void perf_trace_buf_update(void *record, u16 type); | 561 | void perf_trace_buf_update(void *record, u16 type); |
547 | void *perf_trace_buf_alloc(int size, struct pt_regs **regs, int *rctxp); | 562 | void *perf_trace_buf_alloc(int size, struct pt_regs **regs, int *rctxp); |
548 | 563 | ||
564 | void bpf_trace_run1(struct bpf_prog *prog, u64 arg1); | ||
565 | void bpf_trace_run2(struct bpf_prog *prog, u64 arg1, u64 arg2); | ||
566 | void bpf_trace_run3(struct bpf_prog *prog, u64 arg1, u64 arg2, | ||
567 | u64 arg3); | ||
568 | void bpf_trace_run4(struct bpf_prog *prog, u64 arg1, u64 arg2, | ||
569 | u64 arg3, u64 arg4); | ||
570 | void bpf_trace_run5(struct bpf_prog *prog, u64 arg1, u64 arg2, | ||
571 | u64 arg3, u64 arg4, u64 arg5); | ||
572 | void bpf_trace_run6(struct bpf_prog *prog, u64 arg1, u64 arg2, | ||
573 | u64 arg3, u64 arg4, u64 arg5, u64 arg6); | ||
574 | void bpf_trace_run7(struct bpf_prog *prog, u64 arg1, u64 arg2, | ||
575 | u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7); | ||
576 | void bpf_trace_run8(struct bpf_prog *prog, u64 arg1, u64 arg2, | ||
577 | u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7, | ||
578 | u64 arg8); | ||
579 | void bpf_trace_run9(struct bpf_prog *prog, u64 arg1, u64 arg2, | ||
580 | u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7, | ||
581 | u64 arg8, u64 arg9); | ||
582 | void bpf_trace_run10(struct bpf_prog *prog, u64 arg1, u64 arg2, | ||
583 | u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7, | ||
584 | u64 arg8, u64 arg9, u64 arg10); | ||
585 | void bpf_trace_run11(struct bpf_prog *prog, u64 arg1, u64 arg2, | ||
586 | u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7, | ||
587 | u64 arg8, u64 arg9, u64 arg10, u64 arg11); | ||
588 | void bpf_trace_run12(struct bpf_prog *prog, u64 arg1, u64 arg2, | ||
589 | u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7, | ||
590 | u64 arg8, u64 arg9, u64 arg10, u64 arg11, u64 arg12); | ||
549 | void perf_trace_run_bpf_submit(void *raw_data, int size, int rctx, | 591 | void perf_trace_run_bpf_submit(void *raw_data, int size, int rctx, |
550 | struct trace_event_call *call, u64 count, | 592 | struct trace_event_call *call, u64 count, |
551 | struct pt_regs *regs, struct hlist_head *head, | 593 | struct pt_regs *regs, struct hlist_head *head, |
diff --git a/include/linux/tracepoint-defs.h b/include/linux/tracepoint-defs.h index 64ed7064f1fa..22c5a46e9693 100644 --- a/include/linux/tracepoint-defs.h +++ b/include/linux/tracepoint-defs.h | |||
@@ -35,4 +35,10 @@ struct tracepoint { | |||
35 | struct tracepoint_func __rcu *funcs; | 35 | struct tracepoint_func __rcu *funcs; |
36 | }; | 36 | }; |
37 | 37 | ||
38 | struct bpf_raw_event_map { | ||
39 | struct tracepoint *tp; | ||
40 | void *bpf_func; | ||
41 | u32 num_args; | ||
42 | } __aligned(32); | ||
43 | |||
38 | #endif | 44 | #endif |
diff --git a/include/trace/bpf_probe.h b/include/trace/bpf_probe.h new file mode 100644 index 000000000000..505dae0bed80 --- /dev/null +++ b/include/trace/bpf_probe.h | |||
@@ -0,0 +1,92 @@ | |||
1 | /* SPDX-License-Identifier: GPL-2.0 */ | ||
2 | |||
3 | #undef TRACE_SYSTEM_VAR | ||
4 | |||
5 | #ifdef CONFIG_BPF_EVENTS | ||
6 | |||
7 | #undef __entry | ||
8 | #define __entry entry | ||
9 | |||
10 | #undef __get_dynamic_array | ||
11 | #define __get_dynamic_array(field) \ | ||
12 | ((void *)__entry + (__entry->__data_loc_##field & 0xffff)) | ||
13 | |||
14 | #undef __get_dynamic_array_len | ||
15 | #define __get_dynamic_array_len(field) \ | ||
16 | ((__entry->__data_loc_##field >> 16) & 0xffff) | ||
17 | |||
18 | #undef __get_str | ||
19 | #define __get_str(field) ((char *)__get_dynamic_array(field)) | ||
20 | |||
21 | #undef __get_bitmask | ||
22 | #define __get_bitmask(field) (char *)__get_dynamic_array(field) | ||
23 | |||
24 | #undef __perf_count | ||
25 | #define __perf_count(c) (c) | ||
26 | |||
27 | #undef __perf_task | ||
28 | #define __perf_task(t) (t) | ||
29 | |||
30 | /* cast any integer, pointer, or small struct to u64 */ | ||
31 | #define UINTTYPE(size) \ | ||
32 | __typeof__(__builtin_choose_expr(size == 1, (u8)1, \ | ||
33 | __builtin_choose_expr(size == 2, (u16)2, \ | ||
34 | __builtin_choose_expr(size == 4, (u32)3, \ | ||
35 | __builtin_choose_expr(size == 8, (u64)4, \ | ||
36 | (void)5))))) | ||
37 | #define __CAST_TO_U64(x) ({ \ | ||
38 | typeof(x) __src = (x); \ | ||
39 | UINTTYPE(sizeof(x)) __dst; \ | ||
40 | memcpy(&__dst, &__src, sizeof(__dst)); \ | ||
41 | (u64)__dst; }) | ||
42 | |||
43 | #define __CAST1(a,...) __CAST_TO_U64(a) | ||
44 | #define __CAST2(a,...) __CAST_TO_U64(a), __CAST1(__VA_ARGS__) | ||
45 | #define __CAST3(a,...) __CAST_TO_U64(a), __CAST2(__VA_ARGS__) | ||
46 | #define __CAST4(a,...) __CAST_TO_U64(a), __CAST3(__VA_ARGS__) | ||
47 | #define __CAST5(a,...) __CAST_TO_U64(a), __CAST4(__VA_ARGS__) | ||
48 | #define __CAST6(a,...) __CAST_TO_U64(a), __CAST5(__VA_ARGS__) | ||
49 | #define __CAST7(a,...) __CAST_TO_U64(a), __CAST6(__VA_ARGS__) | ||
50 | #define __CAST8(a,...) __CAST_TO_U64(a), __CAST7(__VA_ARGS__) | ||
51 | #define __CAST9(a,...) __CAST_TO_U64(a), __CAST8(__VA_ARGS__) | ||
52 | #define __CAST10(a,...) __CAST_TO_U64(a), __CAST9(__VA_ARGS__) | ||
53 | #define __CAST11(a,...) __CAST_TO_U64(a), __CAST10(__VA_ARGS__) | ||
54 | #define __CAST12(a,...) __CAST_TO_U64(a), __CAST11(__VA_ARGS__) | ||
55 | /* tracepoints with more than 12 arguments will hit build error */ | ||
56 | #define CAST_TO_U64(...) CONCATENATE(__CAST, COUNT_ARGS(__VA_ARGS__))(__VA_ARGS__) | ||
57 | |||
58 | #undef DECLARE_EVENT_CLASS | ||
59 | #define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \ | ||
60 | static notrace void \ | ||
61 | __bpf_trace_##call(void *__data, proto) \ | ||
62 | { \ | ||
63 | struct bpf_prog *prog = __data; \ | ||
64 | CONCATENATE(bpf_trace_run, COUNT_ARGS(args))(prog, CAST_TO_U64(args)); \ | ||
65 | } | ||
66 | |||
67 | /* | ||
68 | * This part is compiled out, it is only here as a build time check | ||
69 | * to make sure that if the tracepoint handling changes, the | ||
70 | * bpf probe will fail to compile unless it too is updated. | ||
71 | */ | ||
72 | #undef DEFINE_EVENT | ||
73 | #define DEFINE_EVENT(template, call, proto, args) \ | ||
74 | static inline void bpf_test_probe_##call(void) \ | ||
75 | { \ | ||
76 | check_trace_callback_type_##call(__bpf_trace_##template); \ | ||
77 | } \ | ||
78 | static struct bpf_raw_event_map __used \ | ||
79 | __attribute__((section("__bpf_raw_tp_map"))) \ | ||
80 | __bpf_trace_tp_map_##call = { \ | ||
81 | .tp = &__tracepoint_##call, \ | ||
82 | .bpf_func = (void *)__bpf_trace_##template, \ | ||
83 | .num_args = COUNT_ARGS(args), \ | ||
84 | }; | ||
85 | |||
86 | |||
87 | #undef DEFINE_EVENT_PRINT | ||
88 | #define DEFINE_EVENT_PRINT(template, name, proto, args, print) \ | ||
89 | DEFINE_EVENT(template, name, PARAMS(proto), PARAMS(args)) | ||
90 | |||
91 | #include TRACE_INCLUDE(TRACE_INCLUDE_FILE) | ||
92 | #endif /* CONFIG_BPF_EVENTS */ | ||
diff --git a/include/trace/define_trace.h b/include/trace/define_trace.h index d9e3d4aa3f6e..cb30c5532144 100644 --- a/include/trace/define_trace.h +++ b/include/trace/define_trace.h | |||
@@ -95,6 +95,7 @@ | |||
95 | #ifdef TRACEPOINTS_ENABLED | 95 | #ifdef TRACEPOINTS_ENABLED |
96 | #include <trace/trace_events.h> | 96 | #include <trace/trace_events.h> |
97 | #include <trace/perf.h> | 97 | #include <trace/perf.h> |
98 | #include <trace/bpf_probe.h> | ||
98 | #endif | 99 | #endif |
99 | 100 | ||
100 | #undef TRACE_EVENT | 101 | #undef TRACE_EVENT |
diff --git a/include/trace/events/f2fs.h b/include/trace/events/f2fs.h index 06c87f9f720c..795698925d20 100644 --- a/include/trace/events/f2fs.h +++ b/include/trace/events/f2fs.h | |||
@@ -491,7 +491,7 @@ DEFINE_EVENT(f2fs__truncate_node, f2fs_truncate_node, | |||
491 | 491 | ||
492 | TRACE_EVENT(f2fs_truncate_partial_nodes, | 492 | TRACE_EVENT(f2fs_truncate_partial_nodes, |
493 | 493 | ||
494 | TP_PROTO(struct inode *inode, nid_t nid[], int depth, int err), | 494 | TP_PROTO(struct inode *inode, nid_t *nid, int depth, int err), |
495 | 495 | ||
496 | TP_ARGS(inode, nid, depth, err), | 496 | TP_ARGS(inode, nid, depth, err), |
497 | 497 | ||
diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 18b7c510c511..1878201c2d77 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h | |||
@@ -94,6 +94,7 @@ enum bpf_cmd { | |||
94 | BPF_MAP_GET_FD_BY_ID, | 94 | BPF_MAP_GET_FD_BY_ID, |
95 | BPF_OBJ_GET_INFO_BY_FD, | 95 | BPF_OBJ_GET_INFO_BY_FD, |
96 | BPF_PROG_QUERY, | 96 | BPF_PROG_QUERY, |
97 | BPF_RAW_TRACEPOINT_OPEN, | ||
97 | }; | 98 | }; |
98 | 99 | ||
99 | enum bpf_map_type { | 100 | enum bpf_map_type { |
@@ -134,6 +135,7 @@ enum bpf_prog_type { | |||
134 | BPF_PROG_TYPE_SK_SKB, | 135 | BPF_PROG_TYPE_SK_SKB, |
135 | BPF_PROG_TYPE_CGROUP_DEVICE, | 136 | BPF_PROG_TYPE_CGROUP_DEVICE, |
136 | BPF_PROG_TYPE_SK_MSG, | 137 | BPF_PROG_TYPE_SK_MSG, |
138 | BPF_PROG_TYPE_RAW_TRACEPOINT, | ||
137 | }; | 139 | }; |
138 | 140 | ||
139 | enum bpf_attach_type { | 141 | enum bpf_attach_type { |
@@ -344,6 +346,11 @@ union bpf_attr { | |||
344 | __aligned_u64 prog_ids; | 346 | __aligned_u64 prog_ids; |
345 | __u32 prog_cnt; | 347 | __u32 prog_cnt; |
346 | } query; | 348 | } query; |
349 | |||
350 | struct { | ||
351 | __u64 name; | ||
352 | __u32 prog_fd; | ||
353 | } raw_tracepoint; | ||
347 | } __attribute__((aligned(8))); | 354 | } __attribute__((aligned(8))); |
348 | 355 | ||
349 | /* BPF helper function descriptions: | 356 | /* BPF helper function descriptions: |
@@ -1152,4 +1159,8 @@ struct bpf_cgroup_dev_ctx { | |||
1152 | __u32 minor; | 1159 | __u32 minor; |
1153 | }; | 1160 | }; |
1154 | 1161 | ||
1162 | struct bpf_raw_tracepoint_args { | ||
1163 | __u64 args[0]; | ||
1164 | }; | ||
1165 | |||
1155 | #endif /* _UAPI__LINUX_BPF_H__ */ | 1166 | #endif /* _UAPI__LINUX_BPF_H__ */ |
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 77d45bd9f507..95ca2523fa6e 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c | |||
@@ -1315,6 +1315,81 @@ static int bpf_obj_get(const union bpf_attr *attr) | |||
1315 | attr->file_flags); | 1315 | attr->file_flags); |
1316 | } | 1316 | } |
1317 | 1317 | ||
1318 | struct bpf_raw_tracepoint { | ||
1319 | struct bpf_raw_event_map *btp; | ||
1320 | struct bpf_prog *prog; | ||
1321 | }; | ||
1322 | |||
1323 | static int bpf_raw_tracepoint_release(struct inode *inode, struct file *filp) | ||
1324 | { | ||
1325 | struct bpf_raw_tracepoint *raw_tp = filp->private_data; | ||
1326 | |||
1327 | if (raw_tp->prog) { | ||
1328 | bpf_probe_unregister(raw_tp->btp, raw_tp->prog); | ||
1329 | bpf_prog_put(raw_tp->prog); | ||
1330 | } | ||
1331 | kfree(raw_tp); | ||
1332 | return 0; | ||
1333 | } | ||
1334 | |||
1335 | static const struct file_operations bpf_raw_tp_fops = { | ||
1336 | .release = bpf_raw_tracepoint_release, | ||
1337 | .read = bpf_dummy_read, | ||
1338 | .write = bpf_dummy_write, | ||
1339 | }; | ||
1340 | |||
1341 | #define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.prog_fd | ||
1342 | |||
1343 | static int bpf_raw_tracepoint_open(const union bpf_attr *attr) | ||
1344 | { | ||
1345 | struct bpf_raw_tracepoint *raw_tp; | ||
1346 | struct bpf_raw_event_map *btp; | ||
1347 | struct bpf_prog *prog; | ||
1348 | char tp_name[128]; | ||
1349 | int tp_fd, err; | ||
1350 | |||
1351 | if (strncpy_from_user(tp_name, u64_to_user_ptr(attr->raw_tracepoint.name), | ||
1352 | sizeof(tp_name) - 1) < 0) | ||
1353 | return -EFAULT; | ||
1354 | tp_name[sizeof(tp_name) - 1] = 0; | ||
1355 | |||
1356 | btp = bpf_find_raw_tracepoint(tp_name); | ||
1357 | if (!btp) | ||
1358 | return -ENOENT; | ||
1359 | |||
1360 | raw_tp = kzalloc(sizeof(*raw_tp), GFP_USER); | ||
1361 | if (!raw_tp) | ||
1362 | return -ENOMEM; | ||
1363 | raw_tp->btp = btp; | ||
1364 | |||
1365 | prog = bpf_prog_get_type(attr->raw_tracepoint.prog_fd, | ||
1366 | BPF_PROG_TYPE_RAW_TRACEPOINT); | ||
1367 | if (IS_ERR(prog)) { | ||
1368 | err = PTR_ERR(prog); | ||
1369 | goto out_free_tp; | ||
1370 | } | ||
1371 | |||
1372 | err = bpf_probe_register(raw_tp->btp, prog); | ||
1373 | if (err) | ||
1374 | goto out_put_prog; | ||
1375 | |||
1376 | raw_tp->prog = prog; | ||
1377 | tp_fd = anon_inode_getfd("bpf-raw-tracepoint", &bpf_raw_tp_fops, raw_tp, | ||
1378 | O_CLOEXEC); | ||
1379 | if (tp_fd < 0) { | ||
1380 | bpf_probe_unregister(raw_tp->btp, prog); | ||
1381 | err = tp_fd; | ||
1382 | goto out_put_prog; | ||
1383 | } | ||
1384 | return tp_fd; | ||
1385 | |||
1386 | out_put_prog: | ||
1387 | bpf_prog_put(prog); | ||
1388 | out_free_tp: | ||
1389 | kfree(raw_tp); | ||
1390 | return err; | ||
1391 | } | ||
1392 | |||
1318 | #ifdef CONFIG_CGROUP_BPF | 1393 | #ifdef CONFIG_CGROUP_BPF |
1319 | 1394 | ||
1320 | #define BPF_PROG_ATTACH_LAST_FIELD attach_flags | 1395 | #define BPF_PROG_ATTACH_LAST_FIELD attach_flags |
@@ -1925,6 +2000,9 @@ SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, siz | |||
1925 | case BPF_OBJ_GET_INFO_BY_FD: | 2000 | case BPF_OBJ_GET_INFO_BY_FD: |
1926 | err = bpf_obj_get_info_by_fd(&attr, uattr); | 2001 | err = bpf_obj_get_info_by_fd(&attr, uattr); |
1927 | break; | 2002 | break; |
2003 | case BPF_RAW_TRACEPOINT_OPEN: | ||
2004 | err = bpf_raw_tracepoint_open(&attr); | ||
2005 | break; | ||
1928 | default: | 2006 | default: |
1929 | err = -EINVAL; | 2007 | err = -EINVAL; |
1930 | break; | 2008 | break; |
diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 7f9691c86b6e..463e72d18c4c 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c | |||
@@ -735,6 +735,86 @@ static const struct bpf_func_proto *pe_prog_func_proto(enum bpf_func_id func_id) | |||
735 | } | 735 | } |
736 | } | 736 | } |
737 | 737 | ||
738 | /* | ||
739 | * bpf_raw_tp_regs are separate from bpf_pt_regs used from skb/xdp | ||
740 | * to avoid potential recursive reuse issue when/if tracepoints are added | ||
741 | * inside bpf_*_event_output and/or bpf_get_stack_id | ||
742 | */ | ||
743 | static DEFINE_PER_CPU(struct pt_regs, bpf_raw_tp_regs); | ||
744 | BPF_CALL_5(bpf_perf_event_output_raw_tp, struct bpf_raw_tracepoint_args *, args, | ||
745 | struct bpf_map *, map, u64, flags, void *, data, u64, size) | ||
746 | { | ||
747 | struct pt_regs *regs = this_cpu_ptr(&bpf_raw_tp_regs); | ||
748 | |||
749 | perf_fetch_caller_regs(regs); | ||
750 | return ____bpf_perf_event_output(regs, map, flags, data, size); | ||
751 | } | ||
752 | |||
753 | static const struct bpf_func_proto bpf_perf_event_output_proto_raw_tp = { | ||
754 | .func = bpf_perf_event_output_raw_tp, | ||
755 | .gpl_only = true, | ||
756 | .ret_type = RET_INTEGER, | ||
757 | .arg1_type = ARG_PTR_TO_CTX, | ||
758 | .arg2_type = ARG_CONST_MAP_PTR, | ||
759 | .arg3_type = ARG_ANYTHING, | ||
760 | .arg4_type = ARG_PTR_TO_MEM, | ||
761 | .arg5_type = ARG_CONST_SIZE_OR_ZERO, | ||
762 | }; | ||
763 | |||
764 | BPF_CALL_3(bpf_get_stackid_raw_tp, struct bpf_raw_tracepoint_args *, args, | ||
765 | struct bpf_map *, map, u64, flags) | ||
766 | { | ||
767 | struct pt_regs *regs = this_cpu_ptr(&bpf_raw_tp_regs); | ||
768 | |||
769 | perf_fetch_caller_regs(regs); | ||
770 | /* similar to bpf_perf_event_output_tp, but pt_regs fetched differently */ | ||
771 | return bpf_get_stackid((unsigned long) regs, (unsigned long) map, | ||
772 | flags, 0, 0); | ||
773 | } | ||
774 | |||
775 | static const struct bpf_func_proto bpf_get_stackid_proto_raw_tp = { | ||
776 | .func = bpf_get_stackid_raw_tp, | ||
777 | .gpl_only = true, | ||
778 | .ret_type = RET_INTEGER, | ||
779 | .arg1_type = ARG_PTR_TO_CTX, | ||
780 | .arg2_type = ARG_CONST_MAP_PTR, | ||
781 | .arg3_type = ARG_ANYTHING, | ||
782 | }; | ||
783 | |||
784 | static const struct bpf_func_proto *raw_tp_prog_func_proto(enum bpf_func_id func_id) | ||
785 | { | ||
786 | switch (func_id) { | ||
787 | case BPF_FUNC_perf_event_output: | ||
788 | return &bpf_perf_event_output_proto_raw_tp; | ||
789 | case BPF_FUNC_get_stackid: | ||
790 | return &bpf_get_stackid_proto_raw_tp; | ||
791 | default: | ||
792 | return tracing_func_proto(func_id); | ||
793 | } | ||
794 | } | ||
795 | |||
796 | static bool raw_tp_prog_is_valid_access(int off, int size, | ||
797 | enum bpf_access_type type, | ||
798 | struct bpf_insn_access_aux *info) | ||
799 | { | ||
800 | /* largest tracepoint in the kernel has 12 args */ | ||
801 | if (off < 0 || off >= sizeof(__u64) * 12) | ||
802 | return false; | ||
803 | if (type != BPF_READ) | ||
804 | return false; | ||
805 | if (off % size != 0) | ||
806 | return false; | ||
807 | return true; | ||
808 | } | ||
809 | |||
810 | const struct bpf_verifier_ops raw_tracepoint_verifier_ops = { | ||
811 | .get_func_proto = raw_tp_prog_func_proto, | ||
812 | .is_valid_access = raw_tp_prog_is_valid_access, | ||
813 | }; | ||
814 | |||
815 | const struct bpf_prog_ops raw_tracepoint_prog_ops = { | ||
816 | }; | ||
817 | |||
738 | static bool pe_prog_is_valid_access(int off, int size, enum bpf_access_type type, | 818 | static bool pe_prog_is_valid_access(int off, int size, enum bpf_access_type type, |
739 | struct bpf_insn_access_aux *info) | 819 | struct bpf_insn_access_aux *info) |
740 | { | 820 | { |
@@ -908,3 +988,106 @@ int perf_event_query_prog_array(struct perf_event *event, void __user *info) | |||
908 | 988 | ||
909 | return ret; | 989 | return ret; |
910 | } | 990 | } |
991 | |||
992 | extern struct bpf_raw_event_map __start__bpf_raw_tp[]; | ||
993 | extern struct bpf_raw_event_map __stop__bpf_raw_tp[]; | ||
994 | |||
995 | struct bpf_raw_event_map *bpf_find_raw_tracepoint(const char *name) | ||
996 | { | ||
997 | struct bpf_raw_event_map *btp = __start__bpf_raw_tp; | ||
998 | |||
999 | for (; btp < __stop__bpf_raw_tp; btp++) { | ||
1000 | if (!strcmp(btp->tp->name, name)) | ||
1001 | return btp; | ||
1002 | } | ||
1003 | return NULL; | ||
1004 | } | ||
1005 | |||
1006 | static __always_inline | ||
1007 | void __bpf_trace_run(struct bpf_prog *prog, u64 *args) | ||
1008 | { | ||
1009 | rcu_read_lock(); | ||
1010 | preempt_disable(); | ||
1011 | (void) BPF_PROG_RUN(prog, args); | ||
1012 | preempt_enable(); | ||
1013 | rcu_read_unlock(); | ||
1014 | } | ||
1015 | |||
1016 | #define UNPACK(...) __VA_ARGS__ | ||
1017 | #define REPEAT_1(FN, DL, X, ...) FN(X) | ||
1018 | #define REPEAT_2(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_1(FN, DL, __VA_ARGS__) | ||
1019 | #define REPEAT_3(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_2(FN, DL, __VA_ARGS__) | ||
1020 | #define REPEAT_4(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_3(FN, DL, __VA_ARGS__) | ||
1021 | #define REPEAT_5(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_4(FN, DL, __VA_ARGS__) | ||
1022 | #define REPEAT_6(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_5(FN, DL, __VA_ARGS__) | ||
1023 | #define REPEAT_7(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_6(FN, DL, __VA_ARGS__) | ||
1024 | #define REPEAT_8(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_7(FN, DL, __VA_ARGS__) | ||
1025 | #define REPEAT_9(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_8(FN, DL, __VA_ARGS__) | ||
1026 | #define REPEAT_10(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_9(FN, DL, __VA_ARGS__) | ||
1027 | #define REPEAT_11(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_10(FN, DL, __VA_ARGS__) | ||
1028 | #define REPEAT_12(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_11(FN, DL, __VA_ARGS__) | ||
1029 | #define REPEAT(X, FN, DL, ...) REPEAT_##X(FN, DL, __VA_ARGS__) | ||
1030 | |||
1031 | #define SARG(X) u64 arg##X | ||
1032 | #define COPY(X) args[X] = arg##X | ||
1033 | |||
1034 | #define __DL_COM (,) | ||
1035 | #define __DL_SEM (;) | ||
1036 | |||
1037 | #define __SEQ_0_11 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 | ||
1038 | |||
1039 | #define BPF_TRACE_DEFN_x(x) \ | ||
1040 | void bpf_trace_run##x(struct bpf_prog *prog, \ | ||
1041 | REPEAT(x, SARG, __DL_COM, __SEQ_0_11)) \ | ||
1042 | { \ | ||
1043 | u64 args[x]; \ | ||
1044 | REPEAT(x, COPY, __DL_SEM, __SEQ_0_11); \ | ||
1045 | __bpf_trace_run(prog, args); \ | ||
1046 | } \ | ||
1047 | EXPORT_SYMBOL_GPL(bpf_trace_run##x) | ||
1048 | BPF_TRACE_DEFN_x(1); | ||
1049 | BPF_TRACE_DEFN_x(2); | ||
1050 | BPF_TRACE_DEFN_x(3); | ||
1051 | BPF_TRACE_DEFN_x(4); | ||
1052 | BPF_TRACE_DEFN_x(5); | ||
1053 | BPF_TRACE_DEFN_x(6); | ||
1054 | BPF_TRACE_DEFN_x(7); | ||
1055 | BPF_TRACE_DEFN_x(8); | ||
1056 | BPF_TRACE_DEFN_x(9); | ||
1057 | BPF_TRACE_DEFN_x(10); | ||
1058 | BPF_TRACE_DEFN_x(11); | ||
1059 | BPF_TRACE_DEFN_x(12); | ||
1060 | |||
1061 | static int __bpf_probe_register(struct bpf_raw_event_map *btp, struct bpf_prog *prog) | ||
1062 | { | ||
1063 | struct tracepoint *tp = btp->tp; | ||
1064 | |||
1065 | /* | ||
1066 | * check that program doesn't access arguments beyond what's | ||
1067 | * available in this tracepoint | ||
1068 | */ | ||
1069 | if (prog->aux->max_ctx_offset > btp->num_args * sizeof(u64)) | ||
1070 | return -EINVAL; | ||
1071 | |||
1072 | return tracepoint_probe_register(tp, (void *)btp->bpf_func, prog); | ||
1073 | } | ||
1074 | |||
1075 | int bpf_probe_register(struct bpf_raw_event_map *btp, struct bpf_prog *prog) | ||
1076 | { | ||
1077 | int err; | ||
1078 | |||
1079 | mutex_lock(&bpf_event_mutex); | ||
1080 | err = __bpf_probe_register(btp, prog); | ||
1081 | mutex_unlock(&bpf_event_mutex); | ||
1082 | return err; | ||
1083 | } | ||
1084 | |||
1085 | int bpf_probe_unregister(struct bpf_raw_event_map *btp, struct bpf_prog *prog) | ||
1086 | { | ||
1087 | int err; | ||
1088 | |||
1089 | mutex_lock(&bpf_event_mutex); | ||
1090 | err = tracepoint_probe_unregister(btp->tp, (void *)btp->bpf_func, prog); | ||
1091 | mutex_unlock(&bpf_event_mutex); | ||
1092 | return err; | ||
1093 | } | ||
diff --git a/net/mac802154/trace.h b/net/mac802154/trace.h index 2c8a43d3607f..df855c33daf2 100644 --- a/net/mac802154/trace.h +++ b/net/mac802154/trace.h | |||
@@ -33,7 +33,7 @@ | |||
33 | 33 | ||
34 | /* Tracing for driver callbacks */ | 34 | /* Tracing for driver callbacks */ |
35 | 35 | ||
36 | DECLARE_EVENT_CLASS(local_only_evt, | 36 | DECLARE_EVENT_CLASS(local_only_evt4, |
37 | TP_PROTO(struct ieee802154_local *local), | 37 | TP_PROTO(struct ieee802154_local *local), |
38 | TP_ARGS(local), | 38 | TP_ARGS(local), |
39 | TP_STRUCT__entry( | 39 | TP_STRUCT__entry( |
@@ -45,7 +45,7 @@ DECLARE_EVENT_CLASS(local_only_evt, | |||
45 | TP_printk(LOCAL_PR_FMT, LOCAL_PR_ARG) | 45 | TP_printk(LOCAL_PR_FMT, LOCAL_PR_ARG) |
46 | ); | 46 | ); |
47 | 47 | ||
48 | DEFINE_EVENT(local_only_evt, 802154_drv_return_void, | 48 | DEFINE_EVENT(local_only_evt4, 802154_drv_return_void, |
49 | TP_PROTO(struct ieee802154_local *local), | 49 | TP_PROTO(struct ieee802154_local *local), |
50 | TP_ARGS(local) | 50 | TP_ARGS(local) |
51 | ); | 51 | ); |
@@ -65,12 +65,12 @@ TRACE_EVENT(802154_drv_return_int, | |||
65 | __entry->ret) | 65 | __entry->ret) |
66 | ); | 66 | ); |
67 | 67 | ||
68 | DEFINE_EVENT(local_only_evt, 802154_drv_start, | 68 | DEFINE_EVENT(local_only_evt4, 802154_drv_start, |
69 | TP_PROTO(struct ieee802154_local *local), | 69 | TP_PROTO(struct ieee802154_local *local), |
70 | TP_ARGS(local) | 70 | TP_ARGS(local) |
71 | ); | 71 | ); |
72 | 72 | ||
73 | DEFINE_EVENT(local_only_evt, 802154_drv_stop, | 73 | DEFINE_EVENT(local_only_evt4, 802154_drv_stop, |
74 | TP_PROTO(struct ieee802154_local *local), | 74 | TP_PROTO(struct ieee802154_local *local), |
75 | TP_ARGS(local) | 75 | TP_ARGS(local) |
76 | ); | 76 | ); |
diff --git a/net/wireless/trace.h b/net/wireless/trace.h index 5152938b358d..018c81fa72fb 100644 --- a/net/wireless/trace.h +++ b/net/wireless/trace.h | |||
@@ -3137,7 +3137,7 @@ TRACE_EVENT(rdev_start_radar_detection, | |||
3137 | 3137 | ||
3138 | TRACE_EVENT(rdev_set_mcast_rate, | 3138 | TRACE_EVENT(rdev_set_mcast_rate, |
3139 | TP_PROTO(struct wiphy *wiphy, struct net_device *netdev, | 3139 | TP_PROTO(struct wiphy *wiphy, struct net_device *netdev, |
3140 | int mcast_rate[NUM_NL80211_BANDS]), | 3140 | int *mcast_rate), |
3141 | TP_ARGS(wiphy, netdev, mcast_rate), | 3141 | TP_ARGS(wiphy, netdev, mcast_rate), |
3142 | TP_STRUCT__entry( | 3142 | TP_STRUCT__entry( |
3143 | WIPHY_ENTRY | 3143 | WIPHY_ENTRY |
diff --git a/samples/bpf/Makefile b/samples/bpf/Makefile index 2c2a587e0942..4d6a6edd4bf6 100644 --- a/samples/bpf/Makefile +++ b/samples/bpf/Makefile | |||
@@ -119,6 +119,7 @@ always += offwaketime_kern.o | |||
119 | always += spintest_kern.o | 119 | always += spintest_kern.o |
120 | always += map_perf_test_kern.o | 120 | always += map_perf_test_kern.o |
121 | always += test_overhead_tp_kern.o | 121 | always += test_overhead_tp_kern.o |
122 | always += test_overhead_raw_tp_kern.o | ||
122 | always += test_overhead_kprobe_kern.o | 123 | always += test_overhead_kprobe_kern.o |
123 | always += parse_varlen.o parse_simple.o parse_ldabs.o | 124 | always += parse_varlen.o parse_simple.o parse_ldabs.o |
124 | always += test_cgrp2_tc_kern.o | 125 | always += test_cgrp2_tc_kern.o |
diff --git a/samples/bpf/bpf_load.c b/samples/bpf/bpf_load.c index b1a310c3ae89..bebe4188b4b3 100644 --- a/samples/bpf/bpf_load.c +++ b/samples/bpf/bpf_load.c | |||
@@ -61,6 +61,7 @@ static int load_and_attach(const char *event, struct bpf_insn *prog, int size) | |||
61 | bool is_kprobe = strncmp(event, "kprobe/", 7) == 0; | 61 | bool is_kprobe = strncmp(event, "kprobe/", 7) == 0; |
62 | bool is_kretprobe = strncmp(event, "kretprobe/", 10) == 0; | 62 | bool is_kretprobe = strncmp(event, "kretprobe/", 10) == 0; |
63 | bool is_tracepoint = strncmp(event, "tracepoint/", 11) == 0; | 63 | bool is_tracepoint = strncmp(event, "tracepoint/", 11) == 0; |
64 | bool is_raw_tracepoint = strncmp(event, "raw_tracepoint/", 15) == 0; | ||
64 | bool is_xdp = strncmp(event, "xdp", 3) == 0; | 65 | bool is_xdp = strncmp(event, "xdp", 3) == 0; |
65 | bool is_perf_event = strncmp(event, "perf_event", 10) == 0; | 66 | bool is_perf_event = strncmp(event, "perf_event", 10) == 0; |
66 | bool is_cgroup_skb = strncmp(event, "cgroup/skb", 10) == 0; | 67 | bool is_cgroup_skb = strncmp(event, "cgroup/skb", 10) == 0; |
@@ -85,6 +86,8 @@ static int load_and_attach(const char *event, struct bpf_insn *prog, int size) | |||
85 | prog_type = BPF_PROG_TYPE_KPROBE; | 86 | prog_type = BPF_PROG_TYPE_KPROBE; |
86 | } else if (is_tracepoint) { | 87 | } else if (is_tracepoint) { |
87 | prog_type = BPF_PROG_TYPE_TRACEPOINT; | 88 | prog_type = BPF_PROG_TYPE_TRACEPOINT; |
89 | } else if (is_raw_tracepoint) { | ||
90 | prog_type = BPF_PROG_TYPE_RAW_TRACEPOINT; | ||
88 | } else if (is_xdp) { | 91 | } else if (is_xdp) { |
89 | prog_type = BPF_PROG_TYPE_XDP; | 92 | prog_type = BPF_PROG_TYPE_XDP; |
90 | } else if (is_perf_event) { | 93 | } else if (is_perf_event) { |
@@ -131,6 +134,16 @@ static int load_and_attach(const char *event, struct bpf_insn *prog, int size) | |||
131 | return populate_prog_array(event, fd); | 134 | return populate_prog_array(event, fd); |
132 | } | 135 | } |
133 | 136 | ||
137 | if (is_raw_tracepoint) { | ||
138 | efd = bpf_raw_tracepoint_open(event + 15, fd); | ||
139 | if (efd < 0) { | ||
140 | printf("tracepoint %s %s\n", event + 15, strerror(errno)); | ||
141 | return -1; | ||
142 | } | ||
143 | event_fd[prog_cnt - 1] = efd; | ||
144 | return 0; | ||
145 | } | ||
146 | |||
134 | if (is_kprobe || is_kretprobe) { | 147 | if (is_kprobe || is_kretprobe) { |
135 | if (is_kprobe) | 148 | if (is_kprobe) |
136 | event += 7; | 149 | event += 7; |
@@ -587,6 +600,7 @@ static int do_load_bpf_file(const char *path, fixup_map_cb fixup_map) | |||
587 | if (memcmp(shname, "kprobe/", 7) == 0 || | 600 | if (memcmp(shname, "kprobe/", 7) == 0 || |
588 | memcmp(shname, "kretprobe/", 10) == 0 || | 601 | memcmp(shname, "kretprobe/", 10) == 0 || |
589 | memcmp(shname, "tracepoint/", 11) == 0 || | 602 | memcmp(shname, "tracepoint/", 11) == 0 || |
603 | memcmp(shname, "raw_tracepoint/", 15) == 0 || | ||
590 | memcmp(shname, "xdp", 3) == 0 || | 604 | memcmp(shname, "xdp", 3) == 0 || |
591 | memcmp(shname, "perf_event", 10) == 0 || | 605 | memcmp(shname, "perf_event", 10) == 0 || |
592 | memcmp(shname, "socket", 6) == 0 || | 606 | memcmp(shname, "socket", 6) == 0 || |
diff --git a/samples/bpf/test_overhead_raw_tp_kern.c b/samples/bpf/test_overhead_raw_tp_kern.c new file mode 100644 index 000000000000..d2af8bc1c805 --- /dev/null +++ b/samples/bpf/test_overhead_raw_tp_kern.c | |||
@@ -0,0 +1,17 @@ | |||
1 | // SPDX-License-Identifier: GPL-2.0 | ||
2 | /* Copyright (c) 2018 Facebook */ | ||
3 | #include <uapi/linux/bpf.h> | ||
4 | #include "bpf_helpers.h" | ||
5 | |||
6 | SEC("raw_tracepoint/task_rename") | ||
7 | int prog(struct bpf_raw_tracepoint_args *ctx) | ||
8 | { | ||
9 | return 0; | ||
10 | } | ||
11 | |||
12 | SEC("raw_tracepoint/urandom_read") | ||
13 | int prog2(struct bpf_raw_tracepoint_args *ctx) | ||
14 | { | ||
15 | return 0; | ||
16 | } | ||
17 | char _license[] SEC("license") = "GPL"; | ||
diff --git a/samples/bpf/test_overhead_user.c b/samples/bpf/test_overhead_user.c index d291167fd3c7..e1d35e07a10e 100644 --- a/samples/bpf/test_overhead_user.c +++ b/samples/bpf/test_overhead_user.c | |||
@@ -158,5 +158,17 @@ int main(int argc, char **argv) | |||
158 | unload_progs(); | 158 | unload_progs(); |
159 | } | 159 | } |
160 | 160 | ||
161 | if (test_flags & 0xC0) { | ||
162 | snprintf(filename, sizeof(filename), | ||
163 | "%s_raw_tp_kern.o", argv[0]); | ||
164 | if (load_bpf_file(filename)) { | ||
165 | printf("%s", bpf_log_buf); | ||
166 | return 1; | ||
167 | } | ||
168 | printf("w/RAW_TRACEPOINT\n"); | ||
169 | run_perf_test(num_cpu, test_flags >> 6); | ||
170 | unload_progs(); | ||
171 | } | ||
172 | |||
161 | return 0; | 173 | return 0; |
162 | } | 174 | } |
diff --git a/security/apparmor/include/path.h b/security/apparmor/include/path.h index 05fb3305671e..e042b994f2b8 100644 --- a/security/apparmor/include/path.h +++ b/security/apparmor/include/path.h | |||
@@ -43,15 +43,10 @@ struct aa_buffers { | |||
43 | 43 | ||
44 | DECLARE_PER_CPU(struct aa_buffers, aa_buffers); | 44 | DECLARE_PER_CPU(struct aa_buffers, aa_buffers); |
45 | 45 | ||
46 | #define COUNT_ARGS(X...) COUNT_ARGS_HELPER(, ##X, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0) | ||
47 | #define COUNT_ARGS_HELPER(_0, _1, _2, _3, _4, _5, _6, _7, _8, _9, n, X...) n | ||
48 | #define CONCAT(X, Y) X ## Y | ||
49 | #define CONCAT_AFTER(X, Y) CONCAT(X, Y) | ||
50 | |||
51 | #define ASSIGN(FN, X, N) ((X) = FN(N)) | 46 | #define ASSIGN(FN, X, N) ((X) = FN(N)) |
52 | #define EVAL1(FN, X) ASSIGN(FN, X, 0) /*X = FN(0)*/ | 47 | #define EVAL1(FN, X) ASSIGN(FN, X, 0) /*X = FN(0)*/ |
53 | #define EVAL2(FN, X, Y...) do { ASSIGN(FN, X, 1); EVAL1(FN, Y); } while (0) | 48 | #define EVAL2(FN, X, Y...) do { ASSIGN(FN, X, 1); EVAL1(FN, Y); } while (0) |
54 | #define EVAL(FN, X...) CONCAT_AFTER(EVAL, COUNT_ARGS(X))(FN, X) | 49 | #define EVAL(FN, X...) CONCATENATE(EVAL, COUNT_ARGS(X))(FN, X) |
55 | 50 | ||
56 | #define for_each_cpu_buffer(I) for ((I) = 0; (I) < MAX_PATH_BUFFERS; (I)++) | 51 | #define for_each_cpu_buffer(I) for ((I) = 0; (I) < MAX_PATH_BUFFERS; (I)++) |
57 | 52 | ||
diff --git a/sound/firewire/amdtp-stream-trace.h b/sound/firewire/amdtp-stream-trace.h index ea0d486652c8..54cdd4ffa9ce 100644 --- a/sound/firewire/amdtp-stream-trace.h +++ b/sound/firewire/amdtp-stream-trace.h | |||
@@ -14,7 +14,7 @@ | |||
14 | #include <linux/tracepoint.h> | 14 | #include <linux/tracepoint.h> |
15 | 15 | ||
16 | TRACE_EVENT(in_packet, | 16 | TRACE_EVENT(in_packet, |
17 | TP_PROTO(const struct amdtp_stream *s, u32 cycles, u32 cip_header[2], unsigned int payload_length, unsigned int index), | 17 | TP_PROTO(const struct amdtp_stream *s, u32 cycles, u32 *cip_header, unsigned int payload_length, unsigned int index), |
18 | TP_ARGS(s, cycles, cip_header, payload_length, index), | 18 | TP_ARGS(s, cycles, cip_header, payload_length, index), |
19 | TP_STRUCT__entry( | 19 | TP_STRUCT__entry( |
20 | __field(unsigned int, second) | 20 | __field(unsigned int, second) |
diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index d245c41213ac..58060bec999d 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h | |||
@@ -94,6 +94,7 @@ enum bpf_cmd { | |||
94 | BPF_MAP_GET_FD_BY_ID, | 94 | BPF_MAP_GET_FD_BY_ID, |
95 | BPF_OBJ_GET_INFO_BY_FD, | 95 | BPF_OBJ_GET_INFO_BY_FD, |
96 | BPF_PROG_QUERY, | 96 | BPF_PROG_QUERY, |
97 | BPF_RAW_TRACEPOINT_OPEN, | ||
97 | }; | 98 | }; |
98 | 99 | ||
99 | enum bpf_map_type { | 100 | enum bpf_map_type { |
@@ -134,6 +135,7 @@ enum bpf_prog_type { | |||
134 | BPF_PROG_TYPE_SK_SKB, | 135 | BPF_PROG_TYPE_SK_SKB, |
135 | BPF_PROG_TYPE_CGROUP_DEVICE, | 136 | BPF_PROG_TYPE_CGROUP_DEVICE, |
136 | BPF_PROG_TYPE_SK_MSG, | 137 | BPF_PROG_TYPE_SK_MSG, |
138 | BPF_PROG_TYPE_RAW_TRACEPOINT, | ||
137 | }; | 139 | }; |
138 | 140 | ||
139 | enum bpf_attach_type { | 141 | enum bpf_attach_type { |
@@ -344,6 +346,11 @@ union bpf_attr { | |||
344 | __aligned_u64 prog_ids; | 346 | __aligned_u64 prog_ids; |
345 | __u32 prog_cnt; | 347 | __u32 prog_cnt; |
346 | } query; | 348 | } query; |
349 | |||
350 | struct { | ||
351 | __u64 name; | ||
352 | __u32 prog_fd; | ||
353 | } raw_tracepoint; | ||
347 | } __attribute__((aligned(8))); | 354 | } __attribute__((aligned(8))); |
348 | 355 | ||
349 | /* BPF helper function descriptions: | 356 | /* BPF helper function descriptions: |
@@ -1151,4 +1158,8 @@ struct bpf_cgroup_dev_ctx { | |||
1151 | __u32 minor; | 1158 | __u32 minor; |
1152 | }; | 1159 | }; |
1153 | 1160 | ||
1161 | struct bpf_raw_tracepoint_args { | ||
1162 | __u64 args[0]; | ||
1163 | }; | ||
1164 | |||
1154 | #endif /* _UAPI__LINUX_BPF_H__ */ | 1165 | #endif /* _UAPI__LINUX_BPF_H__ */ |
diff --git a/tools/lib/bpf/bpf.c b/tools/lib/bpf/bpf.c index 592a58a2b681..e0500055f1a6 100644 --- a/tools/lib/bpf/bpf.c +++ b/tools/lib/bpf/bpf.c | |||
@@ -428,6 +428,17 @@ int bpf_obj_get_info_by_fd(int prog_fd, void *info, __u32 *info_len) | |||
428 | return err; | 428 | return err; |
429 | } | 429 | } |
430 | 430 | ||
431 | int bpf_raw_tracepoint_open(const char *name, int prog_fd) | ||
432 | { | ||
433 | union bpf_attr attr; | ||
434 | |||
435 | bzero(&attr, sizeof(attr)); | ||
436 | attr.raw_tracepoint.name = ptr_to_u64(name); | ||
437 | attr.raw_tracepoint.prog_fd = prog_fd; | ||
438 | |||
439 | return sys_bpf(BPF_RAW_TRACEPOINT_OPEN, &attr, sizeof(attr)); | ||
440 | } | ||
441 | |||
431 | int bpf_set_link_xdp_fd(int ifindex, int fd, __u32 flags) | 442 | int bpf_set_link_xdp_fd(int ifindex, int fd, __u32 flags) |
432 | { | 443 | { |
433 | struct sockaddr_nl sa; | 444 | struct sockaddr_nl sa; |
diff --git a/tools/lib/bpf/bpf.h b/tools/lib/bpf/bpf.h index 8d18fb73d7fb..ee59342c6f42 100644 --- a/tools/lib/bpf/bpf.h +++ b/tools/lib/bpf/bpf.h | |||
@@ -79,4 +79,5 @@ int bpf_map_get_fd_by_id(__u32 id); | |||
79 | int bpf_obj_get_info_by_fd(int prog_fd, void *info, __u32 *info_len); | 79 | int bpf_obj_get_info_by_fd(int prog_fd, void *info, __u32 *info_len); |
80 | int bpf_prog_query(int target_fd, enum bpf_attach_type type, __u32 query_flags, | 80 | int bpf_prog_query(int target_fd, enum bpf_attach_type type, __u32 query_flags, |
81 | __u32 *attach_flags, __u32 *prog_ids, __u32 *prog_cnt); | 81 | __u32 *attach_flags, __u32 *prog_ids, __u32 *prog_cnt); |
82 | int bpf_raw_tracepoint_open(const char *name, int prog_fd); | ||
82 | #endif | 83 | #endif |
diff --git a/tools/testing/selftests/bpf/test_progs.c b/tools/testing/selftests/bpf/test_progs.c index e9df48b306df..faadbe233966 100644 --- a/tools/testing/selftests/bpf/test_progs.c +++ b/tools/testing/selftests/bpf/test_progs.c | |||
@@ -877,7 +877,7 @@ static void test_stacktrace_map() | |||
877 | 877 | ||
878 | err = bpf_prog_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj, &prog_fd); | 878 | err = bpf_prog_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj, &prog_fd); |
879 | if (CHECK(err, "prog_load", "err %d errno %d\n", err, errno)) | 879 | if (CHECK(err, "prog_load", "err %d errno %d\n", err, errno)) |
880 | goto out; | 880 | return; |
881 | 881 | ||
882 | /* Get the ID for the sched/sched_switch tracepoint */ | 882 | /* Get the ID for the sched/sched_switch tracepoint */ |
883 | snprintf(buf, sizeof(buf), | 883 | snprintf(buf, sizeof(buf), |
@@ -888,8 +888,7 @@ static void test_stacktrace_map() | |||
888 | 888 | ||
889 | bytes = read(efd, buf, sizeof(buf)); | 889 | bytes = read(efd, buf, sizeof(buf)); |
890 | close(efd); | 890 | close(efd); |
891 | if (CHECK(bytes <= 0 || bytes >= sizeof(buf), | 891 | if (bytes <= 0 || bytes >= sizeof(buf)) |
892 | "read", "bytes %d errno %d\n", bytes, errno)) | ||
893 | goto close_prog; | 892 | goto close_prog; |
894 | 893 | ||
895 | /* Open the perf event and attach bpf progrram */ | 894 | /* Open the perf event and attach bpf progrram */ |
@@ -906,29 +905,24 @@ static void test_stacktrace_map() | |||
906 | goto close_prog; | 905 | goto close_prog; |
907 | 906 | ||
908 | err = ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0); | 907 | err = ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0); |
909 | if (CHECK(err, "perf_event_ioc_enable", "err %d errno %d\n", | 908 | if (err) |
910 | err, errno)) | 909 | goto disable_pmu; |
911 | goto close_pmu; | ||
912 | 910 | ||
913 | err = ioctl(pmu_fd, PERF_EVENT_IOC_SET_BPF, prog_fd); | 911 | err = ioctl(pmu_fd, PERF_EVENT_IOC_SET_BPF, prog_fd); |
914 | if (CHECK(err, "perf_event_ioc_set_bpf", "err %d errno %d\n", | 912 | if (err) |
915 | err, errno)) | ||
916 | goto disable_pmu; | 913 | goto disable_pmu; |
917 | 914 | ||
918 | /* find map fds */ | 915 | /* find map fds */ |
919 | control_map_fd = bpf_find_map(__func__, obj, "control_map"); | 916 | control_map_fd = bpf_find_map(__func__, obj, "control_map"); |
920 | if (CHECK(control_map_fd < 0, "bpf_find_map control_map", | 917 | if (control_map_fd < 0) |
921 | "err %d errno %d\n", err, errno)) | ||
922 | goto disable_pmu; | 918 | goto disable_pmu; |
923 | 919 | ||
924 | stackid_hmap_fd = bpf_find_map(__func__, obj, "stackid_hmap"); | 920 | stackid_hmap_fd = bpf_find_map(__func__, obj, "stackid_hmap"); |
925 | if (CHECK(stackid_hmap_fd < 0, "bpf_find_map stackid_hmap", | 921 | if (stackid_hmap_fd < 0) |
926 | "err %d errno %d\n", err, errno)) | ||
927 | goto disable_pmu; | 922 | goto disable_pmu; |
928 | 923 | ||
929 | stackmap_fd = bpf_find_map(__func__, obj, "stackmap"); | 924 | stackmap_fd = bpf_find_map(__func__, obj, "stackmap"); |
930 | if (CHECK(stackmap_fd < 0, "bpf_find_map stackmap", "err %d errno %d\n", | 925 | if (stackmap_fd < 0) |
931 | err, errno)) | ||
932 | goto disable_pmu; | 926 | goto disable_pmu; |
933 | 927 | ||
934 | /* give some time for bpf program run */ | 928 | /* give some time for bpf program run */ |
@@ -945,24 +939,78 @@ static void test_stacktrace_map() | |||
945 | err = compare_map_keys(stackid_hmap_fd, stackmap_fd); | 939 | err = compare_map_keys(stackid_hmap_fd, stackmap_fd); |
946 | if (CHECK(err, "compare_map_keys stackid_hmap vs. stackmap", | 940 | if (CHECK(err, "compare_map_keys stackid_hmap vs. stackmap", |
947 | "err %d errno %d\n", err, errno)) | 941 | "err %d errno %d\n", err, errno)) |
948 | goto disable_pmu; | 942 | goto disable_pmu_noerr; |
949 | 943 | ||
950 | err = compare_map_keys(stackmap_fd, stackid_hmap_fd); | 944 | err = compare_map_keys(stackmap_fd, stackid_hmap_fd); |
951 | if (CHECK(err, "compare_map_keys stackmap vs. stackid_hmap", | 945 | if (CHECK(err, "compare_map_keys stackmap vs. stackid_hmap", |
952 | "err %d errno %d\n", err, errno)) | 946 | "err %d errno %d\n", err, errno)) |
953 | ; /* fall through */ | 947 | goto disable_pmu_noerr; |
954 | 948 | ||
949 | goto disable_pmu_noerr; | ||
955 | disable_pmu: | 950 | disable_pmu: |
951 | error_cnt++; | ||
952 | disable_pmu_noerr: | ||
956 | ioctl(pmu_fd, PERF_EVENT_IOC_DISABLE); | 953 | ioctl(pmu_fd, PERF_EVENT_IOC_DISABLE); |
957 | |||
958 | close_pmu: | ||
959 | close(pmu_fd); | 954 | close(pmu_fd); |
960 | |||
961 | close_prog: | 955 | close_prog: |
962 | bpf_object__close(obj); | 956 | bpf_object__close(obj); |
957 | } | ||
963 | 958 | ||
964 | out: | 959 | static void test_stacktrace_map_raw_tp() |
965 | return; | 960 | { |
961 | int control_map_fd, stackid_hmap_fd, stackmap_fd; | ||
962 | const char *file = "./test_stacktrace_map.o"; | ||
963 | int efd, err, prog_fd; | ||
964 | __u32 key, val, duration = 0; | ||
965 | struct bpf_object *obj; | ||
966 | |||
967 | err = bpf_prog_load(file, BPF_PROG_TYPE_RAW_TRACEPOINT, &obj, &prog_fd); | ||
968 | if (CHECK(err, "prog_load raw tp", "err %d errno %d\n", err, errno)) | ||
969 | return; | ||
970 | |||
971 | efd = bpf_raw_tracepoint_open("sched_switch", prog_fd); | ||
972 | if (CHECK(efd < 0, "raw_tp_open", "err %d errno %d\n", efd, errno)) | ||
973 | goto close_prog; | ||
974 | |||
975 | /* find map fds */ | ||
976 | control_map_fd = bpf_find_map(__func__, obj, "control_map"); | ||
977 | if (control_map_fd < 0) | ||
978 | goto close_prog; | ||
979 | |||
980 | stackid_hmap_fd = bpf_find_map(__func__, obj, "stackid_hmap"); | ||
981 | if (stackid_hmap_fd < 0) | ||
982 | goto close_prog; | ||
983 | |||
984 | stackmap_fd = bpf_find_map(__func__, obj, "stackmap"); | ||
985 | if (stackmap_fd < 0) | ||
986 | goto close_prog; | ||
987 | |||
988 | /* give some time for bpf program run */ | ||
989 | sleep(1); | ||
990 | |||
991 | /* disable stack trace collection */ | ||
992 | key = 0; | ||
993 | val = 1; | ||
994 | bpf_map_update_elem(control_map_fd, &key, &val, 0); | ||
995 | |||
996 | /* for every element in stackid_hmap, we can find a corresponding one | ||
997 | * in stackmap, and vise versa. | ||
998 | */ | ||
999 | err = compare_map_keys(stackid_hmap_fd, stackmap_fd); | ||
1000 | if (CHECK(err, "compare_map_keys stackid_hmap vs. stackmap", | ||
1001 | "err %d errno %d\n", err, errno)) | ||
1002 | goto close_prog; | ||
1003 | |||
1004 | err = compare_map_keys(stackmap_fd, stackid_hmap_fd); | ||
1005 | if (CHECK(err, "compare_map_keys stackmap vs. stackid_hmap", | ||
1006 | "err %d errno %d\n", err, errno)) | ||
1007 | goto close_prog; | ||
1008 | |||
1009 | goto close_prog_noerr; | ||
1010 | close_prog: | ||
1011 | error_cnt++; | ||
1012 | close_prog_noerr: | ||
1013 | bpf_object__close(obj); | ||
966 | } | 1014 | } |
967 | 1015 | ||
968 | static int extract_build_id(char *build_id, size_t size) | 1016 | static int extract_build_id(char *build_id, size_t size) |
@@ -1138,6 +1186,7 @@ int main(void) | |||
1138 | test_tp_attach_query(); | 1186 | test_tp_attach_query(); |
1139 | test_stacktrace_map(); | 1187 | test_stacktrace_map(); |
1140 | test_stacktrace_build_id(); | 1188 | test_stacktrace_build_id(); |
1189 | test_stacktrace_map_raw_tp(); | ||
1141 | 1190 | ||
1142 | printf("Summary: %d PASSED, %d FAILED\n", pass_cnt, error_cnt); | 1191 | printf("Summary: %d PASSED, %d FAILED\n", pass_cnt, error_cnt); |
1143 | return error_cnt ? EXIT_FAILURE : EXIT_SUCCESS; | 1192 | return error_cnt ? EXIT_FAILURE : EXIT_SUCCESS; |