aboutsummaryrefslogtreecommitdiffstats
path: root/tools
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2018-01-31 17:31:10 -0500
committerLinus Torvalds <torvalds@linux-foundation.org>2018-01-31 17:31:10 -0500
commitb2fe5fa68642860e7de76167c3111623aa0d5de1 (patch)
treeb7f9b89b7039ecefbc35fe3c8e73a6ff972641dd /tools
parenta103950e0dd2058df5e8a8d4a915707bdcf205f0 (diff)
parenta54667f6728c2714a400f3c884727da74b6d1717 (diff)
Merge git://git.kernel.org/pub/scm/linux/kernel/git/davem/net-next
Pull networking updates from David Miller: 1) Significantly shrink the core networking routing structures. Result of http://vger.kernel.org/~davem/seoul2017_netdev_keynote.pdf 2) Add netdevsim driver for testing various offloads, from Jakub Kicinski. 3) Support cross-chip FDB operations in DSA, from Vivien Didelot. 4) Add a 2nd listener hash table for TCP, similar to what was done for UDP. From Martin KaFai Lau. 5) Add eBPF based queue selection to tun, from Jason Wang. 6) Lockless qdisc support, from John Fastabend. 7) SCTP stream interleave support, from Xin Long. 8) Smoother TCP receive autotuning, from Eric Dumazet. 9) Lots of erspan tunneling enhancements, from William Tu. 10) Add true function call support to BPF, from Alexei Starovoitov. 11) Add explicit support for GRO HW offloading, from Michael Chan. 12) Support extack generation in more netlink subsystems. From Alexander Aring, Quentin Monnet, and Jakub Kicinski. 13) Add 1000BaseX, flow control, and EEE support to mvneta driver. From Russell King. 14) Add flow table abstraction to netfilter, from Pablo Neira Ayuso. 15) Many improvements and simplifications to the NFP driver bpf JIT, from Jakub Kicinski. 16) Support for ipv6 non-equal cost multipath routing, from Ido Schimmel. 17) Add resource abstration to devlink, from Arkadi Sharshevsky. 18) Packet scheduler classifier shared filter block support, from Jiri Pirko. 19) Avoid locking in act_csum, from Davide Caratti. 20) devinet_ioctl() simplifications from Al viro. 21) More TCP bpf improvements from Lawrence Brakmo. 22) Add support for onlink ipv6 route flag, similar to ipv4, from David Ahern. * git://git.kernel.org/pub/scm/linux/kernel/git/davem/net-next: (1925 commits) tls: Add support for encryption using async offload accelerator ip6mr: fix stale iterator net/sched: kconfig: Remove blank help texts openvswitch: meter: Use 64-bit arithmetic instead of 32-bit tcp_nv: fix potential integer overflow in tcpnv_acked r8169: fix RTL8168EP take too long to complete driver initialization. qmi_wwan: Add support for Quectel EP06 rtnetlink: enable IFLA_IF_NETNSID for RTM_NEWLINK ipmr: Fix ptrdiff_t print formatting ibmvnic: Wait for device response when changing MAC qlcnic: fix deadlock bug tcp: release sk_frag.page in tcp_disconnect ipv4: Get the address of interface correctly. net_sched: gen_estimator: fix lockdep splat net: macb: Handle HRESP error net/mlx5e: IPoIB, Fix copy-paste bug in flow steering refactoring ipv6: addrconf: break critical section in addrconf_verify_rtnl() ipv6: change route cache aging logic i40e/i40evf: Update DESC_NEEDED value to reflect larger value bnxt_en: cleanup DIM work on device shutdown ...
Diffstat (limited to 'tools')
-rw-r--r--tools/bpf/Makefile29
-rw-r--r--tools/bpf/bpf_jit_disasm.c14
-rw-r--r--tools/bpf/bpftool/Documentation/Makefile30
-rw-r--r--tools/bpf/bpftool/Documentation/bpftool-cgroup.rst118
-rw-r--r--tools/bpf/bpftool/Documentation/bpftool-map.rst8
-rw-r--r--tools/bpf/bpftool/Documentation/bpftool-prog.rst16
-rw-r--r--tools/bpf/bpftool/Documentation/bpftool.rst12
-rw-r--r--tools/bpf/bpftool/Makefile90
-rw-r--r--tools/bpf/bpftool/bash-completion/bpftool8
-rw-r--r--tools/bpf/bpftool/cgroup.c308
-rw-r--r--tools/bpf/bpftool/common.c195
-rw-r--r--tools/bpf/bpftool/jit_disasm.c23
-rw-r--r--tools/bpf/bpftool/main.c16
-rw-r--r--tools/bpf/bpftool/main.h9
-rw-r--r--tools/bpf/bpftool/map.c11
-rw-r--r--tools/bpf/bpftool/prog.c229
-rw-r--r--tools/build/feature/Makefile4
-rw-r--r--tools/build/feature/test-disassembler-four-args.c15
-rw-r--r--tools/include/uapi/linux/bpf.h116
-rw-r--r--tools/include/uapi/linux/perf_event.h22
-rw-r--r--tools/lib/bpf/Makefile24
-rw-r--r--tools/lib/bpf/bpf.h2
-rw-r--r--tools/lib/bpf/libbpf.c200
-rw-r--r--tools/scripts/Makefile.include1
-rw-r--r--tools/testing/selftests/bpf/.gitignore7
-rw-r--r--tools/testing/selftests/bpf/Makefile20
-rw-r--r--tools/testing/selftests/bpf/bpf_helpers.h5
-rw-r--r--tools/testing/selftests/bpf/config2
-rw-r--r--tools/testing/selftests/bpf/sample_map_ret0.c34
-rw-r--r--tools/testing/selftests/bpf/sample_ret0.c7
-rwxr-xr-xtools/testing/selftests/bpf/tcp_client.py51
-rwxr-xr-xtools/testing/selftests/bpf/tcp_server.py83
-rw-r--r--tools/testing/selftests/bpf/test_align.c178
-rw-r--r--tools/testing/selftests/bpf/test_dev_cgroup.c11
-rw-r--r--tools/testing/selftests/bpf/test_l4lb_noinline.c473
-rw-r--r--tools/testing/selftests/bpf/test_lpm_map.c217
-rw-r--r--tools/testing/selftests/bpf/test_maps.c32
-rwxr-xr-xtools/testing/selftests/bpf/test_offload.py1085
-rw-r--r--tools/testing/selftests/bpf/test_progs.c355
-rw-r--r--tools/testing/selftests/bpf/test_stacktrace_map.c62
-rw-r--r--tools/testing/selftests/bpf/test_tcpbpf.h16
-rw-r--r--tools/testing/selftests/bpf/test_tcpbpf_kern.c115
-rw-r--r--tools/testing/selftests/bpf/test_tcpbpf_user.c126
-rw-r--r--tools/testing/selftests/bpf/test_tracepoint.c26
-rw-r--r--tools/testing/selftests/bpf/test_verifier.c2334
-rw-r--r--tools/testing/selftests/bpf/test_xdp_noinline.c833
-rw-r--r--tools/testing/selftests/net/Makefile1
-rwxr-xr-xtools/testing/selftests/net/fib_tests.sh429
-rw-r--r--tools/testing/selftests/net/msg_zerocopy.c21
-rwxr-xr-xtools/testing/selftests/net/rtnetlink.sh229
-rw-r--r--tools/virtio/linux/kernel.h2
-rw-r--r--tools/virtio/linux/thread_info.h1
-rw-r--r--tools/virtio/ringtest/main.h59
-rw-r--r--tools/virtio/ringtest/ptr_ring.c2
54 files changed, 8019 insertions, 297 deletions
diff --git a/tools/bpf/Makefile b/tools/bpf/Makefile
index 07a6697466ef..c8ec0ae16bf0 100644
--- a/tools/bpf/Makefile
+++ b/tools/bpf/Makefile
@@ -9,6 +9,35 @@ MAKE = make
9CFLAGS += -Wall -O2 9CFLAGS += -Wall -O2
10CFLAGS += -D__EXPORTED_HEADERS__ -I../../include/uapi -I../../include 10CFLAGS += -D__EXPORTED_HEADERS__ -I../../include/uapi -I../../include
11 11
12ifeq ($(srctree),)
13srctree := $(patsubst %/,%,$(dir $(CURDIR)))
14srctree := $(patsubst %/,%,$(dir $(srctree)))
15endif
16
17FEATURE_USER = .bpf
18FEATURE_TESTS = libbfd disassembler-four-args
19FEATURE_DISPLAY = libbfd disassembler-four-args
20
21check_feat := 1
22NON_CHECK_FEAT_TARGETS := clean bpftool_clean
23ifdef MAKECMDGOALS
24ifeq ($(filter-out $(NON_CHECK_FEAT_TARGETS),$(MAKECMDGOALS)),)
25 check_feat := 0
26endif
27endif
28
29ifeq ($(check_feat),1)
30ifeq ($(FEATURES_DUMP),)
31include $(srctree)/tools/build/Makefile.feature
32else
33include $(FEATURES_DUMP)
34endif
35endif
36
37ifeq ($(feature-disassembler-four-args), 1)
38CFLAGS += -DDISASM_FOUR_ARGS_SIGNATURE
39endif
40
12%.yacc.c: %.y 41%.yacc.c: %.y
13 $(YACC) -o $@ -d $< 42 $(YACC) -o $@ -d $<
14 43
diff --git a/tools/bpf/bpf_jit_disasm.c b/tools/bpf/bpf_jit_disasm.c
index 75bf526a0168..58c2bab4ef6e 100644
--- a/tools/bpf/bpf_jit_disasm.c
+++ b/tools/bpf/bpf_jit_disasm.c
@@ -72,7 +72,14 @@ static void get_asm_insns(uint8_t *image, size_t len, int opcodes)
72 72
73 disassemble_init_for_target(&info); 73 disassemble_init_for_target(&info);
74 74
75#ifdef DISASM_FOUR_ARGS_SIGNATURE
76 disassemble = disassembler(info.arch,
77 bfd_big_endian(bfdf),
78 info.mach,
79 bfdf);
80#else
75 disassemble = disassembler(bfdf); 81 disassemble = disassembler(bfdf);
82#endif
76 assert(disassemble); 83 assert(disassemble);
77 84
78 do { 85 do {
@@ -165,7 +172,8 @@ static uint8_t *get_last_jit_image(char *haystack, size_t hlen,
165{ 172{
166 char *ptr, *pptr, *tmp; 173 char *ptr, *pptr, *tmp;
167 off_t off = 0; 174 off_t off = 0;
168 int ret, flen, proglen, pass, ulen = 0; 175 unsigned int proglen;
176 int ret, flen, pass, ulen = 0;
169 regmatch_t pmatch[1]; 177 regmatch_t pmatch[1];
170 unsigned long base; 178 unsigned long base;
171 regex_t regex; 179 regex_t regex;
@@ -192,7 +200,7 @@ static uint8_t *get_last_jit_image(char *haystack, size_t hlen,
192 } 200 }
193 201
194 ptr = haystack + off - (pmatch[0].rm_eo - pmatch[0].rm_so); 202 ptr = haystack + off - (pmatch[0].rm_eo - pmatch[0].rm_so);
195 ret = sscanf(ptr, "flen=%d proglen=%d pass=%d image=%lx", 203 ret = sscanf(ptr, "flen=%d proglen=%u pass=%d image=%lx",
196 &flen, &proglen, &pass, &base); 204 &flen, &proglen, &pass, &base);
197 if (ret != 4) { 205 if (ret != 4) {
198 regfree(&regex); 206 regfree(&regex);
@@ -232,7 +240,7 @@ static uint8_t *get_last_jit_image(char *haystack, size_t hlen,
232 } 240 }
233 241
234 assert(ulen == proglen); 242 assert(ulen == proglen);
235 printf("%d bytes emitted from JIT compiler (pass:%d, flen:%d)\n", 243 printf("%u bytes emitted from JIT compiler (pass:%d, flen:%d)\n",
236 proglen, pass, flen); 244 proglen, pass, flen);
237 printf("%lx + <x>:\n", base); 245 printf("%lx + <x>:\n", base);
238 246
diff --git a/tools/bpf/bpftool/Documentation/Makefile b/tools/bpf/bpftool/Documentation/Makefile
index 37292bb5ce60..c462a928e03d 100644
--- a/tools/bpf/bpftool/Documentation/Makefile
+++ b/tools/bpf/bpftool/Documentation/Makefile
@@ -3,12 +3,16 @@ include ../../../scripts/utilities.mak
3 3
4INSTALL ?= install 4INSTALL ?= install
5RM ?= rm -f 5RM ?= rm -f
6RMDIR ?= rmdir --ignore-fail-on-non-empty
6 7
7# Make the path relative to DESTDIR, not prefix 8ifeq ($(V),1)
8ifndef DESTDIR 9 Q =
9prefix ?= /usr/local 10else
11 Q = @
10endif 12endif
11mandir ?= $(prefix)/share/man 13
14prefix ?= /usr/local
15mandir ?= $(prefix)/man
12man8dir = $(mandir)/man8 16man8dir = $(mandir)/man8
13 17
14MAN8_RST = $(wildcard *.rst) 18MAN8_RST = $(wildcard *.rst)
@@ -20,15 +24,21 @@ man: man8
20man8: $(DOC_MAN8) 24man8: $(DOC_MAN8)
21 25
22$(OUTPUT)%.8: %.rst 26$(OUTPUT)%.8: %.rst
23 rst2man $< > $@ 27 $(QUIET_GEN)rst2man $< > $@
24 28
25clean: 29clean:
26 $(call QUIET_CLEAN, Documentation) $(RM) $(DOC_MAN8) 30 $(call QUIET_CLEAN, Documentation)
31 $(Q)$(RM) $(DOC_MAN8)
27 32
28install: man 33install: man
29 $(call QUIET_INSTALL, Documentation-man) \ 34 $(call QUIET_INSTALL, Documentation-man)
30 $(INSTALL) -d -m 755 $(DESTDIR)$(man8dir); \ 35 $(Q)$(INSTALL) -d -m 755 $(DESTDIR)$(man8dir)
31 $(INSTALL) -m 644 $(DOC_MAN8) $(DESTDIR)$(man8dir); 36 $(Q)$(INSTALL) -m 644 $(DOC_MAN8) $(DESTDIR)$(man8dir)
37
38uninstall:
39 $(call QUIET_UNINST, Documentation-man)
40 $(Q)$(RM) $(addprefix $(DESTDIR)$(man8dir)/,$(_DOC_MAN8))
41 $(Q)$(RMDIR) $(DESTDIR)$(man8dir)
32 42
33.PHONY: man man8 clean install 43.PHONY: man man8 clean install uninstall
34.DEFAULT_GOAL := man 44.DEFAULT_GOAL := man
diff --git a/tools/bpf/bpftool/Documentation/bpftool-cgroup.rst b/tools/bpf/bpftool/Documentation/bpftool-cgroup.rst
new file mode 100644
index 000000000000..2fe2a1bdbe3e
--- /dev/null
+++ b/tools/bpf/bpftool/Documentation/bpftool-cgroup.rst
@@ -0,0 +1,118 @@
1================
2bpftool-cgroup
3================
4-------------------------------------------------------------------------------
5tool for inspection and simple manipulation of eBPF progs
6-------------------------------------------------------------------------------
7
8:Manual section: 8
9
10SYNOPSIS
11========
12
13 **bpftool** [*OPTIONS*] **cgroup** *COMMAND*
14
15 *OPTIONS* := { { **-j** | **--json** } [{ **-p** | **--pretty** }] | { **-f** | **--bpffs** } }
16
17 *COMMANDS* :=
18 { **show** | **list** | **attach** | **detach** | **help** }
19
20MAP COMMANDS
21=============
22
23| **bpftool** **cgroup { show | list }** *CGROUP*
24| **bpftool** **cgroup attach** *CGROUP* *ATTACH_TYPE* *PROG* [*ATTACH_FLAGS*]
25| **bpftool** **cgroup detach** *CGROUP* *ATTACH_TYPE* *PROG*
26| **bpftool** **cgroup help**
27|
28| *PROG* := { **id** *PROG_ID* | **pinned** *FILE* | **tag** *PROG_TAG* }
29| *ATTACH_TYPE* := { *ingress* | *egress* | *sock_create* | *sock_ops* | *device* }
30| *ATTACH_FLAGS* := { *multi* | *override* }
31
32DESCRIPTION
33===========
34 **bpftool cgroup { show | list }** *CGROUP*
35 List all programs attached to the cgroup *CGROUP*.
36
37 Output will start with program ID followed by attach type,
38 attach flags and program name.
39
40 **bpftool cgroup attach** *CGROUP* *ATTACH_TYPE* *PROG* [*ATTACH_FLAGS*]
41 Attach program *PROG* to the cgroup *CGROUP* with attach type
42 *ATTACH_TYPE* and optional *ATTACH_FLAGS*.
43
44 *ATTACH_FLAGS* can be one of: **override** if a sub-cgroup installs
45 some bpf program, the program in this cgroup yields to sub-cgroup
46 program; **multi** if a sub-cgroup installs some bpf program,
47 that cgroup program gets run in addition to the program in this
48 cgroup.
49
50 Only one program is allowed to be attached to a cgroup with
51 no attach flags or the **override** flag. Attaching another
52 program will release old program and attach the new one.
53
54 Multiple programs are allowed to be attached to a cgroup with
55 **multi**. They are executed in FIFO order (those that were
56 attached first, run first).
57
58 Non-default *ATTACH_FLAGS* are supported by kernel version 4.14
59 and later.
60
61 *ATTACH_TYPE* can be on of:
62 **ingress** ingress path of the inet socket (since 4.10);
63 **egress** egress path of the inet socket (since 4.10);
64 **sock_create** opening of an inet socket (since 4.10);
65 **sock_ops** various socket operations (since 4.12);
66 **device** device access (since 4.15).
67
68 **bpftool cgroup detach** *CGROUP* *ATTACH_TYPE* *PROG*
69 Detach *PROG* from the cgroup *CGROUP* and attach type
70 *ATTACH_TYPE*.
71
72 **bpftool prog help**
73 Print short help message.
74
75OPTIONS
76=======
77 -h, --help
78 Print short generic help message (similar to **bpftool help**).
79
80 -v, --version
81 Print version number (similar to **bpftool version**).
82
83 -j, --json
84 Generate JSON output. For commands that cannot produce JSON, this
85 option has no effect.
86
87 -p, --pretty
88 Generate human-readable JSON output. Implies **-j**.
89
90 -f, --bpffs
91 Show file names of pinned programs.
92
93EXAMPLES
94========
95|
96| **# mount -t bpf none /sys/fs/bpf/**
97| **# mkdir /sys/fs/cgroup/test.slice**
98| **# bpftool prog load ./device_cgroup.o /sys/fs/bpf/prog**
99| **# bpftool cgroup attach /sys/fs/cgroup/test.slice/ device id 1 allow_multi**
100
101**# bpftool cgroup list /sys/fs/cgroup/test.slice/**
102
103::
104
105 ID AttachType AttachFlags Name
106 1 device allow_multi bpf_prog1
107
108|
109| **# bpftool cgroup detach /sys/fs/cgroup/test.slice/ device id 1**
110| **# bpftool cgroup list /sys/fs/cgroup/test.slice/**
111
112::
113
114 ID AttachType AttachFlags Name
115
116SEE ALSO
117========
118 **bpftool**\ (8), **bpftool-prog**\ (8), **bpftool-map**\ (8)
diff --git a/tools/bpf/bpftool/Documentation/bpftool-map.rst b/tools/bpf/bpftool/Documentation/bpftool-map.rst
index 9f51a268eb06..0ab32b312aec 100644
--- a/tools/bpf/bpftool/Documentation/bpftool-map.rst
+++ b/tools/bpf/bpftool/Documentation/bpftool-map.rst
@@ -15,13 +15,13 @@ SYNOPSIS
15 *OPTIONS* := { { **-j** | **--json** } [{ **-p** | **--pretty** }] | { **-f** | **--bpffs** } } 15 *OPTIONS* := { { **-j** | **--json** } [{ **-p** | **--pretty** }] | { **-f** | **--bpffs** } }
16 16
17 *COMMANDS* := 17 *COMMANDS* :=
18 { **show** | **dump** | **update** | **lookup** | **getnext** | **delete** 18 { **show** | **list** | **dump** | **update** | **lookup** | **getnext** | **delete**
19 | **pin** | **help** } 19 | **pin** | **help** }
20 20
21MAP COMMANDS 21MAP COMMANDS
22============= 22=============
23 23
24| **bpftool** **map show** [*MAP*] 24| **bpftool** **map { show | list }** [*MAP*]
25| **bpftool** **map dump** *MAP* 25| **bpftool** **map dump** *MAP*
26| **bpftool** **map update** *MAP* **key** *BYTES* **value** *VALUE* [*UPDATE_FLAGS*] 26| **bpftool** **map update** *MAP* **key** *BYTES* **value** *VALUE* [*UPDATE_FLAGS*]
27| **bpftool** **map lookup** *MAP* **key** *BYTES* 27| **bpftool** **map lookup** *MAP* **key** *BYTES*
@@ -36,7 +36,7 @@ MAP COMMANDS
36 36
37DESCRIPTION 37DESCRIPTION
38=========== 38===========
39 **bpftool map show** [*MAP*] 39 **bpftool map { show | list }** [*MAP*]
40 Show information about loaded maps. If *MAP* is specified 40 Show information about loaded maps. If *MAP* is specified
41 show information only about given map, otherwise list all 41 show information only about given map, otherwise list all
42 maps currently loaded on the system. 42 maps currently loaded on the system.
@@ -128,4 +128,4 @@ EXAMPLES
128 128
129SEE ALSO 129SEE ALSO
130======== 130========
131 **bpftool**\ (8), **bpftool-prog**\ (8) 131 **bpftool**\ (8), **bpftool-prog**\ (8), **bpftool-cgroup**\ (8)
diff --git a/tools/bpf/bpftool/Documentation/bpftool-prog.rst b/tools/bpf/bpftool/Documentation/bpftool-prog.rst
index 36e8d1c3c40d..e4ceee7f2dff 100644
--- a/tools/bpf/bpftool/Documentation/bpftool-prog.rst
+++ b/tools/bpf/bpftool/Documentation/bpftool-prog.rst
@@ -15,22 +15,23 @@ SYNOPSIS
15 *OPTIONS* := { { **-j** | **--json** } [{ **-p** | **--pretty** }] | { **-f** | **--bpffs** } } 15 *OPTIONS* := { { **-j** | **--json** } [{ **-p** | **--pretty** }] | { **-f** | **--bpffs** } }
16 16
17 *COMMANDS* := 17 *COMMANDS* :=
18 { **show** | **dump xlated** | **dump jited** | **pin** | **help** } 18 { **show** | **list** | **dump xlated** | **dump jited** | **pin** | **load** | **help** }
19 19
20MAP COMMANDS 20MAP COMMANDS
21============= 21=============
22 22
23| **bpftool** **prog show** [*PROG*] 23| **bpftool** **prog { show | list }** [*PROG*]
24| **bpftool** **prog dump xlated** *PROG* [{**file** *FILE* | **opcodes**}] 24| **bpftool** **prog dump xlated** *PROG* [{**file** *FILE* | **opcodes**}]
25| **bpftool** **prog dump jited** *PROG* [{**file** *FILE* | **opcodes**}] 25| **bpftool** **prog dump jited** *PROG* [{**file** *FILE* | **opcodes**}]
26| **bpftool** **prog pin** *PROG* *FILE* 26| **bpftool** **prog pin** *PROG* *FILE*
27| **bpftool** **prog load** *OBJ* *FILE*
27| **bpftool** **prog help** 28| **bpftool** **prog help**
28| 29|
29| *PROG* := { **id** *PROG_ID* | **pinned** *FILE* | **tag** *PROG_TAG* } 30| *PROG* := { **id** *PROG_ID* | **pinned** *FILE* | **tag** *PROG_TAG* }
30 31
31DESCRIPTION 32DESCRIPTION
32=========== 33===========
33 **bpftool prog show** [*PROG*] 34 **bpftool prog { show | list }** [*PROG*]
34 Show information about loaded programs. If *PROG* is 35 Show information about loaded programs. If *PROG* is
35 specified show information only about given program, otherwise 36 specified show information only about given program, otherwise
36 list all programs currently loaded on the system. 37 list all programs currently loaded on the system.
@@ -57,6 +58,11 @@ DESCRIPTION
57 58
58 Note: *FILE* must be located in *bpffs* mount. 59 Note: *FILE* must be located in *bpffs* mount.
59 60
61 **bpftool prog load** *OBJ* *FILE*
62 Load bpf program from binary *OBJ* and pin as *FILE*.
63
64 Note: *FILE* must be located in *bpffs* mount.
65
60 **bpftool prog help** 66 **bpftool prog help**
61 Print short help message. 67 Print short help message.
62 68
@@ -126,8 +132,10 @@ EXAMPLES
126| 132|
127| **# mount -t bpf none /sys/fs/bpf/** 133| **# mount -t bpf none /sys/fs/bpf/**
128| **# bpftool prog pin id 10 /sys/fs/bpf/prog** 134| **# bpftool prog pin id 10 /sys/fs/bpf/prog**
135| **# bpftool prog load ./my_prog.o /sys/fs/bpf/prog2**
129| **# ls -l /sys/fs/bpf/** 136| **# ls -l /sys/fs/bpf/**
130| -rw------- 1 root root 0 Jul 22 01:43 prog 137| -rw------- 1 root root 0 Jul 22 01:43 prog
138| -rw------- 1 root root 0 Jul 22 01:44 prog2
131 139
132**# bpftool prog dum jited pinned /sys/fs/bpf/prog opcodes** 140**# bpftool prog dum jited pinned /sys/fs/bpf/prog opcodes**
133 141
@@ -147,4 +155,4 @@ EXAMPLES
147 155
148SEE ALSO 156SEE ALSO
149======== 157========
150 **bpftool**\ (8), **bpftool-map**\ (8) 158 **bpftool**\ (8), **bpftool-map**\ (8), **bpftool-cgroup**\ (8)
diff --git a/tools/bpf/bpftool/Documentation/bpftool.rst b/tools/bpf/bpftool/Documentation/bpftool.rst
index 926c03d5a8da..20689a321ffe 100644
--- a/tools/bpf/bpftool/Documentation/bpftool.rst
+++ b/tools/bpf/bpftool/Documentation/bpftool.rst
@@ -16,17 +16,19 @@ SYNOPSIS
16 16
17 **bpftool** **version** 17 **bpftool** **version**
18 18
19 *OBJECT* := { **map** | **program** } 19 *OBJECT* := { **map** | **program** | **cgroup** }
20 20
21 *OPTIONS* := { { **-V** | **--version** } | { **-h** | **--help** } 21 *OPTIONS* := { { **-V** | **--version** } | { **-h** | **--help** }
22 | { **-j** | **--json** } [{ **-p** | **--pretty** }] } 22 | { **-j** | **--json** } [{ **-p** | **--pretty** }] }
23 23
24 *MAP-COMMANDS* := 24 *MAP-COMMANDS* :=
25 { **show** | **dump** | **update** | **lookup** | **getnext** | **delete** 25 { **show** | **list** | **dump** | **update** | **lookup** | **getnext** | **delete**
26 | **pin** | **help** } 26 | **pin** | **help** }
27 27
28 *PROG-COMMANDS* := { **show** | **dump jited** | **dump xlated** | **pin** 28 *PROG-COMMANDS* := { **show** | **list** | **dump jited** | **dump xlated** | **pin**
29 | **help** } 29 | **load** | **help** }
30
31 *CGROUP-COMMANDS* := { **show** | **list** | **attach** | **detach** | **help** }
30 32
31DESCRIPTION 33DESCRIPTION
32=========== 34===========
@@ -53,4 +55,4 @@ OPTIONS
53 55
54SEE ALSO 56SEE ALSO
55======== 57========
56 **bpftool-map**\ (8), **bpftool-prog**\ (8) 58 **bpftool-map**\ (8), **bpftool-prog**\ (8), **bpftool-cgroup**\ (8)
diff --git a/tools/bpf/bpftool/Makefile b/tools/bpf/bpftool/Makefile
index ec3052c0b004..26901ec87361 100644
--- a/tools/bpf/bpftool/Makefile
+++ b/tools/bpf/bpftool/Makefile
@@ -1,25 +1,10 @@
1include ../../scripts/Makefile.include 1include ../../scripts/Makefile.include
2
3include ../../scripts/utilities.mak 2include ../../scripts/utilities.mak
4 3
5ifeq ($(srctree),) 4ifeq ($(srctree),)
6srctree := $(patsubst %/,%,$(dir $(CURDIR))) 5srctree := $(patsubst %/,%,$(dir $(CURDIR)))
7srctree := $(patsubst %/,%,$(dir $(srctree))) 6srctree := $(patsubst %/,%,$(dir $(srctree)))
8srctree := $(patsubst %/,%,$(dir $(srctree))) 7srctree := $(patsubst %/,%,$(dir $(srctree)))
9#$(info Determined 'srctree' to be $(srctree))
10endif
11
12ifneq ($(objtree),)
13#$(info Determined 'objtree' to be $(objtree))
14endif
15
16ifneq ($(OUTPUT),)
17#$(info Determined 'OUTPUT' to be $(OUTPUT))
18# Adding $(OUTPUT) as a directory to look for source files,
19# because use generated output files as sources dependency
20# for flex/bison parsers.
21VPATH += $(OUTPUT)
22export VPATH
23endif 8endif
24 9
25ifeq ($(V),1) 10ifeq ($(V),1)
@@ -28,16 +13,18 @@ else
28 Q = @ 13 Q = @
29endif 14endif
30 15
31BPF_DIR = $(srctree)/tools/lib/bpf/ 16BPF_DIR = $(srctree)/tools/lib/bpf/
32 17
33ifneq ($(OUTPUT),) 18ifneq ($(OUTPUT),)
34 BPF_PATH=$(OUTPUT) 19 BPF_PATH = $(OUTPUT)
35else 20else
36 BPF_PATH=$(BPF_DIR) 21 BPF_PATH = $(BPF_DIR)
37endif 22endif
38 23
39LIBBPF = $(BPF_PATH)libbpf.a 24LIBBPF = $(BPF_PATH)libbpf.a
40 25
26BPFTOOL_VERSION=$(shell make --no-print-directory -sC ../../.. kernelversion)
27
41$(LIBBPF): FORCE 28$(LIBBPF): FORCE
42 $(Q)$(MAKE) -C $(BPF_DIR) OUTPUT=$(OUTPUT) $(OUTPUT)libbpf.a FEATURES_DUMP=$(FEATURE_DUMP_EXPORT) 29 $(Q)$(MAKE) -C $(BPF_DIR) OUTPUT=$(OUTPUT) $(OUTPUT)libbpf.a FEATURES_DUMP=$(FEATURE_DUMP_EXPORT)
43 30
@@ -45,22 +32,50 @@ $(LIBBPF)-clean:
45 $(call QUIET_CLEAN, libbpf) 32 $(call QUIET_CLEAN, libbpf)
46 $(Q)$(MAKE) -C $(BPF_DIR) OUTPUT=$(OUTPUT) clean >/dev/null 33 $(Q)$(MAKE) -C $(BPF_DIR) OUTPUT=$(OUTPUT) clean >/dev/null
47 34
48prefix = /usr/local 35prefix ?= /usr/local
49bash_compdir ?= /usr/share/bash-completion/completions 36bash_compdir ?= /usr/share/bash-completion/completions
50 37
51CC = gcc 38CC = gcc
52 39
53CFLAGS += -O2 40CFLAGS += -O2
54CFLAGS += -W -Wall -Wextra -Wno-unused-parameter -Wshadow 41CFLAGS += -W -Wall -Wextra -Wno-unused-parameter -Wshadow
55CFLAGS += -D__EXPORTED_HEADERS__ -I$(srctree)/tools/include/uapi -I$(srctree)/tools/include -I$(srctree)/tools/lib/bpf -I$(srctree)/kernel/bpf/ 42CFLAGS += -DPACKAGE='"bpftool"' -D__EXPORTED_HEADERS__ -I$(srctree)/tools/include/uapi -I$(srctree)/tools/include -I$(srctree)/tools/lib/bpf -I$(srctree)/kernel/bpf/
43CFLAGS += -DBPFTOOL_VERSION='"$(BPFTOOL_VERSION)"'
56LIBS = -lelf -lbfd -lopcodes $(LIBBPF) 44LIBS = -lelf -lbfd -lopcodes $(LIBBPF)
57 45
46INSTALL ?= install
47RM ?= rm -f
48
49FEATURE_USER = .bpftool
50FEATURE_TESTS = libbfd disassembler-four-args
51FEATURE_DISPLAY = libbfd disassembler-four-args
52
53check_feat := 1
54NON_CHECK_FEAT_TARGETS := clean uninstall doc doc-clean doc-install doc-uninstall
55ifdef MAKECMDGOALS
56ifeq ($(filter-out $(NON_CHECK_FEAT_TARGETS),$(MAKECMDGOALS)),)
57 check_feat := 0
58endif
59endif
60
61ifeq ($(check_feat),1)
62ifeq ($(FEATURES_DUMP),)
63include $(srctree)/tools/build/Makefile.feature
64else
65include $(FEATURES_DUMP)
66endif
67endif
68
69ifeq ($(feature-disassembler-four-args), 1)
70CFLAGS += -DDISASM_FOUR_ARGS_SIGNATURE
71endif
72
58include $(wildcard *.d) 73include $(wildcard *.d)
59 74
60all: $(OUTPUT)bpftool 75all: $(OUTPUT)bpftool
61 76
62SRCS=$(wildcard *.c) 77SRCS = $(wildcard *.c)
63OBJS=$(patsubst %.c,$(OUTPUT)%.o,$(SRCS)) $(OUTPUT)disasm.o 78OBJS = $(patsubst %.c,$(OUTPUT)%.o,$(SRCS)) $(OUTPUT)disasm.o
64 79
65$(OUTPUT)disasm.o: $(srctree)/kernel/bpf/disasm.c 80$(OUTPUT)disasm.o: $(srctree)/kernel/bpf/disasm.c
66 $(QUIET_CC)$(COMPILE.c) -MMD -o $@ $< 81 $(QUIET_CC)$(COMPILE.c) -MMD -o $@ $<
@@ -73,21 +88,34 @@ $(OUTPUT)%.o: %.c
73 88
74clean: $(LIBBPF)-clean 89clean: $(LIBBPF)-clean
75 $(call QUIET_CLEAN, bpftool) 90 $(call QUIET_CLEAN, bpftool)
76 $(Q)rm -rf $(OUTPUT)bpftool $(OUTPUT)*.o $(OUTPUT)*.d 91 $(Q)$(RM) $(OUTPUT)bpftool $(OUTPUT)*.o $(OUTPUT)*.d
92
93install: $(OUTPUT)bpftool
94 $(call QUIET_INSTALL, bpftool)
95 $(Q)$(INSTALL) -m 0755 -d $(DESTDIR)$(prefix)/sbin
96 $(Q)$(INSTALL) $(OUTPUT)bpftool $(DESTDIR)$(prefix)/sbin/bpftool
97 $(Q)$(INSTALL) -m 0755 -d $(DESTDIR)$(bash_compdir)
98 $(Q)$(INSTALL) -m 0644 bash-completion/bpftool $(DESTDIR)$(bash_compdir)
77 99
78install: 100uninstall:
79 install -m 0755 -d $(prefix)/sbin 101 $(call QUIET_UNINST, bpftool)
80 install $(OUTPUT)bpftool $(prefix)/sbin/bpftool 102 $(Q)$(RM) $(DESTDIR)$(prefix)/sbin/bpftool
81 install -m 0755 -d $(bash_compdir) 103 $(Q)$(RM) $(DESTDIR)$(bash_compdir)/bpftool
82 install -m 0644 bash-completion/bpftool $(bash_compdir)
83 104
84doc: 105doc:
85 $(Q)$(MAKE) -C Documentation/ 106 $(call descend,Documentation)
107
108doc-clean:
109 $(call descend,Documentation,clean)
86 110
87doc-install: 111doc-install:
88 $(Q)$(MAKE) -C Documentation/ install 112 $(call descend,Documentation,install)
113
114doc-uninstall:
115 $(call descend,Documentation,uninstall)
89 116
90FORCE: 117FORCE:
91 118
92.PHONY: all clean FORCE install doc doc-install 119.PHONY: all FORCE clean install uninstall
120.PHONY: doc doc-clean doc-install doc-uninstall
93.DEFAULT_GOAL := all 121.DEFAULT_GOAL := all
diff --git a/tools/bpf/bpftool/bash-completion/bpftool b/tools/bpf/bpftool/bash-completion/bpftool
index 7febee05c8e7..0137866bb8f6 100644
--- a/tools/bpf/bpftool/bash-completion/bpftool
+++ b/tools/bpf/bpftool/bash-completion/bpftool
@@ -197,7 +197,7 @@ _bpftool()
197 197
198 local PROG_TYPE='id pinned tag' 198 local PROG_TYPE='id pinned tag'
199 case $command in 199 case $command in
200 show) 200 show|list)
201 [[ $prev != "$command" ]] && return 0 201 [[ $prev != "$command" ]] && return 0
202 COMPREPLY=( $( compgen -W "$PROG_TYPE" -- "$cur" ) ) 202 COMPREPLY=( $( compgen -W "$PROG_TYPE" -- "$cur" ) )
203 return 0 203 return 0
@@ -232,7 +232,7 @@ _bpftool()
232 ;; 232 ;;
233 *) 233 *)
234 [[ $prev == $object ]] && \ 234 [[ $prev == $object ]] && \
235 COMPREPLY=( $( compgen -W 'dump help pin show' -- \ 235 COMPREPLY=( $( compgen -W 'dump help pin show list' -- \
236 "$cur" ) ) 236 "$cur" ) )
237 ;; 237 ;;
238 esac 238 esac
@@ -240,7 +240,7 @@ _bpftool()
240 map) 240 map)
241 local MAP_TYPE='id pinned' 241 local MAP_TYPE='id pinned'
242 case $command in 242 case $command in
243 show|dump) 243 show|list|dump)
244 case $prev in 244 case $prev in
245 $command) 245 $command)
246 COMPREPLY=( $( compgen -W "$MAP_TYPE" -- "$cur" ) ) 246 COMPREPLY=( $( compgen -W "$MAP_TYPE" -- "$cur" ) )
@@ -343,7 +343,7 @@ _bpftool()
343 *) 343 *)
344 [[ $prev == $object ]] && \ 344 [[ $prev == $object ]] && \
345 COMPREPLY=( $( compgen -W 'delete dump getnext help \ 345 COMPREPLY=( $( compgen -W 'delete dump getnext help \
346 lookup pin show update' -- "$cur" ) ) 346 lookup pin show list update' -- "$cur" ) )
347 ;; 347 ;;
348 esac 348 esac
349 ;; 349 ;;
diff --git a/tools/bpf/bpftool/cgroup.c b/tools/bpf/bpftool/cgroup.c
new file mode 100644
index 000000000000..cae32a61cb18
--- /dev/null
+++ b/tools/bpf/bpftool/cgroup.c
@@ -0,0 +1,308 @@
1// SPDX-License-Identifier: GPL-2.0+
2// Copyright (C) 2017 Facebook
3// Author: Roman Gushchin <guro@fb.com>
4
5#include <fcntl.h>
6#include <stdlib.h>
7#include <string.h>
8#include <sys/stat.h>
9#include <sys/types.h>
10#include <unistd.h>
11
12#include <bpf.h>
13
14#include "main.h"
15
16#define HELP_SPEC_ATTACH_FLAGS \
17 "ATTACH_FLAGS := { multi | override }"
18
19#define HELP_SPEC_ATTACH_TYPES \
20 "ATTACH_TYPE := { ingress | egress | sock_create | sock_ops | device }"
21
22static const char * const attach_type_strings[] = {
23 [BPF_CGROUP_INET_INGRESS] = "ingress",
24 [BPF_CGROUP_INET_EGRESS] = "egress",
25 [BPF_CGROUP_INET_SOCK_CREATE] = "sock_create",
26 [BPF_CGROUP_SOCK_OPS] = "sock_ops",
27 [BPF_CGROUP_DEVICE] = "device",
28 [__MAX_BPF_ATTACH_TYPE] = NULL,
29};
30
31static enum bpf_attach_type parse_attach_type(const char *str)
32{
33 enum bpf_attach_type type;
34
35 for (type = 0; type < __MAX_BPF_ATTACH_TYPE; type++) {
36 if (attach_type_strings[type] &&
37 is_prefix(str, attach_type_strings[type]))
38 return type;
39 }
40
41 return __MAX_BPF_ATTACH_TYPE;
42}
43
44static int show_bpf_prog(int id, const char *attach_type_str,
45 const char *attach_flags_str)
46{
47 struct bpf_prog_info info = {};
48 __u32 info_len = sizeof(info);
49 int prog_fd;
50
51 prog_fd = bpf_prog_get_fd_by_id(id);
52 if (prog_fd < 0)
53 return -1;
54
55 if (bpf_obj_get_info_by_fd(prog_fd, &info, &info_len)) {
56 close(prog_fd);
57 return -1;
58 }
59
60 if (json_output) {
61 jsonw_start_object(json_wtr);
62 jsonw_uint_field(json_wtr, "id", info.id);
63 jsonw_string_field(json_wtr, "attach_type",
64 attach_type_str);
65 jsonw_string_field(json_wtr, "attach_flags",
66 attach_flags_str);
67 jsonw_string_field(json_wtr, "name", info.name);
68 jsonw_end_object(json_wtr);
69 } else {
70 printf("%-8u %-15s %-15s %-15s\n", info.id,
71 attach_type_str,
72 attach_flags_str,
73 info.name);
74 }
75
76 close(prog_fd);
77 return 0;
78}
79
80static int show_attached_bpf_progs(int cgroup_fd, enum bpf_attach_type type)
81{
82 __u32 prog_ids[1024] = {0};
83 char *attach_flags_str;
84 __u32 prog_cnt, iter;
85 __u32 attach_flags;
86 char buf[32];
87 int ret;
88
89 prog_cnt = ARRAY_SIZE(prog_ids);
90 ret = bpf_prog_query(cgroup_fd, type, 0, &attach_flags, prog_ids,
91 &prog_cnt);
92 if (ret)
93 return ret;
94
95 if (prog_cnt == 0)
96 return 0;
97
98 switch (attach_flags) {
99 case BPF_F_ALLOW_MULTI:
100 attach_flags_str = "multi";
101 break;
102 case BPF_F_ALLOW_OVERRIDE:
103 attach_flags_str = "override";
104 break;
105 case 0:
106 attach_flags_str = "";
107 break;
108 default:
109 snprintf(buf, sizeof(buf), "unknown(%x)", attach_flags);
110 attach_flags_str = buf;
111 }
112
113 for (iter = 0; iter < prog_cnt; iter++)
114 show_bpf_prog(prog_ids[iter], attach_type_strings[type],
115 attach_flags_str);
116
117 return 0;
118}
119
120static int do_show(int argc, char **argv)
121{
122 enum bpf_attach_type type;
123 int cgroup_fd;
124 int ret = -1;
125
126 if (argc < 1) {
127 p_err("too few parameters for cgroup show");
128 goto exit;
129 } else if (argc > 1) {
130 p_err("too many parameters for cgroup show");
131 goto exit;
132 }
133
134 cgroup_fd = open(argv[0], O_RDONLY);
135 if (cgroup_fd < 0) {
136 p_err("can't open cgroup %s", argv[1]);
137 goto exit;
138 }
139
140 if (json_output)
141 jsonw_start_array(json_wtr);
142 else
143 printf("%-8s %-15s %-15s %-15s\n", "ID", "AttachType",
144 "AttachFlags", "Name");
145
146 for (type = 0; type < __MAX_BPF_ATTACH_TYPE; type++) {
147 /*
148 * Not all attach types may be supported, so it's expected,
149 * that some requests will fail.
150 * If we were able to get the show for at least one
151 * attach type, let's return 0.
152 */
153 if (show_attached_bpf_progs(cgroup_fd, type) == 0)
154 ret = 0;
155 }
156
157 if (json_output)
158 jsonw_end_array(json_wtr);
159
160 close(cgroup_fd);
161exit:
162 return ret;
163}
164
165static int do_attach(int argc, char **argv)
166{
167 enum bpf_attach_type attach_type;
168 int cgroup_fd, prog_fd;
169 int attach_flags = 0;
170 int ret = -1;
171 int i;
172
173 if (argc < 4) {
174 p_err("too few parameters for cgroup attach");
175 goto exit;
176 }
177
178 cgroup_fd = open(argv[0], O_RDONLY);
179 if (cgroup_fd < 0) {
180 p_err("can't open cgroup %s", argv[1]);
181 goto exit;
182 }
183
184 attach_type = parse_attach_type(argv[1]);
185 if (attach_type == __MAX_BPF_ATTACH_TYPE) {
186 p_err("invalid attach type");
187 goto exit_cgroup;
188 }
189
190 argc -= 2;
191 argv = &argv[2];
192 prog_fd = prog_parse_fd(&argc, &argv);
193 if (prog_fd < 0)
194 goto exit_cgroup;
195
196 for (i = 0; i < argc; i++) {
197 if (is_prefix(argv[i], "multi")) {
198 attach_flags |= BPF_F_ALLOW_MULTI;
199 } else if (is_prefix(argv[i], "override")) {
200 attach_flags |= BPF_F_ALLOW_OVERRIDE;
201 } else {
202 p_err("unknown option: %s", argv[i]);
203 goto exit_cgroup;
204 }
205 }
206
207 if (bpf_prog_attach(prog_fd, cgroup_fd, attach_type, attach_flags)) {
208 p_err("failed to attach program");
209 goto exit_prog;
210 }
211
212 if (json_output)
213 jsonw_null(json_wtr);
214
215 ret = 0;
216
217exit_prog:
218 close(prog_fd);
219exit_cgroup:
220 close(cgroup_fd);
221exit:
222 return ret;
223}
224
225static int do_detach(int argc, char **argv)
226{
227 enum bpf_attach_type attach_type;
228 int prog_fd, cgroup_fd;
229 int ret = -1;
230
231 if (argc < 4) {
232 p_err("too few parameters for cgroup detach");
233 goto exit;
234 }
235
236 cgroup_fd = open(argv[0], O_RDONLY);
237 if (cgroup_fd < 0) {
238 p_err("can't open cgroup %s", argv[1]);
239 goto exit;
240 }
241
242 attach_type = parse_attach_type(argv[1]);
243 if (attach_type == __MAX_BPF_ATTACH_TYPE) {
244 p_err("invalid attach type");
245 goto exit_cgroup;
246 }
247
248 argc -= 2;
249 argv = &argv[2];
250 prog_fd = prog_parse_fd(&argc, &argv);
251 if (prog_fd < 0)
252 goto exit_cgroup;
253
254 if (bpf_prog_detach2(prog_fd, cgroup_fd, attach_type)) {
255 p_err("failed to detach program");
256 goto exit_prog;
257 }
258
259 if (json_output)
260 jsonw_null(json_wtr);
261
262 ret = 0;
263
264exit_prog:
265 close(prog_fd);
266exit_cgroup:
267 close(cgroup_fd);
268exit:
269 return ret;
270}
271
272static int do_help(int argc, char **argv)
273{
274 if (json_output) {
275 jsonw_null(json_wtr);
276 return 0;
277 }
278
279 fprintf(stderr,
280 "Usage: %s %s { show | list } CGROUP\n"
281 " %s %s attach CGROUP ATTACH_TYPE PROG [ATTACH_FLAGS]\n"
282 " %s %s detach CGROUP ATTACH_TYPE PROG\n"
283 " %s %s help\n"
284 "\n"
285 " " HELP_SPEC_ATTACH_TYPES "\n"
286 " " HELP_SPEC_ATTACH_FLAGS "\n"
287 " " HELP_SPEC_PROGRAM "\n"
288 " " HELP_SPEC_OPTIONS "\n"
289 "",
290 bin_name, argv[-2], bin_name, argv[-2],
291 bin_name, argv[-2], bin_name, argv[-2]);
292
293 return 0;
294}
295
296static const struct cmd cmds[] = {
297 { "show", do_show },
298 { "list", do_show },
299 { "attach", do_attach },
300 { "detach", do_detach },
301 { "help", do_help },
302 { 0 }
303};
304
305int do_cgroup(int argc, char **argv)
306{
307 return cmd_select(cmds, argc, argv, do_help);
308}
diff --git a/tools/bpf/bpftool/common.c b/tools/bpf/bpftool/common.c
index 2bd3b280e6dd..0b482c0070e0 100644
--- a/tools/bpf/bpftool/common.c
+++ b/tools/bpf/bpftool/common.c
@@ -34,6 +34,7 @@
34/* Author: Jakub Kicinski <kubakici@wp.pl> */ 34/* Author: Jakub Kicinski <kubakici@wp.pl> */
35 35
36#include <errno.h> 36#include <errno.h>
37#include <fcntl.h>
37#include <fts.h> 38#include <fts.h>
38#include <libgen.h> 39#include <libgen.h>
39#include <mntent.h> 40#include <mntent.h>
@@ -44,7 +45,9 @@
44#include <unistd.h> 45#include <unistd.h>
45#include <linux/limits.h> 46#include <linux/limits.h>
46#include <linux/magic.h> 47#include <linux/magic.h>
48#include <net/if.h>
47#include <sys/mount.h> 49#include <sys/mount.h>
50#include <sys/stat.h>
48#include <sys/types.h> 51#include <sys/types.h>
49#include <sys/vfs.h> 52#include <sys/vfs.h>
50 53
@@ -163,13 +166,49 @@ int open_obj_pinned_any(char *path, enum bpf_obj_type exp_type)
163 return fd; 166 return fd;
164} 167}
165 168
166int do_pin_any(int argc, char **argv, int (*get_fd_by_id)(__u32)) 169int do_pin_fd(int fd, const char *name)
167{ 170{
168 char err_str[ERR_MAX_LEN]; 171 char err_str[ERR_MAX_LEN];
169 unsigned int id;
170 char *endptr;
171 char *file; 172 char *file;
172 char *dir; 173 char *dir;
174 int err = 0;
175
176 err = bpf_obj_pin(fd, name);
177 if (!err)
178 goto out;
179
180 file = malloc(strlen(name) + 1);
181 strcpy(file, name);
182 dir = dirname(file);
183
184 if (errno != EPERM || is_bpffs(dir)) {
185 p_err("can't pin the object (%s): %s", name, strerror(errno));
186 goto out_free;
187 }
188
189 /* Attempt to mount bpffs, then retry pinning. */
190 err = mnt_bpffs(dir, err_str, ERR_MAX_LEN);
191 if (!err) {
192 err = bpf_obj_pin(fd, name);
193 if (err)
194 p_err("can't pin the object (%s): %s", name,
195 strerror(errno));
196 } else {
197 err_str[ERR_MAX_LEN - 1] = '\0';
198 p_err("can't mount BPF file system to pin the object (%s): %s",
199 name, err_str);
200 }
201
202out_free:
203 free(file);
204out:
205 return err;
206}
207
208int do_pin_any(int argc, char **argv, int (*get_fd_by_id)(__u32))
209{
210 unsigned int id;
211 char *endptr;
173 int err; 212 int err;
174 int fd; 213 int fd;
175 214
@@ -195,35 +234,8 @@ int do_pin_any(int argc, char **argv, int (*get_fd_by_id)(__u32))
195 return -1; 234 return -1;
196 } 235 }
197 236
198 err = bpf_obj_pin(fd, *argv); 237 err = do_pin_fd(fd, *argv);
199 if (!err)
200 goto out_close;
201
202 file = malloc(strlen(*argv) + 1);
203 strcpy(file, *argv);
204 dir = dirname(file);
205
206 if (errno != EPERM || is_bpffs(dir)) {
207 p_err("can't pin the object (%s): %s", *argv, strerror(errno));
208 goto out_free;
209 }
210
211 /* Attempt to mount bpffs, then retry pinning. */
212 err = mnt_bpffs(dir, err_str, ERR_MAX_LEN);
213 if (!err) {
214 err = bpf_obj_pin(fd, *argv);
215 if (err)
216 p_err("can't pin the object (%s): %s", *argv,
217 strerror(errno));
218 } else {
219 err_str[ERR_MAX_LEN - 1] = '\0';
220 p_err("can't mount BPF file system to pin the object (%s): %s",
221 *argv, err_str);
222 }
223 238
224out_free:
225 free(file);
226out_close:
227 close(fd); 239 close(fd);
228 return err; 240 return err;
229} 241}
@@ -403,3 +415,124 @@ void delete_pinned_obj_table(struct pinned_obj_table *tab)
403 free(obj); 415 free(obj);
404 } 416 }
405} 417}
418
419static char *
420ifindex_to_name_ns(__u32 ifindex, __u32 ns_dev, __u32 ns_ino, char *buf)
421{
422 struct stat st;
423 int err;
424
425 err = stat("/proc/self/ns/net", &st);
426 if (err) {
427 p_err("Can't stat /proc/self: %s", strerror(errno));
428 return NULL;
429 }
430
431 if (st.st_dev != ns_dev || st.st_ino != ns_ino)
432 return NULL;
433
434 return if_indextoname(ifindex, buf);
435}
436
437static int read_sysfs_hex_int(char *path)
438{
439 char vendor_id_buf[8];
440 int len;
441 int fd;
442
443 fd = open(path, O_RDONLY);
444 if (fd < 0) {
445 p_err("Can't open %s: %s", path, strerror(errno));
446 return -1;
447 }
448
449 len = read(fd, vendor_id_buf, sizeof(vendor_id_buf));
450 close(fd);
451 if (len < 0) {
452 p_err("Can't read %s: %s", path, strerror(errno));
453 return -1;
454 }
455 if (len >= (int)sizeof(vendor_id_buf)) {
456 p_err("Value in %s too long", path);
457 return -1;
458 }
459
460 vendor_id_buf[len] = 0;
461
462 return strtol(vendor_id_buf, NULL, 0);
463}
464
465static int read_sysfs_netdev_hex_int(char *devname, const char *entry_name)
466{
467 char full_path[64];
468
469 snprintf(full_path, sizeof(full_path), "/sys/class/net/%s/device/%s",
470 devname, entry_name);
471
472 return read_sysfs_hex_int(full_path);
473}
474
475const char *ifindex_to_bfd_name_ns(__u32 ifindex, __u64 ns_dev, __u64 ns_ino)
476{
477 char devname[IF_NAMESIZE];
478 int vendor_id;
479 int device_id;
480
481 if (!ifindex_to_name_ns(ifindex, ns_dev, ns_ino, devname)) {
482 p_err("Can't get net device name for ifindex %d: %s", ifindex,
483 strerror(errno));
484 return NULL;
485 }
486
487 vendor_id = read_sysfs_netdev_hex_int(devname, "vendor");
488 if (vendor_id < 0) {
489 p_err("Can't get device vendor id for %s", devname);
490 return NULL;
491 }
492
493 switch (vendor_id) {
494 case 0x19ee:
495 device_id = read_sysfs_netdev_hex_int(devname, "device");
496 if (device_id != 0x4000 &&
497 device_id != 0x6000 &&
498 device_id != 0x6003)
499 p_info("Unknown NFP device ID, assuming it is NFP-6xxx arch");
500 return "NFP-6xxx";
501 default:
502 p_err("Can't get bfd arch name for device vendor id 0x%04x",
503 vendor_id);
504 return NULL;
505 }
506}
507
508void print_dev_plain(__u32 ifindex, __u64 ns_dev, __u64 ns_inode)
509{
510 char name[IF_NAMESIZE];
511
512 if (!ifindex)
513 return;
514
515 printf(" dev ");
516 if (ifindex_to_name_ns(ifindex, ns_dev, ns_inode, name))
517 printf("%s", name);
518 else
519 printf("ifindex %u ns_dev %llu ns_ino %llu",
520 ifindex, ns_dev, ns_inode);
521}
522
523void print_dev_json(__u32 ifindex, __u64 ns_dev, __u64 ns_inode)
524{
525 char name[IF_NAMESIZE];
526
527 if (!ifindex)
528 return;
529
530 jsonw_name(json_wtr, "dev");
531 jsonw_start_object(json_wtr);
532 jsonw_uint_field(json_wtr, "ifindex", ifindex);
533 jsonw_uint_field(json_wtr, "ns_dev", ns_dev);
534 jsonw_uint_field(json_wtr, "ns_inode", ns_inode);
535 if (ifindex_to_name_ns(ifindex, ns_dev, ns_inode, name))
536 jsonw_string_field(json_wtr, "ifname", name);
537 jsonw_end_object(json_wtr);
538}
diff --git a/tools/bpf/bpftool/jit_disasm.c b/tools/bpf/bpftool/jit_disasm.c
index 1551d3918d4c..87439320ef70 100644
--- a/tools/bpf/bpftool/jit_disasm.c
+++ b/tools/bpf/bpftool/jit_disasm.c
@@ -76,7 +76,8 @@ static int fprintf_json(void *out, const char *fmt, ...)
76 return 0; 76 return 0;
77} 77}
78 78
79void disasm_print_insn(unsigned char *image, ssize_t len, int opcodes) 79void disasm_print_insn(unsigned char *image, ssize_t len, int opcodes,
80 const char *arch)
80{ 81{
81 disassembler_ftype disassemble; 82 disassembler_ftype disassemble;
82 struct disassemble_info info; 83 struct disassemble_info info;
@@ -100,6 +101,19 @@ void disasm_print_insn(unsigned char *image, ssize_t len, int opcodes)
100 else 101 else
101 init_disassemble_info(&info, stdout, 102 init_disassemble_info(&info, stdout,
102 (fprintf_ftype) fprintf); 103 (fprintf_ftype) fprintf);
104
105 /* Update architecture info for offload. */
106 if (arch) {
107 const bfd_arch_info_type *inf = bfd_scan_arch(arch);
108
109 if (inf) {
110 bfdf->arch_info = inf;
111 } else {
112 p_err("No libfd support for %s", arch);
113 return;
114 }
115 }
116
103 info.arch = bfd_get_arch(bfdf); 117 info.arch = bfd_get_arch(bfdf);
104 info.mach = bfd_get_mach(bfdf); 118 info.mach = bfd_get_mach(bfdf);
105 info.buffer = image; 119 info.buffer = image;
@@ -107,7 +121,14 @@ void disasm_print_insn(unsigned char *image, ssize_t len, int opcodes)
107 121
108 disassemble_init_for_target(&info); 122 disassemble_init_for_target(&info);
109 123
124#ifdef DISASM_FOUR_ARGS_SIGNATURE
125 disassemble = disassembler(info.arch,
126 bfd_big_endian(bfdf),
127 info.mach,
128 bfdf);
129#else
110 disassemble = disassembler(bfdf); 130 disassemble = disassembler(bfdf);
131#endif
111 assert(disassemble); 132 assert(disassemble);
112 133
113 if (json_output) 134 if (json_output)
diff --git a/tools/bpf/bpftool/main.c b/tools/bpf/bpftool/main.c
index d294bc8168be..3a0396d87c42 100644
--- a/tools/bpf/bpftool/main.c
+++ b/tools/bpf/bpftool/main.c
@@ -38,7 +38,6 @@
38#include <errno.h> 38#include <errno.h>
39#include <getopt.h> 39#include <getopt.h>
40#include <linux/bpf.h> 40#include <linux/bpf.h>
41#include <linux/version.h>
42#include <stdio.h> 41#include <stdio.h>
43#include <stdlib.h> 42#include <stdlib.h>
44#include <string.h> 43#include <string.h>
@@ -85,7 +84,7 @@ static int do_help(int argc, char **argv)
85 " %s batch file FILE\n" 84 " %s batch file FILE\n"
86 " %s version\n" 85 " %s version\n"
87 "\n" 86 "\n"
88 " OBJECT := { prog | map }\n" 87 " OBJECT := { prog | map | cgroup }\n"
89 " " HELP_SPEC_OPTIONS "\n" 88 " " HELP_SPEC_OPTIONS "\n"
90 "", 89 "",
91 bin_name, bin_name, bin_name); 90 bin_name, bin_name, bin_name);
@@ -95,21 +94,13 @@ static int do_help(int argc, char **argv)
95 94
96static int do_version(int argc, char **argv) 95static int do_version(int argc, char **argv)
97{ 96{
98 unsigned int version[3];
99
100 version[0] = LINUX_VERSION_CODE >> 16;
101 version[1] = LINUX_VERSION_CODE >> 8 & 0xf;
102 version[2] = LINUX_VERSION_CODE & 0xf;
103
104 if (json_output) { 97 if (json_output) {
105 jsonw_start_object(json_wtr); 98 jsonw_start_object(json_wtr);
106 jsonw_name(json_wtr, "version"); 99 jsonw_name(json_wtr, "version");
107 jsonw_printf(json_wtr, "\"%u.%u.%u\"", 100 jsonw_printf(json_wtr, "\"%s\"", BPFTOOL_VERSION);
108 version[0], version[1], version[2]);
109 jsonw_end_object(json_wtr); 101 jsonw_end_object(json_wtr);
110 } else { 102 } else {
111 printf("%s v%u.%u.%u\n", bin_name, 103 printf("%s v%s\n", bin_name, BPFTOOL_VERSION);
112 version[0], version[1], version[2]);
113 } 104 }
114 return 0; 105 return 0;
115} 106}
@@ -173,6 +164,7 @@ static const struct cmd cmds[] = {
173 { "batch", do_batch }, 164 { "batch", do_batch },
174 { "prog", do_prog }, 165 { "prog", do_prog },
175 { "map", do_map }, 166 { "map", do_map },
167 { "cgroup", do_cgroup },
176 { "version", do_version }, 168 { "version", do_version },
177 { 0 } 169 { 0 }
178}; 170};
diff --git a/tools/bpf/bpftool/main.h b/tools/bpf/bpftool/main.h
index bff330b49791..b8e9584d6246 100644
--- a/tools/bpf/bpftool/main.h
+++ b/tools/bpf/bpftool/main.h
@@ -96,6 +96,8 @@ struct pinned_obj {
96int build_pinned_obj_table(struct pinned_obj_table *table, 96int build_pinned_obj_table(struct pinned_obj_table *table,
97 enum bpf_obj_type type); 97 enum bpf_obj_type type);
98void delete_pinned_obj_table(struct pinned_obj_table *tab); 98void delete_pinned_obj_table(struct pinned_obj_table *tab);
99void print_dev_plain(__u32 ifindex, __u64 ns_dev, __u64 ns_inode);
100void print_dev_json(__u32 ifindex, __u64 ns_dev, __u64 ns_inode);
99 101
100struct cmd { 102struct cmd {
101 const char *cmd; 103 const char *cmd;
@@ -111,13 +113,18 @@ char *get_fdinfo(int fd, const char *key);
111int open_obj_pinned(char *path); 113int open_obj_pinned(char *path);
112int open_obj_pinned_any(char *path, enum bpf_obj_type exp_type); 114int open_obj_pinned_any(char *path, enum bpf_obj_type exp_type);
113int do_pin_any(int argc, char **argv, int (*get_fd_by_id)(__u32)); 115int do_pin_any(int argc, char **argv, int (*get_fd_by_id)(__u32));
116int do_pin_fd(int fd, const char *name);
114 117
115int do_prog(int argc, char **arg); 118int do_prog(int argc, char **arg);
116int do_map(int argc, char **arg); 119int do_map(int argc, char **arg);
120int do_cgroup(int argc, char **arg);
117 121
118int prog_parse_fd(int *argc, char ***argv); 122int prog_parse_fd(int *argc, char ***argv);
119 123
120void disasm_print_insn(unsigned char *image, ssize_t len, int opcodes); 124void disasm_print_insn(unsigned char *image, ssize_t len, int opcodes,
125 const char *arch);
121void print_hex_data_json(uint8_t *data, size_t len); 126void print_hex_data_json(uint8_t *data, size_t len);
122 127
128const char *ifindex_to_bfd_name_ns(__u32 ifindex, __u64 ns_dev, __u64 ns_ino);
129
123#endif 130#endif
diff --git a/tools/bpf/bpftool/map.c b/tools/bpf/bpftool/map.c
index a8c3a33dd185..f95fa67bb498 100644
--- a/tools/bpf/bpftool/map.c
+++ b/tools/bpf/bpftool/map.c
@@ -66,6 +66,7 @@ static const char * const map_type_name[] = {
66 [BPF_MAP_TYPE_HASH_OF_MAPS] = "hash_of_maps", 66 [BPF_MAP_TYPE_HASH_OF_MAPS] = "hash_of_maps",
67 [BPF_MAP_TYPE_DEVMAP] = "devmap", 67 [BPF_MAP_TYPE_DEVMAP] = "devmap",
68 [BPF_MAP_TYPE_SOCKMAP] = "sockmap", 68 [BPF_MAP_TYPE_SOCKMAP] = "sockmap",
69 [BPF_MAP_TYPE_CPUMAP] = "cpumap",
69}; 70};
70 71
71static unsigned int get_possible_cpus(void) 72static unsigned int get_possible_cpus(void)
@@ -428,6 +429,9 @@ static int show_map_close_json(int fd, struct bpf_map_info *info)
428 429
429 jsonw_name(json_wtr, "flags"); 430 jsonw_name(json_wtr, "flags");
430 jsonw_printf(json_wtr, "%#x", info->map_flags); 431 jsonw_printf(json_wtr, "%#x", info->map_flags);
432
433 print_dev_json(info->ifindex, info->netns_dev, info->netns_ino);
434
431 jsonw_uint_field(json_wtr, "bytes_key", info->key_size); 435 jsonw_uint_field(json_wtr, "bytes_key", info->key_size);
432 jsonw_uint_field(json_wtr, "bytes_value", info->value_size); 436 jsonw_uint_field(json_wtr, "bytes_value", info->value_size);
433 jsonw_uint_field(json_wtr, "max_entries", info->max_entries); 437 jsonw_uint_field(json_wtr, "max_entries", info->max_entries);
@@ -469,7 +473,9 @@ static int show_map_close_plain(int fd, struct bpf_map_info *info)
469 if (*info->name) 473 if (*info->name)
470 printf("name %s ", info->name); 474 printf("name %s ", info->name);
471 475
472 printf("flags 0x%x\n", info->map_flags); 476 printf("flags 0x%x", info->map_flags);
477 print_dev_plain(info->ifindex, info->netns_dev, info->netns_ino);
478 printf("\n");
473 printf("\tkey %uB value %uB max_entries %u", 479 printf("\tkey %uB value %uB max_entries %u",
474 info->key_size, info->value_size, info->max_entries); 480 info->key_size, info->value_size, info->max_entries);
475 481
@@ -861,7 +867,7 @@ static int do_help(int argc, char **argv)
861 } 867 }
862 868
863 fprintf(stderr, 869 fprintf(stderr,
864 "Usage: %s %s show [MAP]\n" 870 "Usage: %s %s { show | list } [MAP]\n"
865 " %s %s dump MAP\n" 871 " %s %s dump MAP\n"
866 " %s %s update MAP key BYTES value VALUE [UPDATE_FLAGS]\n" 872 " %s %s update MAP key BYTES value VALUE [UPDATE_FLAGS]\n"
867 " %s %s lookup MAP key BYTES\n" 873 " %s %s lookup MAP key BYTES\n"
@@ -885,6 +891,7 @@ static int do_help(int argc, char **argv)
885 891
886static const struct cmd cmds[] = { 892static const struct cmd cmds[] = {
887 { "show", do_show }, 893 { "show", do_show },
894 { "list", do_show },
888 { "help", do_help }, 895 { "help", do_help },
889 { "dump", do_dump }, 896 { "dump", do_dump },
890 { "update", do_update }, 897 { "update", do_update },
diff --git a/tools/bpf/bpftool/prog.c b/tools/bpf/bpftool/prog.c
index dded77345bfb..e8e2baaf93c2 100644
--- a/tools/bpf/bpftool/prog.c
+++ b/tools/bpf/bpftool/prog.c
@@ -45,6 +45,7 @@
45#include <sys/stat.h> 45#include <sys/stat.h>
46 46
47#include <bpf.h> 47#include <bpf.h>
48#include <libbpf.h>
48 49
49#include "main.h" 50#include "main.h"
50#include "disasm.h" 51#include "disasm.h"
@@ -65,6 +66,7 @@ static const char * const prog_type_name[] = {
65 [BPF_PROG_TYPE_LWT_XMIT] = "lwt_xmit", 66 [BPF_PROG_TYPE_LWT_XMIT] = "lwt_xmit",
66 [BPF_PROG_TYPE_SOCK_OPS] = "sock_ops", 67 [BPF_PROG_TYPE_SOCK_OPS] = "sock_ops",
67 [BPF_PROG_TYPE_SK_SKB] = "sk_skb", 68 [BPF_PROG_TYPE_SK_SKB] = "sk_skb",
69 [BPF_PROG_TYPE_CGROUP_DEVICE] = "cgroup_device",
68}; 70};
69 71
70static void print_boot_time(__u64 nsecs, char *buf, unsigned int size) 72static void print_boot_time(__u64 nsecs, char *buf, unsigned int size)
@@ -229,6 +231,8 @@ static void print_prog_json(struct bpf_prog_info *info, int fd)
229 info->tag[0], info->tag[1], info->tag[2], info->tag[3], 231 info->tag[0], info->tag[1], info->tag[2], info->tag[3],
230 info->tag[4], info->tag[5], info->tag[6], info->tag[7]); 232 info->tag[4], info->tag[5], info->tag[6], info->tag[7]);
231 233
234 print_dev_json(info->ifindex, info->netns_dev, info->netns_ino);
235
232 if (info->load_time) { 236 if (info->load_time) {
233 char buf[32]; 237 char buf[32];
234 238
@@ -286,6 +290,7 @@ static void print_prog_plain(struct bpf_prog_info *info, int fd)
286 290
287 printf("tag "); 291 printf("tag ");
288 fprint_hex(stdout, info->tag, BPF_TAG_SIZE, ""); 292 fprint_hex(stdout, info->tag, BPF_TAG_SIZE, "");
293 print_dev_plain(info->ifindex, info->netns_dev, info->netns_ino);
289 printf("\n"); 294 printf("\n");
290 295
291 if (info->load_time) { 296 if (info->load_time) {
@@ -402,6 +407,88 @@ static int do_show(int argc, char **argv)
402 return err; 407 return err;
403} 408}
404 409
410#define SYM_MAX_NAME 256
411
412struct kernel_sym {
413 unsigned long address;
414 char name[SYM_MAX_NAME];
415};
416
417struct dump_data {
418 unsigned long address_call_base;
419 struct kernel_sym *sym_mapping;
420 __u32 sym_count;
421 char scratch_buff[SYM_MAX_NAME];
422};
423
424static int kernel_syms_cmp(const void *sym_a, const void *sym_b)
425{
426 return ((struct kernel_sym *)sym_a)->address -
427 ((struct kernel_sym *)sym_b)->address;
428}
429
430static void kernel_syms_load(struct dump_data *dd)
431{
432 struct kernel_sym *sym;
433 char buff[256];
434 void *tmp, *address;
435 FILE *fp;
436
437 fp = fopen("/proc/kallsyms", "r");
438 if (!fp)
439 return;
440
441 while (!feof(fp)) {
442 if (!fgets(buff, sizeof(buff), fp))
443 break;
444 tmp = realloc(dd->sym_mapping,
445 (dd->sym_count + 1) *
446 sizeof(*dd->sym_mapping));
447 if (!tmp) {
448out:
449 free(dd->sym_mapping);
450 dd->sym_mapping = NULL;
451 fclose(fp);
452 return;
453 }
454 dd->sym_mapping = tmp;
455 sym = &dd->sym_mapping[dd->sym_count];
456 if (sscanf(buff, "%p %*c %s", &address, sym->name) != 2)
457 continue;
458 sym->address = (unsigned long)address;
459 if (!strcmp(sym->name, "__bpf_call_base")) {
460 dd->address_call_base = sym->address;
461 /* sysctl kernel.kptr_restrict was set */
462 if (!sym->address)
463 goto out;
464 }
465 if (sym->address)
466 dd->sym_count++;
467 }
468
469 fclose(fp);
470
471 qsort(dd->sym_mapping, dd->sym_count,
472 sizeof(*dd->sym_mapping), kernel_syms_cmp);
473}
474
475static void kernel_syms_destroy(struct dump_data *dd)
476{
477 free(dd->sym_mapping);
478}
479
480static struct kernel_sym *kernel_syms_search(struct dump_data *dd,
481 unsigned long key)
482{
483 struct kernel_sym sym = {
484 .address = key,
485 };
486
487 return dd->sym_mapping ?
488 bsearch(&sym, dd->sym_mapping, dd->sym_count,
489 sizeof(*dd->sym_mapping), kernel_syms_cmp) : NULL;
490}
491
405static void print_insn(struct bpf_verifier_env *env, const char *fmt, ...) 492static void print_insn(struct bpf_verifier_env *env, const char *fmt, ...)
406{ 493{
407 va_list args; 494 va_list args;
@@ -411,8 +498,71 @@ static void print_insn(struct bpf_verifier_env *env, const char *fmt, ...)
411 va_end(args); 498 va_end(args);
412} 499}
413 500
414static void dump_xlated_plain(void *buf, unsigned int len, bool opcodes) 501static const char *print_call_pcrel(struct dump_data *dd,
502 struct kernel_sym *sym,
503 unsigned long address,
504 const struct bpf_insn *insn)
505{
506 if (sym)
507 snprintf(dd->scratch_buff, sizeof(dd->scratch_buff),
508 "%+d#%s", insn->off, sym->name);
509 else
510 snprintf(dd->scratch_buff, sizeof(dd->scratch_buff),
511 "%+d#0x%lx", insn->off, address);
512 return dd->scratch_buff;
513}
514
515static const char *print_call_helper(struct dump_data *dd,
516 struct kernel_sym *sym,
517 unsigned long address)
518{
519 if (sym)
520 snprintf(dd->scratch_buff, sizeof(dd->scratch_buff),
521 "%s", sym->name);
522 else
523 snprintf(dd->scratch_buff, sizeof(dd->scratch_buff),
524 "0x%lx", address);
525 return dd->scratch_buff;
526}
527
528static const char *print_call(void *private_data,
529 const struct bpf_insn *insn)
530{
531 struct dump_data *dd = private_data;
532 unsigned long address = dd->address_call_base + insn->imm;
533 struct kernel_sym *sym;
534
535 sym = kernel_syms_search(dd, address);
536 if (insn->src_reg == BPF_PSEUDO_CALL)
537 return print_call_pcrel(dd, sym, address, insn);
538 else
539 return print_call_helper(dd, sym, address);
540}
541
542static const char *print_imm(void *private_data,
543 const struct bpf_insn *insn,
544 __u64 full_imm)
545{
546 struct dump_data *dd = private_data;
547
548 if (insn->src_reg == BPF_PSEUDO_MAP_FD)
549 snprintf(dd->scratch_buff, sizeof(dd->scratch_buff),
550 "map[id:%u]", insn->imm);
551 else
552 snprintf(dd->scratch_buff, sizeof(dd->scratch_buff),
553 "0x%llx", (unsigned long long)full_imm);
554 return dd->scratch_buff;
555}
556
557static void dump_xlated_plain(struct dump_data *dd, void *buf,
558 unsigned int len, bool opcodes)
415{ 559{
560 const struct bpf_insn_cbs cbs = {
561 .cb_print = print_insn,
562 .cb_call = print_call,
563 .cb_imm = print_imm,
564 .private_data = dd,
565 };
416 struct bpf_insn *insn = buf; 566 struct bpf_insn *insn = buf;
417 bool double_insn = false; 567 bool double_insn = false;
418 unsigned int i; 568 unsigned int i;
@@ -426,7 +576,7 @@ static void dump_xlated_plain(void *buf, unsigned int len, bool opcodes)
426 double_insn = insn[i].code == (BPF_LD | BPF_IMM | BPF_DW); 576 double_insn = insn[i].code == (BPF_LD | BPF_IMM | BPF_DW);
427 577
428 printf("% 4d: ", i); 578 printf("% 4d: ", i);
429 print_bpf_insn(print_insn, NULL, insn + i, true); 579 print_bpf_insn(&cbs, NULL, insn + i, true);
430 580
431 if (opcodes) { 581 if (opcodes) {
432 printf(" "); 582 printf(" ");
@@ -455,8 +605,15 @@ static void print_insn_json(struct bpf_verifier_env *env, const char *fmt, ...)
455 va_end(args); 605 va_end(args);
456} 606}
457 607
458static void dump_xlated_json(void *buf, unsigned int len, bool opcodes) 608static void dump_xlated_json(struct dump_data *dd, void *buf,
609 unsigned int len, bool opcodes)
459{ 610{
611 const struct bpf_insn_cbs cbs = {
612 .cb_print = print_insn_json,
613 .cb_call = print_call,
614 .cb_imm = print_imm,
615 .private_data = dd,
616 };
460 struct bpf_insn *insn = buf; 617 struct bpf_insn *insn = buf;
461 bool double_insn = false; 618 bool double_insn = false;
462 unsigned int i; 619 unsigned int i;
@@ -471,7 +628,7 @@ static void dump_xlated_json(void *buf, unsigned int len, bool opcodes)
471 628
472 jsonw_start_object(json_wtr); 629 jsonw_start_object(json_wtr);
473 jsonw_name(json_wtr, "disasm"); 630 jsonw_name(json_wtr, "disasm");
474 print_bpf_insn(print_insn_json, NULL, insn + i, true); 631 print_bpf_insn(&cbs, NULL, insn + i, true);
475 632
476 if (opcodes) { 633 if (opcodes) {
477 jsonw_name(json_wtr, "opcodes"); 634 jsonw_name(json_wtr, "opcodes");
@@ -506,6 +663,7 @@ static void dump_xlated_json(void *buf, unsigned int len, bool opcodes)
506static int do_dump(int argc, char **argv) 663static int do_dump(int argc, char **argv)
507{ 664{
508 struct bpf_prog_info info = {}; 665 struct bpf_prog_info info = {};
666 struct dump_data dd = {};
509 __u32 len = sizeof(info); 667 __u32 len = sizeof(info);
510 unsigned int buf_size; 668 unsigned int buf_size;
511 char *filepath = NULL; 669 char *filepath = NULL;
@@ -593,6 +751,14 @@ static int do_dump(int argc, char **argv)
593 goto err_free; 751 goto err_free;
594 } 752 }
595 753
754 if ((member_len == &info.jited_prog_len &&
755 info.jited_prog_insns == 0) ||
756 (member_len == &info.xlated_prog_len &&
757 info.xlated_prog_insns == 0)) {
758 p_err("error retrieving insn dump: kernel.kptr_restrict set?");
759 goto err_free;
760 }
761
596 if (filepath) { 762 if (filepath) {
597 fd = open(filepath, O_WRONLY | O_CREAT | O_TRUNC, 0600); 763 fd = open(filepath, O_WRONLY | O_CREAT | O_TRUNC, 0600);
598 if (fd < 0) { 764 if (fd < 0) {
@@ -609,17 +775,29 @@ static int do_dump(int argc, char **argv)
609 goto err_free; 775 goto err_free;
610 } 776 }
611 } else { 777 } else {
612 if (member_len == &info.jited_prog_len) 778 if (member_len == &info.jited_prog_len) {
613 disasm_print_insn(buf, *member_len, opcodes); 779 const char *name = NULL;
614 else 780
781 if (info.ifindex) {
782 name = ifindex_to_bfd_name_ns(info.ifindex,
783 info.netns_dev,
784 info.netns_ino);
785 if (!name)
786 goto err_free;
787 }
788
789 disasm_print_insn(buf, *member_len, opcodes, name);
790 } else {
791 kernel_syms_load(&dd);
615 if (json_output) 792 if (json_output)
616 dump_xlated_json(buf, *member_len, opcodes); 793 dump_xlated_json(&dd, buf, *member_len, opcodes);
617 else 794 else
618 dump_xlated_plain(buf, *member_len, opcodes); 795 dump_xlated_plain(&dd, buf, *member_len, opcodes);
796 kernel_syms_destroy(&dd);
797 }
619 } 798 }
620 799
621 free(buf); 800 free(buf);
622
623 return 0; 801 return 0;
624 802
625err_free: 803err_free:
@@ -637,6 +815,30 @@ static int do_pin(int argc, char **argv)
637 return err; 815 return err;
638} 816}
639 817
818static int do_load(int argc, char **argv)
819{
820 struct bpf_object *obj;
821 int prog_fd;
822
823 if (argc != 2)
824 usage();
825
826 if (bpf_prog_load(argv[0], BPF_PROG_TYPE_UNSPEC, &obj, &prog_fd)) {
827 p_err("failed to load program");
828 return -1;
829 }
830
831 if (do_pin_fd(prog_fd, argv[1])) {
832 p_err("failed to pin program");
833 return -1;
834 }
835
836 if (json_output)
837 jsonw_null(json_wtr);
838
839 return 0;
840}
841
640static int do_help(int argc, char **argv) 842static int do_help(int argc, char **argv)
641{ 843{
642 if (json_output) { 844 if (json_output) {
@@ -645,26 +847,29 @@ static int do_help(int argc, char **argv)
645 } 847 }
646 848
647 fprintf(stderr, 849 fprintf(stderr,
648 "Usage: %s %s show [PROG]\n" 850 "Usage: %s %s { show | list } [PROG]\n"
649 " %s %s dump xlated PROG [{ file FILE | opcodes }]\n" 851 " %s %s dump xlated PROG [{ file FILE | opcodes }]\n"
650 " %s %s dump jited PROG [{ file FILE | opcodes }]\n" 852 " %s %s dump jited PROG [{ file FILE | opcodes }]\n"
651 " %s %s pin PROG FILE\n" 853 " %s %s pin PROG FILE\n"
854 " %s %s load OBJ FILE\n"
652 " %s %s help\n" 855 " %s %s help\n"
653 "\n" 856 "\n"
654 " " HELP_SPEC_PROGRAM "\n" 857 " " HELP_SPEC_PROGRAM "\n"
655 " " HELP_SPEC_OPTIONS "\n" 858 " " HELP_SPEC_OPTIONS "\n"
656 "", 859 "",
657 bin_name, argv[-2], bin_name, argv[-2], bin_name, argv[-2], 860 bin_name, argv[-2], bin_name, argv[-2], bin_name, argv[-2],
658 bin_name, argv[-2], bin_name, argv[-2]); 861 bin_name, argv[-2], bin_name, argv[-2], bin_name, argv[-2]);
659 862
660 return 0; 863 return 0;
661} 864}
662 865
663static const struct cmd cmds[] = { 866static const struct cmd cmds[] = {
664 { "show", do_show }, 867 { "show", do_show },
868 { "list", do_show },
665 { "help", do_help }, 869 { "help", do_help },
666 { "dump", do_dump }, 870 { "dump", do_dump },
667 { "pin", do_pin }, 871 { "pin", do_pin },
872 { "load", do_load },
668 { 0 } 873 { 0 }
669}; 874};
670 875
diff --git a/tools/build/feature/Makefile b/tools/build/feature/Makefile
index 59585fe20221..0a490cb15149 100644
--- a/tools/build/feature/Makefile
+++ b/tools/build/feature/Makefile
@@ -13,6 +13,7 @@ FILES= \
13 test-hello.bin \ 13 test-hello.bin \
14 test-libaudit.bin \ 14 test-libaudit.bin \
15 test-libbfd.bin \ 15 test-libbfd.bin \
16 test-disassembler-four-args.bin \
16 test-liberty.bin \ 17 test-liberty.bin \
17 test-liberty-z.bin \ 18 test-liberty-z.bin \
18 test-cplus-demangle.bin \ 19 test-cplus-demangle.bin \
@@ -197,6 +198,9 @@ $(OUTPUT)test-libpython-version.bin:
197$(OUTPUT)test-libbfd.bin: 198$(OUTPUT)test-libbfd.bin:
198 $(BUILD) -DPACKAGE='"perf"' -lbfd -lz -liberty -ldl 199 $(BUILD) -DPACKAGE='"perf"' -lbfd -lz -liberty -ldl
199 200
201$(OUTPUT)test-disassembler-four-args.bin:
202 $(BUILD) -DPACKAGE='"perf"' -lbfd -lopcodes
203
200$(OUTPUT)test-liberty.bin: 204$(OUTPUT)test-liberty.bin:
201 $(CC) $(CFLAGS) -Wall -Werror -o $@ test-libbfd.c -DPACKAGE='"perf"' $(LDFLAGS) -lbfd -ldl -liberty 205 $(CC) $(CFLAGS) -Wall -Werror -o $@ test-libbfd.c -DPACKAGE='"perf"' $(LDFLAGS) -lbfd -ldl -liberty
202 206
diff --git a/tools/build/feature/test-disassembler-four-args.c b/tools/build/feature/test-disassembler-four-args.c
new file mode 100644
index 000000000000..45ce65cfddf0
--- /dev/null
+++ b/tools/build/feature/test-disassembler-four-args.c
@@ -0,0 +1,15 @@
1// SPDX-License-Identifier: GPL-2.0
2#include <bfd.h>
3#include <dis-asm.h>
4
5int main(void)
6{
7 bfd *abfd = bfd_openr(NULL, NULL);
8
9 disassembler(bfd_get_arch(abfd),
10 bfd_big_endian(abfd),
11 bfd_get_mach(abfd),
12 abfd);
13
14 return 0;
15}
diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
index 4c223ab30293..db6bdc375126 100644
--- a/tools/include/uapi/linux/bpf.h
+++ b/tools/include/uapi/linux/bpf.h
@@ -17,7 +17,7 @@
17#define BPF_ALU64 0x07 /* alu mode in double word width */ 17#define BPF_ALU64 0x07 /* alu mode in double word width */
18 18
19/* ld/ldx fields */ 19/* ld/ldx fields */
20#define BPF_DW 0x18 /* double word */ 20#define BPF_DW 0x18 /* double word (64-bit) */
21#define BPF_XADD 0xc0 /* exclusive add */ 21#define BPF_XADD 0xc0 /* exclusive add */
22 22
23/* alu/jmp fields */ 23/* alu/jmp fields */
@@ -197,8 +197,14 @@ enum bpf_attach_type {
197 */ 197 */
198#define BPF_F_STRICT_ALIGNMENT (1U << 0) 198#define BPF_F_STRICT_ALIGNMENT (1U << 0)
199 199
200/* when bpf_ldimm64->src_reg == BPF_PSEUDO_MAP_FD, bpf_ldimm64->imm == fd */
200#define BPF_PSEUDO_MAP_FD 1 201#define BPF_PSEUDO_MAP_FD 1
201 202
203/* when bpf_call->src_reg == BPF_PSEUDO_CALL, bpf_call->imm == pc-relative
204 * offset to another bpf function
205 */
206#define BPF_PSEUDO_CALL 1
207
202/* flags for BPF_MAP_UPDATE_ELEM command */ 208/* flags for BPF_MAP_UPDATE_ELEM command */
203#define BPF_ANY 0 /* create new element or update existing */ 209#define BPF_ANY 0 /* create new element or update existing */
204#define BPF_NOEXIST 1 /* create new element if it didn't exist */ 210#define BPF_NOEXIST 1 /* create new element if it didn't exist */
@@ -239,6 +245,7 @@ union bpf_attr {
239 * BPF_F_NUMA_NODE is set). 245 * BPF_F_NUMA_NODE is set).
240 */ 246 */
241 char map_name[BPF_OBJ_NAME_LEN]; 247 char map_name[BPF_OBJ_NAME_LEN];
248 __u32 map_ifindex; /* ifindex of netdev to create on */
242 }; 249 };
243 250
244 struct { /* anonymous struct used by BPF_MAP_*_ELEM commands */ 251 struct { /* anonymous struct used by BPF_MAP_*_ELEM commands */
@@ -635,6 +642,14 @@ union bpf_attr {
635 * @optlen: length of optval in bytes 642 * @optlen: length of optval in bytes
636 * Return: 0 or negative error 643 * Return: 0 or negative error
637 * 644 *
645 * int bpf_sock_ops_cb_flags_set(bpf_sock_ops, flags)
646 * Set callback flags for sock_ops
647 * @bpf_sock_ops: pointer to bpf_sock_ops_kern struct
648 * @flags: flags value
649 * Return: 0 for no error
650 * -EINVAL if there is no full tcp socket
651 * bits in flags that are not supported by current kernel
652 *
638 * int bpf_skb_adjust_room(skb, len_diff, mode, flags) 653 * int bpf_skb_adjust_room(skb, len_diff, mode, flags)
639 * Grow or shrink room in sk_buff. 654 * Grow or shrink room in sk_buff.
640 * @skb: pointer to skb 655 * @skb: pointer to skb
@@ -677,6 +692,10 @@ union bpf_attr {
677 * @buf: buf to fill 692 * @buf: buf to fill
678 * @buf_size: size of the buf 693 * @buf_size: size of the buf
679 * Return : 0 on success or negative error code 694 * Return : 0 on success or negative error code
695 *
696 * int bpf_override_return(pt_regs, rc)
697 * @pt_regs: pointer to struct pt_regs
698 * @rc: the return value to set
680 */ 699 */
681#define __BPF_FUNC_MAPPER(FN) \ 700#define __BPF_FUNC_MAPPER(FN) \
682 FN(unspec), \ 701 FN(unspec), \
@@ -736,7 +755,9 @@ union bpf_attr {
736 FN(xdp_adjust_meta), \ 755 FN(xdp_adjust_meta), \
737 FN(perf_event_read_value), \ 756 FN(perf_event_read_value), \
738 FN(perf_prog_read_value), \ 757 FN(perf_prog_read_value), \
739 FN(getsockopt), 758 FN(getsockopt), \
759 FN(override_return), \
760 FN(sock_ops_cb_flags_set),
740 761
741/* integer value in 'imm' field of BPF_CALL instruction selects which helper 762/* integer value in 'imm' field of BPF_CALL instruction selects which helper
742 * function eBPF program intends to call 763 * function eBPF program intends to call
@@ -888,6 +909,9 @@ struct xdp_md {
888 __u32 data; 909 __u32 data;
889 __u32 data_end; 910 __u32 data_end;
890 __u32 data_meta; 911 __u32 data_meta;
912 /* Below access go through struct xdp_rxq_info */
913 __u32 ingress_ifindex; /* rxq->dev->ifindex */
914 __u32 rx_queue_index; /* rxq->queue_index */
891}; 915};
892 916
893enum sk_action { 917enum sk_action {
@@ -910,6 +934,9 @@ struct bpf_prog_info {
910 __u32 nr_map_ids; 934 __u32 nr_map_ids;
911 __aligned_u64 map_ids; 935 __aligned_u64 map_ids;
912 char name[BPF_OBJ_NAME_LEN]; 936 char name[BPF_OBJ_NAME_LEN];
937 __u32 ifindex;
938 __u64 netns_dev;
939 __u64 netns_ino;
913} __attribute__((aligned(8))); 940} __attribute__((aligned(8)));
914 941
915struct bpf_map_info { 942struct bpf_map_info {
@@ -920,6 +947,9 @@ struct bpf_map_info {
920 __u32 max_entries; 947 __u32 max_entries;
921 __u32 map_flags; 948 __u32 map_flags;
922 char name[BPF_OBJ_NAME_LEN]; 949 char name[BPF_OBJ_NAME_LEN];
950 __u32 ifindex;
951 __u64 netns_dev;
952 __u64 netns_ino;
923} __attribute__((aligned(8))); 953} __attribute__((aligned(8)));
924 954
925/* User bpf_sock_ops struct to access socket values and specify request ops 955/* User bpf_sock_ops struct to access socket values and specify request ops
@@ -931,8 +961,9 @@ struct bpf_map_info {
931struct bpf_sock_ops { 961struct bpf_sock_ops {
932 __u32 op; 962 __u32 op;
933 union { 963 union {
934 __u32 reply; 964 __u32 args[4]; /* Optionally passed to bpf program */
935 __u32 replylong[4]; 965 __u32 reply; /* Returned by bpf program */
966 __u32 replylong[4]; /* Optionally returned by bpf prog */
936 }; 967 };
937 __u32 family; 968 __u32 family;
938 __u32 remote_ip4; /* Stored in network byte order */ 969 __u32 remote_ip4; /* Stored in network byte order */
@@ -941,8 +972,45 @@ struct bpf_sock_ops {
941 __u32 local_ip6[4]; /* Stored in network byte order */ 972 __u32 local_ip6[4]; /* Stored in network byte order */
942 __u32 remote_port; /* Stored in network byte order */ 973 __u32 remote_port; /* Stored in network byte order */
943 __u32 local_port; /* stored in host byte order */ 974 __u32 local_port; /* stored in host byte order */
975 __u32 is_fullsock; /* Some TCP fields are only valid if
976 * there is a full socket. If not, the
977 * fields read as zero.
978 */
979 __u32 snd_cwnd;
980 __u32 srtt_us; /* Averaged RTT << 3 in usecs */
981 __u32 bpf_sock_ops_cb_flags; /* flags defined in uapi/linux/tcp.h */
982 __u32 state;
983 __u32 rtt_min;
984 __u32 snd_ssthresh;
985 __u32 rcv_nxt;
986 __u32 snd_nxt;
987 __u32 snd_una;
988 __u32 mss_cache;
989 __u32 ecn_flags;
990 __u32 rate_delivered;
991 __u32 rate_interval_us;
992 __u32 packets_out;
993 __u32 retrans_out;
994 __u32 total_retrans;
995 __u32 segs_in;
996 __u32 data_segs_in;
997 __u32 segs_out;
998 __u32 data_segs_out;
999 __u32 lost_out;
1000 __u32 sacked_out;
1001 __u32 sk_txhash;
1002 __u64 bytes_received;
1003 __u64 bytes_acked;
944}; 1004};
945 1005
1006/* Definitions for bpf_sock_ops_cb_flags */
1007#define BPF_SOCK_OPS_RTO_CB_FLAG (1<<0)
1008#define BPF_SOCK_OPS_RETRANS_CB_FLAG (1<<1)
1009#define BPF_SOCK_OPS_STATE_CB_FLAG (1<<2)
1010#define BPF_SOCK_OPS_ALL_CB_FLAGS 0x7 /* Mask of all currently
1011 * supported cb flags
1012 */
1013
946/* List of known BPF sock_ops operators. 1014/* List of known BPF sock_ops operators.
947 * New entries can only be added at the end 1015 * New entries can only be added at the end
948 */ 1016 */
@@ -976,6 +1044,43 @@ enum {
976 * a congestion threshold. RTTs above 1044 * a congestion threshold. RTTs above
977 * this indicate congestion 1045 * this indicate congestion
978 */ 1046 */
1047 BPF_SOCK_OPS_RTO_CB, /* Called when an RTO has triggered.
1048 * Arg1: value of icsk_retransmits
1049 * Arg2: value of icsk_rto
1050 * Arg3: whether RTO has expired
1051 */
1052 BPF_SOCK_OPS_RETRANS_CB, /* Called when skb is retransmitted.
1053 * Arg1: sequence number of 1st byte
1054 * Arg2: # segments
1055 * Arg3: return value of
1056 * tcp_transmit_skb (0 => success)
1057 */
1058 BPF_SOCK_OPS_STATE_CB, /* Called when TCP changes state.
1059 * Arg1: old_state
1060 * Arg2: new_state
1061 */
1062};
1063
1064/* List of TCP states. There is a build check in net/ipv4/tcp.c to detect
1065 * changes between the TCP and BPF versions. Ideally this should never happen.
1066 * If it does, we need to add code to convert them before calling
1067 * the BPF sock_ops function.
1068 */
1069enum {
1070 BPF_TCP_ESTABLISHED = 1,
1071 BPF_TCP_SYN_SENT,
1072 BPF_TCP_SYN_RECV,
1073 BPF_TCP_FIN_WAIT1,
1074 BPF_TCP_FIN_WAIT2,
1075 BPF_TCP_TIME_WAIT,
1076 BPF_TCP_CLOSE,
1077 BPF_TCP_CLOSE_WAIT,
1078 BPF_TCP_LAST_ACK,
1079 BPF_TCP_LISTEN,
1080 BPF_TCP_CLOSING, /* Now a valid state */
1081 BPF_TCP_NEW_SYN_RECV,
1082
1083 BPF_TCP_MAX_STATES /* Leave at the end! */
979}; 1084};
980 1085
981#define TCP_BPF_IW 1001 /* Set TCP initial congestion window */ 1086#define TCP_BPF_IW 1001 /* Set TCP initial congestion window */
@@ -995,7 +1100,8 @@ struct bpf_perf_event_value {
995#define BPF_DEVCG_DEV_CHAR (1ULL << 1) 1100#define BPF_DEVCG_DEV_CHAR (1ULL << 1)
996 1101
997struct bpf_cgroup_dev_ctx { 1102struct bpf_cgroup_dev_ctx {
998 __u32 access_type; /* (access << 16) | type */ 1103 /* access_type encoded as (BPF_DEVCG_ACC_* << 16) | BPF_DEVCG_DEV_* */
1104 __u32 access_type;
999 __u32 major; 1105 __u32 major;
1000 __u32 minor; 1106 __u32 minor;
1001}; 1107};
diff --git a/tools/include/uapi/linux/perf_event.h b/tools/include/uapi/linux/perf_event.h
index c77c9a2ebbbb..e0739a1aa4b2 100644
--- a/tools/include/uapi/linux/perf_event.h
+++ b/tools/include/uapi/linux/perf_event.h
@@ -418,6 +418,27 @@ struct perf_event_attr {
418 __u16 __reserved_2; /* align to __u64 */ 418 __u16 __reserved_2; /* align to __u64 */
419}; 419};
420 420
421/*
422 * Structure used by below PERF_EVENT_IOC_QUERY_BPF command
423 * to query bpf programs attached to the same perf tracepoint
424 * as the given perf event.
425 */
426struct perf_event_query_bpf {
427 /*
428 * The below ids array length
429 */
430 __u32 ids_len;
431 /*
432 * Set by the kernel to indicate the number of
433 * available programs
434 */
435 __u32 prog_cnt;
436 /*
437 * User provided buffer to store program ids
438 */
439 __u32 ids[0];
440};
441
421#define perf_flags(attr) (*(&(attr)->read_format + 1)) 442#define perf_flags(attr) (*(&(attr)->read_format + 1))
422 443
423/* 444/*
@@ -433,6 +454,7 @@ struct perf_event_attr {
433#define PERF_EVENT_IOC_ID _IOR('$', 7, __u64 *) 454#define PERF_EVENT_IOC_ID _IOR('$', 7, __u64 *)
434#define PERF_EVENT_IOC_SET_BPF _IOW('$', 8, __u32) 455#define PERF_EVENT_IOC_SET_BPF _IOW('$', 8, __u32)
435#define PERF_EVENT_IOC_PAUSE_OUTPUT _IOW('$', 9, __u32) 456#define PERF_EVENT_IOC_PAUSE_OUTPUT _IOW('$', 9, __u32)
457#define PERF_EVENT_IOC_QUERY_BPF _IOWR('$', 10, struct perf_event_query_bpf *)
436 458
437enum perf_event_ioc_flags { 459enum perf_event_ioc_flags {
438 PERF_IOC_FLAG_GROUP = 1U << 0, 460 PERF_IOC_FLAG_GROUP = 1U << 0,
diff --git a/tools/lib/bpf/Makefile b/tools/lib/bpf/Makefile
index 4555304dc18e..83714ca1f22b 100644
--- a/tools/lib/bpf/Makefile
+++ b/tools/lib/bpf/Makefile
@@ -93,7 +93,6 @@ export prefix libdir src obj
93# Shell quotes 93# Shell quotes
94libdir_SQ = $(subst ','\'',$(libdir)) 94libdir_SQ = $(subst ','\'',$(libdir))
95libdir_relative_SQ = $(subst ','\'',$(libdir_relative)) 95libdir_relative_SQ = $(subst ','\'',$(libdir_relative))
96plugin_dir_SQ = $(subst ','\'',$(plugin_dir))
97 96
98LIB_FILE = libbpf.a libbpf.so 97LIB_FILE = libbpf.a libbpf.so
99 98
@@ -150,7 +149,7 @@ CMD_TARGETS = $(LIB_FILE)
150 149
151TARGETS = $(CMD_TARGETS) 150TARGETS = $(CMD_TARGETS)
152 151
153all: fixdep $(VERSION_FILES) all_cmd 152all: fixdep all_cmd
154 153
155all_cmd: $(CMD_TARGETS) 154all_cmd: $(CMD_TARGETS)
156 155
@@ -169,21 +168,11 @@ $(OUTPUT)libbpf.so: $(BPF_IN)
169$(OUTPUT)libbpf.a: $(BPF_IN) 168$(OUTPUT)libbpf.a: $(BPF_IN)
170 $(QUIET_LINK)$(RM) $@; $(AR) rcs $@ $^ 169 $(QUIET_LINK)$(RM) $@; $(AR) rcs $@ $^
171 170
172define update_dir
173 (echo $1 > $@.tmp; \
174 if [ -r $@ ] && cmp -s $@ $@.tmp; then \
175 rm -f $@.tmp; \
176 else \
177 echo ' UPDATE $@'; \
178 mv -f $@.tmp $@; \
179 fi);
180endef
181
182define do_install 171define do_install
183 if [ ! -d '$(DESTDIR_SQ)$2' ]; then \ 172 if [ ! -d '$(DESTDIR_SQ)$2' ]; then \
184 $(INSTALL) -d -m 755 '$(DESTDIR_SQ)$2'; \ 173 $(INSTALL) -d -m 755 '$(DESTDIR_SQ)$2'; \
185 fi; \ 174 fi; \
186 $(INSTALL) $1 '$(DESTDIR_SQ)$2' 175 $(INSTALL) $1 $(if $3,-m $3,) '$(DESTDIR_SQ)$2'
187endef 176endef
188 177
189install_lib: all_cmd 178install_lib: all_cmd
@@ -192,7 +181,8 @@ install_lib: all_cmd
192 181
193install_headers: 182install_headers:
194 $(call QUIET_INSTALL, headers) \ 183 $(call QUIET_INSTALL, headers) \
195 $(call do_install,bpf.h,$(prefix)/include/bpf,644) 184 $(call do_install,bpf.h,$(prefix)/include/bpf,644); \
185 $(call do_install,libbpf.h,$(prefix)/include/bpf,644);
196 186
197install: install_lib 187install: install_lib
198 188
@@ -203,7 +193,7 @@ config-clean:
203 $(Q)$(MAKE) -C $(srctree)/tools/build/feature/ clean >/dev/null 193 $(Q)$(MAKE) -C $(srctree)/tools/build/feature/ clean >/dev/null
204 194
205clean: 195clean:
206 $(call QUIET_CLEAN, libbpf) $(RM) *.o *~ $(TARGETS) *.a *.so $(VERSION_FILES) .*.d .*.cmd \ 196 $(call QUIET_CLEAN, libbpf) $(RM) *.o *~ $(TARGETS) *.a *.so .*.d .*.cmd \
207 $(RM) LIBBPF-CFLAGS 197 $(RM) LIBBPF-CFLAGS
208 $(call QUIET_CLEAN, core-gen) $(RM) $(OUTPUT)FEATURE-DUMP.libbpf 198 $(call QUIET_CLEAN, core-gen) $(RM) $(OUTPUT)FEATURE-DUMP.libbpf
209 199
@@ -213,10 +203,10 @@ PHONY += force elfdep bpfdep
213force: 203force:
214 204
215elfdep: 205elfdep:
216 @if [ "$(feature-libelf)" != "1" ]; then echo "No libelf found"; exit -1 ; fi 206 @if [ "$(feature-libelf)" != "1" ]; then echo "No libelf found"; exit 1 ; fi
217 207
218bpfdep: 208bpfdep:
219 @if [ "$(feature-bpf)" != "1" ]; then echo "BPF API too old"; exit -1 ; fi 209 @if [ "$(feature-bpf)" != "1" ]; then echo "BPF API too old"; exit 1 ; fi
220 210
221# Declare the contents of the .PHONY variable as phony. We keep that 211# Declare the contents of the .PHONY variable as phony. We keep that
222# information in a variable so we can use it in if_changed and friends. 212# information in a variable so we can use it in if_changed and friends.
diff --git a/tools/lib/bpf/bpf.h b/tools/lib/bpf/bpf.h
index 6534889e2b2f..9f44c196931e 100644
--- a/tools/lib/bpf/bpf.h
+++ b/tools/lib/bpf/bpf.h
@@ -40,7 +40,7 @@ int bpf_create_map_in_map(enum bpf_map_type map_type, const char *name,
40 __u32 map_flags); 40 __u32 map_flags);
41 41
42/* Recommend log buffer size */ 42/* Recommend log buffer size */
43#define BPF_LOG_BUF_SIZE 65536 43#define BPF_LOG_BUF_SIZE (256 * 1024)
44int bpf_load_program_name(enum bpf_prog_type type, const char *name, 44int bpf_load_program_name(enum bpf_prog_type type, const char *name,
45 const struct bpf_insn *insns, 45 const struct bpf_insn *insns,
46 size_t insns_cnt, const char *license, 46 size_t insns_cnt, const char *license,
diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c
index 5aa45f89da93..30c776375118 100644
--- a/tools/lib/bpf/libbpf.c
+++ b/tools/lib/bpf/libbpf.c
@@ -174,12 +174,19 @@ struct bpf_program {
174 char *name; 174 char *name;
175 char *section_name; 175 char *section_name;
176 struct bpf_insn *insns; 176 struct bpf_insn *insns;
177 size_t insns_cnt; 177 size_t insns_cnt, main_prog_cnt;
178 enum bpf_prog_type type; 178 enum bpf_prog_type type;
179 179
180 struct { 180 struct reloc_desc {
181 enum {
182 RELO_LD64,
183 RELO_CALL,
184 } type;
181 int insn_idx; 185 int insn_idx;
182 int map_idx; 186 union {
187 int map_idx;
188 int text_off;
189 };
183 } *reloc_desc; 190 } *reloc_desc;
184 int nr_reloc; 191 int nr_reloc;
185 192
@@ -234,6 +241,7 @@ struct bpf_object {
234 } *reloc; 241 } *reloc;
235 int nr_reloc; 242 int nr_reloc;
236 int maps_shndx; 243 int maps_shndx;
244 int text_shndx;
237 } efile; 245 } efile;
238 /* 246 /*
239 * All loaded bpf_object is linked in a list, which is 247 * All loaded bpf_object is linked in a list, which is
@@ -375,9 +383,13 @@ bpf_object__init_prog_names(struct bpf_object *obj)
375 size_t pi, si; 383 size_t pi, si;
376 384
377 for (pi = 0; pi < obj->nr_programs; pi++) { 385 for (pi = 0; pi < obj->nr_programs; pi++) {
378 char *name = NULL; 386 const char *name = NULL;
379 387
380 prog = &obj->programs[pi]; 388 prog = &obj->programs[pi];
389 if (prog->idx == obj->efile.text_shndx) {
390 name = ".text";
391 goto skip_search;
392 }
381 393
382 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym) && !name; 394 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym) && !name;
383 si++) { 395 si++) {
@@ -387,6 +399,8 @@ bpf_object__init_prog_names(struct bpf_object *obj)
387 continue; 399 continue;
388 if (sym.st_shndx != prog->idx) 400 if (sym.st_shndx != prog->idx)
389 continue; 401 continue;
402 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL)
403 continue;
390 404
391 name = elf_strptr(obj->efile.elf, 405 name = elf_strptr(obj->efile.elf,
392 obj->efile.strtabidx, 406 obj->efile.strtabidx,
@@ -403,7 +417,7 @@ bpf_object__init_prog_names(struct bpf_object *obj)
403 prog->section_name); 417 prog->section_name);
404 return -EINVAL; 418 return -EINVAL;
405 } 419 }
406 420skip_search:
407 prog->name = strdup(name); 421 prog->name = strdup(name);
408 if (!prog->name) { 422 if (!prog->name) {
409 pr_warning("failed to allocate memory for prog sym %s\n", 423 pr_warning("failed to allocate memory for prog sym %s\n",
@@ -793,6 +807,8 @@ static int bpf_object__elf_collect(struct bpf_object *obj)
793 } else if ((sh.sh_type == SHT_PROGBITS) && 807 } else if ((sh.sh_type == SHT_PROGBITS) &&
794 (sh.sh_flags & SHF_EXECINSTR) && 808 (sh.sh_flags & SHF_EXECINSTR) &&
795 (data->d_size > 0)) { 809 (data->d_size > 0)) {
810 if (strcmp(name, ".text") == 0)
811 obj->efile.text_shndx = idx;
796 err = bpf_object__add_program(obj, data->d_buf, 812 err = bpf_object__add_program(obj, data->d_buf,
797 data->d_size, name, idx); 813 data->d_size, name, idx);
798 if (err) { 814 if (err) {
@@ -854,11 +870,14 @@ bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx)
854} 870}
855 871
856static int 872static int
857bpf_program__collect_reloc(struct bpf_program *prog, 873bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr,
858 size_t nr_maps, GElf_Shdr *shdr, 874 Elf_Data *data, struct bpf_object *obj)
859 Elf_Data *data, Elf_Data *symbols,
860 int maps_shndx, struct bpf_map *maps)
861{ 875{
876 Elf_Data *symbols = obj->efile.symbols;
877 int text_shndx = obj->efile.text_shndx;
878 int maps_shndx = obj->efile.maps_shndx;
879 struct bpf_map *maps = obj->maps;
880 size_t nr_maps = obj->nr_maps;
862 int i, nrels; 881 int i, nrels;
863 882
864 pr_debug("collecting relocating info for: '%s'\n", 883 pr_debug("collecting relocating info for: '%s'\n",
@@ -891,8 +910,11 @@ bpf_program__collect_reloc(struct bpf_program *prog,
891 GELF_R_SYM(rel.r_info)); 910 GELF_R_SYM(rel.r_info));
892 return -LIBBPF_ERRNO__FORMAT; 911 return -LIBBPF_ERRNO__FORMAT;
893 } 912 }
913 pr_debug("relo for %lld value %lld name %d\n",
914 (long long) (rel.r_info >> 32),
915 (long long) sym.st_value, sym.st_name);
894 916
895 if (sym.st_shndx != maps_shndx) { 917 if (sym.st_shndx != maps_shndx && sym.st_shndx != text_shndx) {
896 pr_warning("Program '%s' contains non-map related relo data pointing to section %u\n", 918 pr_warning("Program '%s' contains non-map related relo data pointing to section %u\n",
897 prog->section_name, sym.st_shndx); 919 prog->section_name, sym.st_shndx);
898 return -LIBBPF_ERRNO__RELOC; 920 return -LIBBPF_ERRNO__RELOC;
@@ -901,6 +923,17 @@ bpf_program__collect_reloc(struct bpf_program *prog,
901 insn_idx = rel.r_offset / sizeof(struct bpf_insn); 923 insn_idx = rel.r_offset / sizeof(struct bpf_insn);
902 pr_debug("relocation: insn_idx=%u\n", insn_idx); 924 pr_debug("relocation: insn_idx=%u\n", insn_idx);
903 925
926 if (insns[insn_idx].code == (BPF_JMP | BPF_CALL)) {
927 if (insns[insn_idx].src_reg != BPF_PSEUDO_CALL) {
928 pr_warning("incorrect bpf_call opcode\n");
929 return -LIBBPF_ERRNO__RELOC;
930 }
931 prog->reloc_desc[i].type = RELO_CALL;
932 prog->reloc_desc[i].insn_idx = insn_idx;
933 prog->reloc_desc[i].text_off = sym.st_value;
934 continue;
935 }
936
904 if (insns[insn_idx].code != (BPF_LD | BPF_IMM | BPF_DW)) { 937 if (insns[insn_idx].code != (BPF_LD | BPF_IMM | BPF_DW)) {
905 pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n", 938 pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n",
906 insn_idx, insns[insn_idx].code); 939 insn_idx, insns[insn_idx].code);
@@ -922,6 +955,7 @@ bpf_program__collect_reloc(struct bpf_program *prog,
922 return -LIBBPF_ERRNO__RELOC; 955 return -LIBBPF_ERRNO__RELOC;
923 } 956 }
924 957
958 prog->reloc_desc[i].type = RELO_LD64;
925 prog->reloc_desc[i].insn_idx = insn_idx; 959 prog->reloc_desc[i].insn_idx = insn_idx;
926 prog->reloc_desc[i].map_idx = map_idx; 960 prog->reloc_desc[i].map_idx = map_idx;
927 } 961 }
@@ -961,27 +995,76 @@ bpf_object__create_maps(struct bpf_object *obj)
961} 995}
962 996
963static int 997static int
998bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj,
999 struct reloc_desc *relo)
1000{
1001 struct bpf_insn *insn, *new_insn;
1002 struct bpf_program *text;
1003 size_t new_cnt;
1004
1005 if (relo->type != RELO_CALL)
1006 return -LIBBPF_ERRNO__RELOC;
1007
1008 if (prog->idx == obj->efile.text_shndx) {
1009 pr_warning("relo in .text insn %d into off %d\n",
1010 relo->insn_idx, relo->text_off);
1011 return -LIBBPF_ERRNO__RELOC;
1012 }
1013
1014 if (prog->main_prog_cnt == 0) {
1015 text = bpf_object__find_prog_by_idx(obj, obj->efile.text_shndx);
1016 if (!text) {
1017 pr_warning("no .text section found yet relo into text exist\n");
1018 return -LIBBPF_ERRNO__RELOC;
1019 }
1020 new_cnt = prog->insns_cnt + text->insns_cnt;
1021 new_insn = realloc(prog->insns, new_cnt * sizeof(*insn));
1022 if (!new_insn) {
1023 pr_warning("oom in prog realloc\n");
1024 return -ENOMEM;
1025 }
1026 memcpy(new_insn + prog->insns_cnt, text->insns,
1027 text->insns_cnt * sizeof(*insn));
1028 prog->insns = new_insn;
1029 prog->main_prog_cnt = prog->insns_cnt;
1030 prog->insns_cnt = new_cnt;
1031 }
1032 insn = &prog->insns[relo->insn_idx];
1033 insn->imm += prog->main_prog_cnt - relo->insn_idx;
1034 pr_debug("added %zd insn from %s to prog %s\n",
1035 text->insns_cnt, text->section_name, prog->section_name);
1036 return 0;
1037}
1038
1039static int
964bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj) 1040bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj)
965{ 1041{
966 int i; 1042 int i, err;
967 1043
968 if (!prog || !prog->reloc_desc) 1044 if (!prog || !prog->reloc_desc)
969 return 0; 1045 return 0;
970 1046
971 for (i = 0; i < prog->nr_reloc; i++) { 1047 for (i = 0; i < prog->nr_reloc; i++) {
972 int insn_idx, map_idx; 1048 if (prog->reloc_desc[i].type == RELO_LD64) {
973 struct bpf_insn *insns = prog->insns; 1049 struct bpf_insn *insns = prog->insns;
1050 int insn_idx, map_idx;
974 1051
975 insn_idx = prog->reloc_desc[i].insn_idx; 1052 insn_idx = prog->reloc_desc[i].insn_idx;
976 map_idx = prog->reloc_desc[i].map_idx; 1053 map_idx = prog->reloc_desc[i].map_idx;
977 1054
978 if (insn_idx >= (int)prog->insns_cnt) { 1055 if (insn_idx >= (int)prog->insns_cnt) {
979 pr_warning("relocation out of range: '%s'\n", 1056 pr_warning("relocation out of range: '%s'\n",
980 prog->section_name); 1057 prog->section_name);
981 return -LIBBPF_ERRNO__RELOC; 1058 return -LIBBPF_ERRNO__RELOC;
1059 }
1060 insns[insn_idx].src_reg = BPF_PSEUDO_MAP_FD;
1061 insns[insn_idx].imm = obj->maps[map_idx].fd;
1062 } else {
1063 err = bpf_program__reloc_text(prog, obj,
1064 &prog->reloc_desc[i]);
1065 if (err)
1066 return err;
982 } 1067 }
983 insns[insn_idx].src_reg = BPF_PSEUDO_MAP_FD;
984 insns[insn_idx].imm = obj->maps[map_idx].fd;
985 } 1068 }
986 1069
987 zfree(&prog->reloc_desc); 1070 zfree(&prog->reloc_desc);
@@ -1024,7 +1107,6 @@ static int bpf_object__collect_reloc(struct bpf_object *obj)
1024 Elf_Data *data = obj->efile.reloc[i].data; 1107 Elf_Data *data = obj->efile.reloc[i].data;
1025 int idx = shdr->sh_info; 1108 int idx = shdr->sh_info;
1026 struct bpf_program *prog; 1109 struct bpf_program *prog;
1027 size_t nr_maps = obj->nr_maps;
1028 1110
1029 if (shdr->sh_type != SHT_REL) { 1111 if (shdr->sh_type != SHT_REL) {
1030 pr_warning("internal error at %d\n", __LINE__); 1112 pr_warning("internal error at %d\n", __LINE__);
@@ -1038,11 +1120,9 @@ static int bpf_object__collect_reloc(struct bpf_object *obj)
1038 return -LIBBPF_ERRNO__RELOC; 1120 return -LIBBPF_ERRNO__RELOC;
1039 } 1121 }
1040 1122
1041 err = bpf_program__collect_reloc(prog, nr_maps, 1123 err = bpf_program__collect_reloc(prog,
1042 shdr, data, 1124 shdr, data,
1043 obj->efile.symbols, 1125 obj);
1044 obj->efile.maps_shndx,
1045 obj->maps);
1046 if (err) 1126 if (err)
1047 return err; 1127 return err;
1048 } 1128 }
@@ -1195,6 +1275,8 @@ bpf_object__load_progs(struct bpf_object *obj)
1195 int err; 1275 int err;
1196 1276
1197 for (i = 0; i < obj->nr_programs; i++) { 1277 for (i = 0; i < obj->nr_programs; i++) {
1278 if (obj->programs[i].idx == obj->efile.text_shndx)
1279 continue;
1198 err = bpf_program__load(&obj->programs[i], 1280 err = bpf_program__load(&obj->programs[i],
1199 obj->license, 1281 obj->license,
1200 obj->kern_version); 1282 obj->kern_version);
@@ -1721,6 +1803,45 @@ BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT);
1721BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP); 1803BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP);
1722BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT); 1804BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT);
1723 1805
1806#define BPF_PROG_SEC(string, type) { string, sizeof(string) - 1, type }
1807static const struct {
1808 const char *sec;
1809 size_t len;
1810 enum bpf_prog_type prog_type;
1811} section_names[] = {
1812 BPF_PROG_SEC("socket", BPF_PROG_TYPE_SOCKET_FILTER),
1813 BPF_PROG_SEC("kprobe/", BPF_PROG_TYPE_KPROBE),
1814 BPF_PROG_SEC("kretprobe/", BPF_PROG_TYPE_KPROBE),
1815 BPF_PROG_SEC("tracepoint/", BPF_PROG_TYPE_TRACEPOINT),
1816 BPF_PROG_SEC("xdp", BPF_PROG_TYPE_XDP),
1817 BPF_PROG_SEC("perf_event", BPF_PROG_TYPE_PERF_EVENT),
1818 BPF_PROG_SEC("cgroup/skb", BPF_PROG_TYPE_CGROUP_SKB),
1819 BPF_PROG_SEC("cgroup/sock", BPF_PROG_TYPE_CGROUP_SOCK),
1820 BPF_PROG_SEC("cgroup/dev", BPF_PROG_TYPE_CGROUP_DEVICE),
1821 BPF_PROG_SEC("sockops", BPF_PROG_TYPE_SOCK_OPS),
1822 BPF_PROG_SEC("sk_skb", BPF_PROG_TYPE_SK_SKB),
1823};
1824#undef BPF_PROG_SEC
1825
1826static enum bpf_prog_type bpf_program__guess_type(struct bpf_program *prog)
1827{
1828 int i;
1829
1830 if (!prog->section_name)
1831 goto err;
1832
1833 for (i = 0; i < ARRAY_SIZE(section_names); i++)
1834 if (strncmp(prog->section_name, section_names[i].sec,
1835 section_names[i].len) == 0)
1836 return section_names[i].prog_type;
1837
1838err:
1839 pr_warning("failed to guess program type based on section name %s\n",
1840 prog->section_name);
1841
1842 return BPF_PROG_TYPE_UNSPEC;
1843}
1844
1724int bpf_map__fd(struct bpf_map *map) 1845int bpf_map__fd(struct bpf_map *map)
1725{ 1846{
1726 return map ? map->fd : -EINVAL; 1847 return map ? map->fd : -EINVAL;
@@ -1818,7 +1939,7 @@ long libbpf_get_error(const void *ptr)
1818int bpf_prog_load(const char *file, enum bpf_prog_type type, 1939int bpf_prog_load(const char *file, enum bpf_prog_type type,
1819 struct bpf_object **pobj, int *prog_fd) 1940 struct bpf_object **pobj, int *prog_fd)
1820{ 1941{
1821 struct bpf_program *prog; 1942 struct bpf_program *prog, *first_prog = NULL;
1822 struct bpf_object *obj; 1943 struct bpf_object *obj;
1823 int err; 1944 int err;
1824 1945
@@ -1826,13 +1947,30 @@ int bpf_prog_load(const char *file, enum bpf_prog_type type,
1826 if (IS_ERR(obj)) 1947 if (IS_ERR(obj))
1827 return -ENOENT; 1948 return -ENOENT;
1828 1949
1829 prog = bpf_program__next(NULL, obj); 1950 bpf_object__for_each_program(prog, obj) {
1830 if (!prog) { 1951 /*
1952 * If type is not specified, try to guess it based on
1953 * section name.
1954 */
1955 if (type == BPF_PROG_TYPE_UNSPEC) {
1956 type = bpf_program__guess_type(prog);
1957 if (type == BPF_PROG_TYPE_UNSPEC) {
1958 bpf_object__close(obj);
1959 return -EINVAL;
1960 }
1961 }
1962
1963 bpf_program__set_type(prog, type);
1964 if (prog->idx != obj->efile.text_shndx && !first_prog)
1965 first_prog = prog;
1966 }
1967
1968 if (!first_prog) {
1969 pr_warning("object file doesn't contain bpf program\n");
1831 bpf_object__close(obj); 1970 bpf_object__close(obj);
1832 return -ENOENT; 1971 return -ENOENT;
1833 } 1972 }
1834 1973
1835 bpf_program__set_type(prog, type);
1836 err = bpf_object__load(obj); 1974 err = bpf_object__load(obj);
1837 if (err) { 1975 if (err) {
1838 bpf_object__close(obj); 1976 bpf_object__close(obj);
@@ -1840,6 +1978,6 @@ int bpf_prog_load(const char *file, enum bpf_prog_type type,
1840 } 1978 }
1841 1979
1842 *pobj = obj; 1980 *pobj = obj;
1843 *prog_fd = bpf_program__fd(prog); 1981 *prog_fd = bpf_program__fd(first_prog);
1844 return 0; 1982 return 0;
1845} 1983}
diff --git a/tools/scripts/Makefile.include b/tools/scripts/Makefile.include
index 3fab179b1aba..fcb3ed0be5f8 100644
--- a/tools/scripts/Makefile.include
+++ b/tools/scripts/Makefile.include
@@ -99,5 +99,6 @@ ifneq ($(silent),1)
99 99
100 QUIET_CLEAN = @printf ' CLEAN %s\n' $1; 100 QUIET_CLEAN = @printf ' CLEAN %s\n' $1;
101 QUIET_INSTALL = @printf ' INSTALL %s\n' $1; 101 QUIET_INSTALL = @printf ' INSTALL %s\n' $1;
102 QUIET_UNINST = @printf ' UNINST %s\n' $1;
102 endif 103 endif
103endif 104endif
diff --git a/tools/testing/selftests/bpf/.gitignore b/tools/testing/selftests/bpf/.gitignore
index 541d9d7fad5a..1e09d77f1948 100644
--- a/tools/testing/selftests/bpf/.gitignore
+++ b/tools/testing/selftests/bpf/.gitignore
@@ -3,3 +3,10 @@ test_maps
3test_lru_map 3test_lru_map
4test_lpm_map 4test_lpm_map
5test_tag 5test_tag
6FEATURE-DUMP.libbpf
7fixdep
8test_align
9test_dev_cgroup
10test_progs
11test_verifier_log
12feature
diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile
index 9316e648a880..bf05bc5e36e5 100644
--- a/tools/testing/selftests/bpf/Makefile
+++ b/tools/testing/selftests/bpf/Makefile
@@ -11,16 +11,19 @@ ifneq ($(wildcard $(GENHDR)),)
11endif 11endif
12 12
13CFLAGS += -Wall -O2 -I$(APIDIR) -I$(LIBDIR) -I$(GENDIR) $(GENFLAGS) -I../../../include 13CFLAGS += -Wall -O2 -I$(APIDIR) -I$(LIBDIR) -I$(GENDIR) $(GENFLAGS) -I../../../include
14LDLIBS += -lcap -lelf -lrt 14LDLIBS += -lcap -lelf -lrt -lpthread
15 15
16TEST_GEN_PROGS = test_verifier test_tag test_maps test_lru_map test_lpm_map test_progs \ 16TEST_GEN_PROGS = test_verifier test_tag test_maps test_lru_map test_lpm_map test_progs \
17 test_align test_verifier_log test_dev_cgroup 17 test_align test_verifier_log test_dev_cgroup test_tcpbpf_user
18 18
19TEST_GEN_FILES = test_pkt_access.o test_xdp.o test_l4lb.o test_tcp_estats.o test_obj_id.o \ 19TEST_GEN_FILES = test_pkt_access.o test_xdp.o test_l4lb.o test_tcp_estats.o test_obj_id.o \
20 test_pkt_md_access.o test_xdp_redirect.o test_xdp_meta.o sockmap_parse_prog.o \ 20 test_pkt_md_access.o test_xdp_redirect.o test_xdp_meta.o sockmap_parse_prog.o \
21 sockmap_verdict_prog.o dev_cgroup.o 21 sockmap_verdict_prog.o dev_cgroup.o sample_ret0.o test_tracepoint.o \
22 test_l4lb_noinline.o test_xdp_noinline.o test_stacktrace_map.o \
23 sample_map_ret0.o test_tcpbpf_kern.o
22 24
23TEST_PROGS := test_kmod.sh test_xdp_redirect.sh test_xdp_meta.sh 25TEST_PROGS := test_kmod.sh test_xdp_redirect.sh test_xdp_meta.sh \
26 test_offload.py
24 27
25include ../lib.mk 28include ../lib.mk
26 29
@@ -49,8 +52,13 @@ else
49 CPU ?= generic 52 CPU ?= generic
50endif 53endif
51 54
55CLANG_FLAGS = -I. -I./include/uapi -I../../../include/uapi \
56 -Wno-compare-distinct-pointer-types
57
58$(OUTPUT)/test_l4lb_noinline.o: CLANG_FLAGS += -fno-inline
59$(OUTPUT)/test_xdp_noinline.o: CLANG_FLAGS += -fno-inline
60
52%.o: %.c 61%.o: %.c
53 $(CLANG) -I. -I./include/uapi -I../../../include/uapi \ 62 $(CLANG) $(CLANG_FLAGS) \
54 -Wno-compare-distinct-pointer-types \
55 -O2 -target bpf -emit-llvm -c $< -o - | \ 63 -O2 -target bpf -emit-llvm -c $< -o - | \
56 $(LLC) -march=bpf -mcpu=$(CPU) -filetype=obj -o $@ 64 $(LLC) -march=bpf -mcpu=$(CPU) -filetype=obj -o $@
diff --git a/tools/testing/selftests/bpf/bpf_helpers.h b/tools/testing/selftests/bpf/bpf_helpers.h
index fd9a17fa8a8b..dde2c11d7771 100644
--- a/tools/testing/selftests/bpf/bpf_helpers.h
+++ b/tools/testing/selftests/bpf/bpf_helpers.h
@@ -71,6 +71,8 @@ static int (*bpf_setsockopt)(void *ctx, int level, int optname, void *optval,
71static int (*bpf_getsockopt)(void *ctx, int level, int optname, void *optval, 71static int (*bpf_getsockopt)(void *ctx, int level, int optname, void *optval,
72 int optlen) = 72 int optlen) =
73 (void *) BPF_FUNC_getsockopt; 73 (void *) BPF_FUNC_getsockopt;
74static int (*bpf_sock_ops_cb_flags_set)(void *ctx, int flags) =
75 (void *) BPF_FUNC_sock_ops_cb_flags_set;
74static int (*bpf_sk_redirect_map)(void *ctx, void *map, int key, int flags) = 76static int (*bpf_sk_redirect_map)(void *ctx, void *map, int key, int flags) =
75 (void *) BPF_FUNC_sk_redirect_map; 77 (void *) BPF_FUNC_sk_redirect_map;
76static int (*bpf_sock_map_update)(void *map, void *key, void *value, 78static int (*bpf_sock_map_update)(void *map, void *key, void *value,
@@ -82,7 +84,8 @@ static int (*bpf_perf_event_read_value)(void *map, unsigned long long flags,
82static int (*bpf_perf_prog_read_value)(void *ctx, void *buf, 84static int (*bpf_perf_prog_read_value)(void *ctx, void *buf,
83 unsigned int buf_size) = 85 unsigned int buf_size) =
84 (void *) BPF_FUNC_perf_prog_read_value; 86 (void *) BPF_FUNC_perf_prog_read_value;
85 87static int (*bpf_override_return)(void *ctx, unsigned long rc) =
88 (void *) BPF_FUNC_override_return;
86 89
87/* llvm builtin functions that eBPF C program may use to 90/* llvm builtin functions that eBPF C program may use to
88 * emit BPF_LD_ABS and BPF_LD_IND instructions 91 * emit BPF_LD_ABS and BPF_LD_IND instructions
diff --git a/tools/testing/selftests/bpf/config b/tools/testing/selftests/bpf/config
index 52d53ed08769..983dd25d49f4 100644
--- a/tools/testing/selftests/bpf/config
+++ b/tools/testing/selftests/bpf/config
@@ -3,3 +3,5 @@ CONFIG_BPF_SYSCALL=y
3CONFIG_NET_CLS_BPF=m 3CONFIG_NET_CLS_BPF=m
4CONFIG_BPF_EVENTS=y 4CONFIG_BPF_EVENTS=y
5CONFIG_TEST_BPF=m 5CONFIG_TEST_BPF=m
6CONFIG_CGROUP_BPF=y
7CONFIG_NETDEVSIM=m
diff --git a/tools/testing/selftests/bpf/sample_map_ret0.c b/tools/testing/selftests/bpf/sample_map_ret0.c
new file mode 100644
index 000000000000..0756303676ac
--- /dev/null
+++ b/tools/testing/selftests/bpf/sample_map_ret0.c
@@ -0,0 +1,34 @@
1/* SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) */
2#include <linux/bpf.h>
3#include "bpf_helpers.h"
4
5struct bpf_map_def SEC("maps") htab = {
6 .type = BPF_MAP_TYPE_HASH,
7 .key_size = sizeof(__u32),
8 .value_size = sizeof(long),
9 .max_entries = 2,
10};
11
12struct bpf_map_def SEC("maps") array = {
13 .type = BPF_MAP_TYPE_ARRAY,
14 .key_size = sizeof(__u32),
15 .value_size = sizeof(long),
16 .max_entries = 2,
17};
18
19/* Sample program which should always load for testing control paths. */
20SEC(".text") int func()
21{
22 __u64 key64 = 0;
23 __u32 key = 0;
24 long *value;
25
26 value = bpf_map_lookup_elem(&htab, &key);
27 if (!value)
28 return 1;
29 value = bpf_map_lookup_elem(&array, &key64);
30 if (!value)
31 return 1;
32
33 return 0;
34}
diff --git a/tools/testing/selftests/bpf/sample_ret0.c b/tools/testing/selftests/bpf/sample_ret0.c
new file mode 100644
index 000000000000..fec99750d6ea
--- /dev/null
+++ b/tools/testing/selftests/bpf/sample_ret0.c
@@ -0,0 +1,7 @@
1/* SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) */
2
3/* Sample program which should always load for testing control paths. */
4int func()
5{
6 return 0;
7}
diff --git a/tools/testing/selftests/bpf/tcp_client.py b/tools/testing/selftests/bpf/tcp_client.py
new file mode 100755
index 000000000000..481dccdf140c
--- /dev/null
+++ b/tools/testing/selftests/bpf/tcp_client.py
@@ -0,0 +1,51 @@
1#!/usr/bin/env python2
2#
3# SPDX-License-Identifier: GPL-2.0
4#
5
6import sys, os, os.path, getopt
7import socket, time
8import subprocess
9import select
10
11def read(sock, n):
12 buf = ''
13 while len(buf) < n:
14 rem = n - len(buf)
15 try: s = sock.recv(rem)
16 except (socket.error), e: return ''
17 buf += s
18 return buf
19
20def send(sock, s):
21 total = len(s)
22 count = 0
23 while count < total:
24 try: n = sock.send(s)
25 except (socket.error), e: n = 0
26 if n == 0:
27 return count;
28 count += n
29 return count
30
31
32serverPort = int(sys.argv[1])
33HostName = socket.gethostname()
34
35# create active socket
36sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
37try:
38 sock.connect((HostName, serverPort))
39except socket.error as e:
40 sys.exit(1)
41
42buf = ''
43n = 0
44while n < 1000:
45 buf += '+'
46 n += 1
47
48sock.settimeout(1);
49n = send(sock, buf)
50n = read(sock, 500)
51sys.exit(0)
diff --git a/tools/testing/selftests/bpf/tcp_server.py b/tools/testing/selftests/bpf/tcp_server.py
new file mode 100755
index 000000000000..bc454d7d0be2
--- /dev/null
+++ b/tools/testing/selftests/bpf/tcp_server.py
@@ -0,0 +1,83 @@
1#!/usr/bin/env python2
2#
3# SPDX-License-Identifier: GPL-2.0
4#
5
6import sys, os, os.path, getopt
7import socket, time
8import subprocess
9import select
10
11def read(sock, n):
12 buf = ''
13 while len(buf) < n:
14 rem = n - len(buf)
15 try: s = sock.recv(rem)
16 except (socket.error), e: return ''
17 buf += s
18 return buf
19
20def send(sock, s):
21 total = len(s)
22 count = 0
23 while count < total:
24 try: n = sock.send(s)
25 except (socket.error), e: n = 0
26 if n == 0:
27 return count;
28 count += n
29 return count
30
31
32SERVER_PORT = 12877
33MAX_PORTS = 2
34
35serverPort = SERVER_PORT
36serverSocket = None
37
38HostName = socket.gethostname()
39
40# create passive socket
41serverSocket = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
42host = socket.gethostname()
43
44try: serverSocket.bind((host, 0))
45except socket.error as msg:
46 print 'bind fails: ', msg
47
48sn = serverSocket.getsockname()
49serverPort = sn[1]
50
51cmdStr = ("./tcp_client.py %d &") % (serverPort)
52os.system(cmdStr)
53
54buf = ''
55n = 0
56while n < 500:
57 buf += '.'
58 n += 1
59
60serverSocket.listen(MAX_PORTS)
61readList = [serverSocket]
62
63while True:
64 readyRead, readyWrite, inError = \
65 select.select(readList, [], [], 2)
66
67 if len(readyRead) > 0:
68 waitCount = 0
69 for sock in readyRead:
70 if sock == serverSocket:
71 (clientSocket, address) = serverSocket.accept()
72 address = str(address[0])
73 readList.append(clientSocket)
74 else:
75 sock.settimeout(1);
76 s = read(sock, 1000)
77 n = send(sock, buf)
78 sock.close()
79 serverSocket.close()
80 sys.exit(0)
81 else:
82 print 'Select timeout!'
83 sys.exit(1)
diff --git a/tools/testing/selftests/bpf/test_align.c b/tools/testing/selftests/bpf/test_align.c
index 471bbbdb94db..ff8bd7e3e50c 100644
--- a/tools/testing/selftests/bpf/test_align.c
+++ b/tools/testing/selftests/bpf/test_align.c
@@ -64,11 +64,11 @@ static struct bpf_align_test tests[] = {
64 .matches = { 64 .matches = {
65 {1, "R1=ctx(id=0,off=0,imm=0)"}, 65 {1, "R1=ctx(id=0,off=0,imm=0)"},
66 {1, "R10=fp0"}, 66 {1, "R10=fp0"},
67 {1, "R3=inv2"}, 67 {1, "R3_w=inv2"},
68 {2, "R3=inv4"}, 68 {2, "R3_w=inv4"},
69 {3, "R3=inv8"}, 69 {3, "R3_w=inv8"},
70 {4, "R3=inv16"}, 70 {4, "R3_w=inv16"},
71 {5, "R3=inv32"}, 71 {5, "R3_w=inv32"},
72 }, 72 },
73 }, 73 },
74 { 74 {
@@ -92,17 +92,17 @@ static struct bpf_align_test tests[] = {
92 .matches = { 92 .matches = {
93 {1, "R1=ctx(id=0,off=0,imm=0)"}, 93 {1, "R1=ctx(id=0,off=0,imm=0)"},
94 {1, "R10=fp0"}, 94 {1, "R10=fp0"},
95 {1, "R3=inv1"}, 95 {1, "R3_w=inv1"},
96 {2, "R3=inv2"}, 96 {2, "R3_w=inv2"},
97 {3, "R3=inv4"}, 97 {3, "R3_w=inv4"},
98 {4, "R3=inv8"}, 98 {4, "R3_w=inv8"},
99 {5, "R3=inv16"}, 99 {5, "R3_w=inv16"},
100 {6, "R3=inv1"}, 100 {6, "R3_w=inv1"},
101 {7, "R4=inv32"}, 101 {7, "R4_w=inv32"},
102 {8, "R4=inv16"}, 102 {8, "R4_w=inv16"},
103 {9, "R4=inv8"}, 103 {9, "R4_w=inv8"},
104 {10, "R4=inv4"}, 104 {10, "R4_w=inv4"},
105 {11, "R4=inv2"}, 105 {11, "R4_w=inv2"},
106 }, 106 },
107 }, 107 },
108 { 108 {
@@ -121,12 +121,12 @@ static struct bpf_align_test tests[] = {
121 .matches = { 121 .matches = {
122 {1, "R1=ctx(id=0,off=0,imm=0)"}, 122 {1, "R1=ctx(id=0,off=0,imm=0)"},
123 {1, "R10=fp0"}, 123 {1, "R10=fp0"},
124 {1, "R3=inv4"}, 124 {1, "R3_w=inv4"},
125 {2, "R3=inv8"}, 125 {2, "R3_w=inv8"},
126 {3, "R3=inv10"}, 126 {3, "R3_w=inv10"},
127 {4, "R4=inv8"}, 127 {4, "R4_w=inv8"},
128 {5, "R4=inv12"}, 128 {5, "R4_w=inv12"},
129 {6, "R4=inv14"}, 129 {6, "R4_w=inv14"},
130 }, 130 },
131 }, 131 },
132 { 132 {
@@ -143,10 +143,10 @@ static struct bpf_align_test tests[] = {
143 .matches = { 143 .matches = {
144 {1, "R1=ctx(id=0,off=0,imm=0)"}, 144 {1, "R1=ctx(id=0,off=0,imm=0)"},
145 {1, "R10=fp0"}, 145 {1, "R10=fp0"},
146 {1, "R3=inv7"}, 146 {1, "R3_w=inv7"},
147 {2, "R3=inv7"}, 147 {2, "R3_w=inv7"},
148 {3, "R3=inv14"}, 148 {3, "R3_w=inv14"},
149 {4, "R3=inv56"}, 149 {4, "R3_w=inv56"},
150 }, 150 },
151 }, 151 },
152 152
@@ -185,18 +185,18 @@ static struct bpf_align_test tests[] = {
185 .prog_type = BPF_PROG_TYPE_SCHED_CLS, 185 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
186 .matches = { 186 .matches = {
187 {7, "R0=pkt(id=0,off=8,r=8,imm=0)"}, 187 {7, "R0=pkt(id=0,off=8,r=8,imm=0)"},
188 {7, "R3=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"}, 188 {7, "R3_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},
189 {8, "R3=inv(id=0,umax_value=510,var_off=(0x0; 0x1fe))"}, 189 {8, "R3_w=inv(id=0,umax_value=510,var_off=(0x0; 0x1fe))"},
190 {9, "R3=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"}, 190 {9, "R3_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
191 {10, "R3=inv(id=0,umax_value=2040,var_off=(0x0; 0x7f8))"}, 191 {10, "R3_w=inv(id=0,umax_value=2040,var_off=(0x0; 0x7f8))"},
192 {11, "R3=inv(id=0,umax_value=4080,var_off=(0x0; 0xff0))"}, 192 {11, "R3_w=inv(id=0,umax_value=4080,var_off=(0x0; 0xff0))"},
193 {18, "R3=pkt_end(id=0,off=0,imm=0)"}, 193 {18, "R3=pkt_end(id=0,off=0,imm=0)"},
194 {18, "R4=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"}, 194 {18, "R4_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},
195 {19, "R4=inv(id=0,umax_value=8160,var_off=(0x0; 0x1fe0))"}, 195 {19, "R4_w=inv(id=0,umax_value=8160,var_off=(0x0; 0x1fe0))"},
196 {20, "R4=inv(id=0,umax_value=4080,var_off=(0x0; 0xff0))"}, 196 {20, "R4_w=inv(id=0,umax_value=4080,var_off=(0x0; 0xff0))"},
197 {21, "R4=inv(id=0,umax_value=2040,var_off=(0x0; 0x7f8))"}, 197 {21, "R4_w=inv(id=0,umax_value=2040,var_off=(0x0; 0x7f8))"},
198 {22, "R4=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"}, 198 {22, "R4_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
199 {23, "R4=inv(id=0,umax_value=510,var_off=(0x0; 0x1fe))"}, 199 {23, "R4_w=inv(id=0,umax_value=510,var_off=(0x0; 0x1fe))"},
200 }, 200 },
201 }, 201 },
202 { 202 {
@@ -217,16 +217,16 @@ static struct bpf_align_test tests[] = {
217 }, 217 },
218 .prog_type = BPF_PROG_TYPE_SCHED_CLS, 218 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
219 .matches = { 219 .matches = {
220 {7, "R3=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"}, 220 {7, "R3_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},
221 {8, "R4=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"}, 221 {8, "R4_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},
222 {9, "R4=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"}, 222 {9, "R4_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},
223 {10, "R4=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"}, 223 {10, "R4_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},
224 {11, "R4=inv(id=0,umax_value=510,var_off=(0x0; 0x1fe))"}, 224 {11, "R4_w=inv(id=0,umax_value=510,var_off=(0x0; 0x1fe))"},
225 {12, "R4=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"}, 225 {12, "R4_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},
226 {13, "R4=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"}, 226 {13, "R4_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
227 {14, "R4=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"}, 227 {14, "R4_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},
228 {15, "R4=inv(id=0,umax_value=2040,var_off=(0x0; 0x7f8))"}, 228 {15, "R4_w=inv(id=0,umax_value=2040,var_off=(0x0; 0x7f8))"},
229 {16, "R4=inv(id=0,umax_value=4080,var_off=(0x0; 0xff0))"}, 229 {16, "R4_w=inv(id=0,umax_value=4080,var_off=(0x0; 0xff0))"},
230 }, 230 },
231 }, 231 },
232 { 232 {
@@ -257,14 +257,14 @@ static struct bpf_align_test tests[] = {
257 }, 257 },
258 .prog_type = BPF_PROG_TYPE_SCHED_CLS, 258 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
259 .matches = { 259 .matches = {
260 {4, "R5=pkt(id=0,off=0,r=0,imm=0)"}, 260 {4, "R5_w=pkt(id=0,off=0,r=0,imm=0)"},
261 {5, "R5=pkt(id=0,off=14,r=0,imm=0)"}, 261 {5, "R5_w=pkt(id=0,off=14,r=0,imm=0)"},
262 {6, "R4=pkt(id=0,off=14,r=0,imm=0)"}, 262 {6, "R4_w=pkt(id=0,off=14,r=0,imm=0)"},
263 {10, "R2=pkt(id=0,off=0,r=18,imm=0)"}, 263 {10, "R2=pkt(id=0,off=0,r=18,imm=0)"},
264 {10, "R5=pkt(id=0,off=14,r=18,imm=0)"}, 264 {10, "R5=pkt(id=0,off=14,r=18,imm=0)"},
265 {10, "R4=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"}, 265 {10, "R4_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},
266 {14, "R4=inv(id=0,umax_value=65535,var_off=(0x0; 0xffff))"}, 266 {14, "R4_w=inv(id=0,umax_value=65535,var_off=(0x0; 0xffff))"},
267 {15, "R4=inv(id=0,umax_value=65535,var_off=(0x0; 0xffff))"}, 267 {15, "R4_w=inv(id=0,umax_value=65535,var_off=(0x0; 0xffff))"},
268 }, 268 },
269 }, 269 },
270 { 270 {
@@ -320,11 +320,11 @@ static struct bpf_align_test tests[] = {
320 * alignment of 4. 320 * alignment of 4.
321 */ 321 */
322 {8, "R2=pkt(id=0,off=0,r=8,imm=0)"}, 322 {8, "R2=pkt(id=0,off=0,r=8,imm=0)"},
323 {8, "R6=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"}, 323 {8, "R6_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
324 /* Offset is added to packet pointer R5, resulting in 324 /* Offset is added to packet pointer R5, resulting in
325 * known fixed offset, and variable offset from R6. 325 * known fixed offset, and variable offset from R6.
326 */ 326 */
327 {11, "R5=pkt(id=1,off=14,r=0,umax_value=1020,var_off=(0x0; 0x3fc))"}, 327 {11, "R5_w=pkt(id=1,off=14,r=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
328 /* At the time the word size load is performed from R5, 328 /* At the time the word size load is performed from R5,
329 * it's total offset is NET_IP_ALIGN + reg->off (0) + 329 * it's total offset is NET_IP_ALIGN + reg->off (0) +
330 * reg->aux_off (14) which is 16. Then the variable 330 * reg->aux_off (14) which is 16. Then the variable
@@ -336,11 +336,11 @@ static struct bpf_align_test tests[] = {
336 /* Variable offset is added to R5 packet pointer, 336 /* Variable offset is added to R5 packet pointer,
337 * resulting in auxiliary alignment of 4. 337 * resulting in auxiliary alignment of 4.
338 */ 338 */
339 {18, "R5=pkt(id=2,off=0,r=0,umax_value=1020,var_off=(0x0; 0x3fc))"}, 339 {18, "R5_w=pkt(id=2,off=0,r=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
340 /* Constant offset is added to R5, resulting in 340 /* Constant offset is added to R5, resulting in
341 * reg->off of 14. 341 * reg->off of 14.
342 */ 342 */
343 {19, "R5=pkt(id=2,off=14,r=0,umax_value=1020,var_off=(0x0; 0x3fc))"}, 343 {19, "R5_w=pkt(id=2,off=14,r=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
344 /* At the time the word size load is performed from R5, 344 /* At the time the word size load is performed from R5,
345 * its total fixed offset is NET_IP_ALIGN + reg->off 345 * its total fixed offset is NET_IP_ALIGN + reg->off
346 * (14) which is 16. Then the variable offset is 4-byte 346 * (14) which is 16. Then the variable offset is 4-byte
@@ -352,18 +352,18 @@ static struct bpf_align_test tests[] = {
352 /* Constant offset is added to R5 packet pointer, 352 /* Constant offset is added to R5 packet pointer,
353 * resulting in reg->off value of 14. 353 * resulting in reg->off value of 14.
354 */ 354 */
355 {26, "R5=pkt(id=0,off=14,r=8"}, 355 {26, "R5_w=pkt(id=0,off=14,r=8"},
356 /* Variable offset is added to R5, resulting in a 356 /* Variable offset is added to R5, resulting in a
357 * variable offset of (4n). 357 * variable offset of (4n).
358 */ 358 */
359 {27, "R5=pkt(id=3,off=14,r=0,umax_value=1020,var_off=(0x0; 0x3fc))"}, 359 {27, "R5_w=pkt(id=3,off=14,r=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
360 /* Constant is added to R5 again, setting reg->off to 18. */ 360 /* Constant is added to R5 again, setting reg->off to 18. */
361 {28, "R5=pkt(id=3,off=18,r=0,umax_value=1020,var_off=(0x0; 0x3fc))"}, 361 {28, "R5_w=pkt(id=3,off=18,r=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
362 /* And once more we add a variable; resulting var_off 362 /* And once more we add a variable; resulting var_off
363 * is still (4n), fixed offset is not changed. 363 * is still (4n), fixed offset is not changed.
364 * Also, we create a new reg->id. 364 * Also, we create a new reg->id.
365 */ 365 */
366 {29, "R5=pkt(id=4,off=18,r=0,umax_value=2040,var_off=(0x0; 0x7fc))"}, 366 {29, "R5_w=pkt(id=4,off=18,r=0,umax_value=2040,var_off=(0x0; 0x7fc))"},
367 /* At the time the word size load is performed from R5, 367 /* At the time the word size load is performed from R5,
368 * its total fixed offset is NET_IP_ALIGN + reg->off (18) 368 * its total fixed offset is NET_IP_ALIGN + reg->off (18)
369 * which is 20. Then the variable offset is (4n), so 369 * which is 20. Then the variable offset is (4n), so
@@ -410,11 +410,11 @@ static struct bpf_align_test tests[] = {
410 * alignment of 4. 410 * alignment of 4.
411 */ 411 */
412 {8, "R2=pkt(id=0,off=0,r=8,imm=0)"}, 412 {8, "R2=pkt(id=0,off=0,r=8,imm=0)"},
413 {8, "R6=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"}, 413 {8, "R6_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
414 /* Adding 14 makes R6 be (4n+2) */ 414 /* Adding 14 makes R6 be (4n+2) */
415 {9, "R6=inv(id=0,umin_value=14,umax_value=1034,var_off=(0x2; 0x7fc))"}, 415 {9, "R6_w=inv(id=0,umin_value=14,umax_value=1034,var_off=(0x2; 0x7fc))"},
416 /* Packet pointer has (4n+2) offset */ 416 /* Packet pointer has (4n+2) offset */
417 {11, "R5=pkt(id=1,off=0,r=0,umin_value=14,umax_value=1034,var_off=(0x2; 0x7fc))"}, 417 {11, "R5_w=pkt(id=1,off=0,r=0,umin_value=14,umax_value=1034,var_off=(0x2; 0x7fc))"},
418 {13, "R4=pkt(id=1,off=4,r=0,umin_value=14,umax_value=1034,var_off=(0x2; 0x7fc))"}, 418 {13, "R4=pkt(id=1,off=4,r=0,umin_value=14,umax_value=1034,var_off=(0x2; 0x7fc))"},
419 /* At the time the word size load is performed from R5, 419 /* At the time the word size load is performed from R5,
420 * its total fixed offset is NET_IP_ALIGN + reg->off (0) 420 * its total fixed offset is NET_IP_ALIGN + reg->off (0)
@@ -426,11 +426,11 @@ static struct bpf_align_test tests[] = {
426 /* Newly read value in R6 was shifted left by 2, so has 426 /* Newly read value in R6 was shifted left by 2, so has
427 * known alignment of 4. 427 * known alignment of 4.
428 */ 428 */
429 {18, "R6=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"}, 429 {18, "R6_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
430 /* Added (4n) to packet pointer's (4n+2) var_off, giving 430 /* Added (4n) to packet pointer's (4n+2) var_off, giving
431 * another (4n+2). 431 * another (4n+2).
432 */ 432 */
433 {19, "R5=pkt(id=2,off=0,r=0,umin_value=14,umax_value=2054,var_off=(0x2; 0xffc))"}, 433 {19, "R5_w=pkt(id=2,off=0,r=0,umin_value=14,umax_value=2054,var_off=(0x2; 0xffc))"},
434 {21, "R4=pkt(id=2,off=4,r=0,umin_value=14,umax_value=2054,var_off=(0x2; 0xffc))"}, 434 {21, "R4=pkt(id=2,off=4,r=0,umin_value=14,umax_value=2054,var_off=(0x2; 0xffc))"},
435 /* At the time the word size load is performed from R5, 435 /* At the time the word size load is performed from R5,
436 * its total fixed offset is NET_IP_ALIGN + reg->off (0) 436 * its total fixed offset is NET_IP_ALIGN + reg->off (0)
@@ -446,11 +446,9 @@ static struct bpf_align_test tests[] = {
446 .insns = { 446 .insns = {
447 PREP_PKT_POINTERS, 447 PREP_PKT_POINTERS,
448 BPF_MOV64_IMM(BPF_REG_0, 0), 448 BPF_MOV64_IMM(BPF_REG_0, 0),
449 /* ptr & const => unknown & const */ 449 /* (ptr - ptr) << 2 */
450 BPF_MOV64_REG(BPF_REG_5, BPF_REG_2), 450 BPF_MOV64_REG(BPF_REG_5, BPF_REG_3),
451 BPF_ALU64_IMM(BPF_AND, BPF_REG_5, 0x40), 451 BPF_ALU64_REG(BPF_SUB, BPF_REG_5, BPF_REG_2),
452 /* ptr << const => unknown << const */
453 BPF_MOV64_REG(BPF_REG_5, BPF_REG_2),
454 BPF_ALU64_IMM(BPF_LSH, BPF_REG_5, 2), 452 BPF_ALU64_IMM(BPF_LSH, BPF_REG_5, 2),
455 /* We have a (4n) value. Let's make a packet offset 453 /* We have a (4n) value. Let's make a packet offset
456 * out of it. First add 14, to make it a (4n+2) 454 * out of it. First add 14, to make it a (4n+2)
@@ -473,8 +471,26 @@ static struct bpf_align_test tests[] = {
473 .prog_type = BPF_PROG_TYPE_SCHED_CLS, 471 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
474 .result = REJECT, 472 .result = REJECT,
475 .matches = { 473 .matches = {
476 {4, "R5=pkt(id=0,off=0,r=0,imm=0)"}, 474 {4, "R5_w=pkt_end(id=0,off=0,imm=0)"},
477 /* R5 bitwise operator &= on pointer prohibited */ 475 /* (ptr - ptr) << 2 == unknown, (4n) */
476 {6, "R5_w=inv(id=0,smax_value=9223372036854775804,umax_value=18446744073709551612,var_off=(0x0; 0xfffffffffffffffc))"},
477 /* (4n) + 14 == (4n+2). We blow our bounds, because
478 * the add could overflow.
479 */
480 {7, "R5=inv(id=0,var_off=(0x2; 0xfffffffffffffffc))"},
481 /* Checked s>=0 */
482 {9, "R5=inv(id=0,umin_value=2,umax_value=9223372036854775806,var_off=(0x2; 0x7ffffffffffffffc))"},
483 /* packet pointer + nonnegative (4n+2) */
484 {11, "R6_w=pkt(id=1,off=0,r=0,umin_value=2,umax_value=9223372036854775806,var_off=(0x2; 0x7ffffffffffffffc))"},
485 {13, "R4=pkt(id=1,off=4,r=0,umin_value=2,umax_value=9223372036854775806,var_off=(0x2; 0x7ffffffffffffffc))"},
486 /* NET_IP_ALIGN + (4n+2) == (4n), alignment is fine.
487 * We checked the bounds, but it might have been able
488 * to overflow if the packet pointer started in the
489 * upper half of the address space.
490 * So we did not get a 'range' on R6, and the access
491 * attempt will fail.
492 */
493 {15, "R6=pkt(id=1,off=0,r=0,umin_value=2,umax_value=9223372036854775806,var_off=(0x2; 0x7ffffffffffffffc))"},
478 } 494 }
479 }, 495 },
480 { 496 {
@@ -510,11 +526,11 @@ static struct bpf_align_test tests[] = {
510 * alignment of 4. 526 * alignment of 4.
511 */ 527 */
512 {7, "R2=pkt(id=0,off=0,r=8,imm=0)"}, 528 {7, "R2=pkt(id=0,off=0,r=8,imm=0)"},
513 {9, "R6=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"}, 529 {9, "R6_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
514 /* Adding 14 makes R6 be (4n+2) */ 530 /* Adding 14 makes R6 be (4n+2) */
515 {10, "R6=inv(id=0,umin_value=14,umax_value=1034,var_off=(0x2; 0x7fc))"}, 531 {10, "R6_w=inv(id=0,umin_value=14,umax_value=1034,var_off=(0x2; 0x7fc))"},
516 /* New unknown value in R7 is (4n) */ 532 /* New unknown value in R7 is (4n) */
517 {11, "R7=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"}, 533 {11, "R7_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
518 /* Subtracting it from R6 blows our unsigned bounds */ 534 /* Subtracting it from R6 blows our unsigned bounds */
519 {12, "R6=inv(id=0,smin_value=-1006,smax_value=1034,var_off=(0x2; 0xfffffffffffffffc))"}, 535 {12, "R6=inv(id=0,smin_value=-1006,smax_value=1034,var_off=(0x2; 0xfffffffffffffffc))"},
520 /* Checked s>= 0 */ 536 /* Checked s>= 0 */
@@ -563,15 +579,15 @@ static struct bpf_align_test tests[] = {
563 * alignment of 4. 579 * alignment of 4.
564 */ 580 */
565 {7, "R2=pkt(id=0,off=0,r=8,imm=0)"}, 581 {7, "R2=pkt(id=0,off=0,r=8,imm=0)"},
566 {10, "R6=inv(id=0,umax_value=60,var_off=(0x0; 0x3c))"}, 582 {10, "R6_w=inv(id=0,umax_value=60,var_off=(0x0; 0x3c))"},
567 /* Adding 14 makes R6 be (4n+2) */ 583 /* Adding 14 makes R6 be (4n+2) */
568 {11, "R6=inv(id=0,umin_value=14,umax_value=74,var_off=(0x2; 0x7c))"}, 584 {11, "R6_w=inv(id=0,umin_value=14,umax_value=74,var_off=(0x2; 0x7c))"},
569 /* Subtracting from packet pointer overflows ubounds */ 585 /* Subtracting from packet pointer overflows ubounds */
570 {13, "R5=pkt(id=1,off=0,r=8,umin_value=18446744073709551542,umax_value=18446744073709551602,var_off=(0xffffffffffffff82; 0x7c))"}, 586 {13, "R5_w=pkt(id=1,off=0,r=8,umin_value=18446744073709551542,umax_value=18446744073709551602,var_off=(0xffffffffffffff82; 0x7c))"},
571 /* New unknown value in R7 is (4n), >= 76 */ 587 /* New unknown value in R7 is (4n), >= 76 */
572 {15, "R7=inv(id=0,umin_value=76,umax_value=1096,var_off=(0x0; 0x7fc))"}, 588 {15, "R7_w=inv(id=0,umin_value=76,umax_value=1096,var_off=(0x0; 0x7fc))"},
573 /* Adding it to packet pointer gives nice bounds again */ 589 /* Adding it to packet pointer gives nice bounds again */
574 {16, "R5=pkt(id=2,off=0,r=0,umin_value=2,umax_value=1082,var_off=(0x2; 0x7fc))"}, 590 {16, "R5_w=pkt(id=2,off=0,r=0,umin_value=2,umax_value=1082,var_off=(0x2; 0x7fc))"},
575 /* At the time the word size load is performed from R5, 591 /* At the time the word size load is performed from R5,
576 * its total fixed offset is NET_IP_ALIGN + reg->off (0) 592 * its total fixed offset is NET_IP_ALIGN + reg->off (0)
577 * which is 2. Then the variable offset is (4n+2), so 593 * which is 2. Then the variable offset is (4n+2), so
diff --git a/tools/testing/selftests/bpf/test_dev_cgroup.c b/tools/testing/selftests/bpf/test_dev_cgroup.c
index 02c85d6c89b0..3489cc283433 100644
--- a/tools/testing/selftests/bpf/test_dev_cgroup.c
+++ b/tools/testing/selftests/bpf/test_dev_cgroup.c
@@ -10,6 +10,8 @@
10#include <string.h> 10#include <string.h>
11#include <errno.h> 11#include <errno.h>
12#include <assert.h> 12#include <assert.h>
13#include <sys/time.h>
14#include <sys/resource.h>
13 15
14#include <linux/bpf.h> 16#include <linux/bpf.h>
15#include <bpf/bpf.h> 17#include <bpf/bpf.h>
@@ -19,19 +21,23 @@
19 21
20#define DEV_CGROUP_PROG "./dev_cgroup.o" 22#define DEV_CGROUP_PROG "./dev_cgroup.o"
21 23
22#define TEST_CGROUP "test-bpf-based-device-cgroup/" 24#define TEST_CGROUP "/test-bpf-based-device-cgroup/"
23 25
24int main(int argc, char **argv) 26int main(int argc, char **argv)
25{ 27{
28 struct rlimit limit = { RLIM_INFINITY, RLIM_INFINITY };
26 struct bpf_object *obj; 29 struct bpf_object *obj;
27 int error = EXIT_FAILURE; 30 int error = EXIT_FAILURE;
28 int prog_fd, cgroup_fd; 31 int prog_fd, cgroup_fd;
29 __u32 prog_cnt; 32 __u32 prog_cnt;
30 33
34 if (setrlimit(RLIMIT_MEMLOCK, &limit) < 0)
35 perror("Unable to lift memlock rlimit");
36
31 if (bpf_prog_load(DEV_CGROUP_PROG, BPF_PROG_TYPE_CGROUP_DEVICE, 37 if (bpf_prog_load(DEV_CGROUP_PROG, BPF_PROG_TYPE_CGROUP_DEVICE,
32 &obj, &prog_fd)) { 38 &obj, &prog_fd)) {
33 printf("Failed to load DEV_CGROUP program\n"); 39 printf("Failed to load DEV_CGROUP program\n");
34 goto err; 40 goto out;
35 } 41 }
36 42
37 if (setup_cgroup_environment()) { 43 if (setup_cgroup_environment()) {
@@ -89,5 +95,6 @@ int main(int argc, char **argv)
89err: 95err:
90 cleanup_cgroup_environment(); 96 cleanup_cgroup_environment();
91 97
98out:
92 return error; 99 return error;
93} 100}
diff --git a/tools/testing/selftests/bpf/test_l4lb_noinline.c b/tools/testing/selftests/bpf/test_l4lb_noinline.c
new file mode 100644
index 000000000000..ba44a14e6dc4
--- /dev/null
+++ b/tools/testing/selftests/bpf/test_l4lb_noinline.c
@@ -0,0 +1,473 @@
1// SPDX-License-Identifier: GPL-2.0
2// Copyright (c) 2017 Facebook
3#include <stddef.h>
4#include <stdbool.h>
5#include <string.h>
6#include <linux/pkt_cls.h>
7#include <linux/bpf.h>
8#include <linux/in.h>
9#include <linux/if_ether.h>
10#include <linux/ip.h>
11#include <linux/ipv6.h>
12#include <linux/icmp.h>
13#include <linux/icmpv6.h>
14#include <linux/tcp.h>
15#include <linux/udp.h>
16#include "bpf_helpers.h"
17#include "test_iptunnel_common.h"
18#include "bpf_endian.h"
19
20int _version SEC("version") = 1;
21
22static __u32 rol32(__u32 word, unsigned int shift)
23{
24 return (word << shift) | (word >> ((-shift) & 31));
25}
26
27/* copy paste of jhash from kernel sources to make sure llvm
28 * can compile it into valid sequence of bpf instructions
29 */
30#define __jhash_mix(a, b, c) \
31{ \
32 a -= c; a ^= rol32(c, 4); c += b; \
33 b -= a; b ^= rol32(a, 6); a += c; \
34 c -= b; c ^= rol32(b, 8); b += a; \
35 a -= c; a ^= rol32(c, 16); c += b; \
36 b -= a; b ^= rol32(a, 19); a += c; \
37 c -= b; c ^= rol32(b, 4); b += a; \
38}
39
40#define __jhash_final(a, b, c) \
41{ \
42 c ^= b; c -= rol32(b, 14); \
43 a ^= c; a -= rol32(c, 11); \
44 b ^= a; b -= rol32(a, 25); \
45 c ^= b; c -= rol32(b, 16); \
46 a ^= c; a -= rol32(c, 4); \
47 b ^= a; b -= rol32(a, 14); \
48 c ^= b; c -= rol32(b, 24); \
49}
50
51#define JHASH_INITVAL 0xdeadbeef
52
53typedef unsigned int u32;
54
55static u32 jhash(const void *key, u32 length, u32 initval)
56{
57 u32 a, b, c;
58 const unsigned char *k = key;
59
60 a = b = c = JHASH_INITVAL + length + initval;
61
62 while (length > 12) {
63 a += *(u32 *)(k);
64 b += *(u32 *)(k + 4);
65 c += *(u32 *)(k + 8);
66 __jhash_mix(a, b, c);
67 length -= 12;
68 k += 12;
69 }
70 switch (length) {
71 case 12: c += (u32)k[11]<<24;
72 case 11: c += (u32)k[10]<<16;
73 case 10: c += (u32)k[9]<<8;
74 case 9: c += k[8];
75 case 8: b += (u32)k[7]<<24;
76 case 7: b += (u32)k[6]<<16;
77 case 6: b += (u32)k[5]<<8;
78 case 5: b += k[4];
79 case 4: a += (u32)k[3]<<24;
80 case 3: a += (u32)k[2]<<16;
81 case 2: a += (u32)k[1]<<8;
82 case 1: a += k[0];
83 __jhash_final(a, b, c);
84 case 0: /* Nothing left to add */
85 break;
86 }
87
88 return c;
89}
90
91static u32 __jhash_nwords(u32 a, u32 b, u32 c, u32 initval)
92{
93 a += initval;
94 b += initval;
95 c += initval;
96 __jhash_final(a, b, c);
97 return c;
98}
99
100static u32 jhash_2words(u32 a, u32 b, u32 initval)
101{
102 return __jhash_nwords(a, b, 0, initval + JHASH_INITVAL + (2 << 2));
103}
104
105#define PCKT_FRAGMENTED 65343
106#define IPV4_HDR_LEN_NO_OPT 20
107#define IPV4_PLUS_ICMP_HDR 28
108#define IPV6_PLUS_ICMP_HDR 48
109#define RING_SIZE 2
110#define MAX_VIPS 12
111#define MAX_REALS 5
112#define CTL_MAP_SIZE 16
113#define CH_RINGS_SIZE (MAX_VIPS * RING_SIZE)
114#define F_IPV6 (1 << 0)
115#define F_HASH_NO_SRC_PORT (1 << 0)
116#define F_ICMP (1 << 0)
117#define F_SYN_SET (1 << 1)
118
119struct packet_description {
120 union {
121 __be32 src;
122 __be32 srcv6[4];
123 };
124 union {
125 __be32 dst;
126 __be32 dstv6[4];
127 };
128 union {
129 __u32 ports;
130 __u16 port16[2];
131 };
132 __u8 proto;
133 __u8 flags;
134};
135
136struct ctl_value {
137 union {
138 __u64 value;
139 __u32 ifindex;
140 __u8 mac[6];
141 };
142};
143
144struct vip_meta {
145 __u32 flags;
146 __u32 vip_num;
147};
148
149struct real_definition {
150 union {
151 __be32 dst;
152 __be32 dstv6[4];
153 };
154 __u8 flags;
155};
156
157struct vip_stats {
158 __u64 bytes;
159 __u64 pkts;
160};
161
162struct eth_hdr {
163 unsigned char eth_dest[ETH_ALEN];
164 unsigned char eth_source[ETH_ALEN];
165 unsigned short eth_proto;
166};
167
168struct bpf_map_def SEC("maps") vip_map = {
169 .type = BPF_MAP_TYPE_HASH,
170 .key_size = sizeof(struct vip),
171 .value_size = sizeof(struct vip_meta),
172 .max_entries = MAX_VIPS,
173};
174
175struct bpf_map_def SEC("maps") ch_rings = {
176 .type = BPF_MAP_TYPE_ARRAY,
177 .key_size = sizeof(__u32),
178 .value_size = sizeof(__u32),
179 .max_entries = CH_RINGS_SIZE,
180};
181
182struct bpf_map_def SEC("maps") reals = {
183 .type = BPF_MAP_TYPE_ARRAY,
184 .key_size = sizeof(__u32),
185 .value_size = sizeof(struct real_definition),
186 .max_entries = MAX_REALS,
187};
188
189struct bpf_map_def SEC("maps") stats = {
190 .type = BPF_MAP_TYPE_PERCPU_ARRAY,
191 .key_size = sizeof(__u32),
192 .value_size = sizeof(struct vip_stats),
193 .max_entries = MAX_VIPS,
194};
195
196struct bpf_map_def SEC("maps") ctl_array = {
197 .type = BPF_MAP_TYPE_ARRAY,
198 .key_size = sizeof(__u32),
199 .value_size = sizeof(struct ctl_value),
200 .max_entries = CTL_MAP_SIZE,
201};
202
203static __u32 get_packet_hash(struct packet_description *pckt,
204 bool ipv6)
205{
206 if (ipv6)
207 return jhash_2words(jhash(pckt->srcv6, 16, MAX_VIPS),
208 pckt->ports, CH_RINGS_SIZE);
209 else
210 return jhash_2words(pckt->src, pckt->ports, CH_RINGS_SIZE);
211}
212
213static bool get_packet_dst(struct real_definition **real,
214 struct packet_description *pckt,
215 struct vip_meta *vip_info,
216 bool is_ipv6)
217{
218 __u32 hash = get_packet_hash(pckt, is_ipv6);
219 __u32 key = RING_SIZE * vip_info->vip_num + hash % RING_SIZE;
220 __u32 *real_pos;
221
222 if (hash != 0x358459b7 /* jhash of ipv4 packet */ &&
223 hash != 0x2f4bc6bb /* jhash of ipv6 packet */)
224 return 0;
225
226 real_pos = bpf_map_lookup_elem(&ch_rings, &key);
227 if (!real_pos)
228 return false;
229 key = *real_pos;
230 *real = bpf_map_lookup_elem(&reals, &key);
231 if (!(*real))
232 return false;
233 return true;
234}
235
236static int parse_icmpv6(void *data, void *data_end, __u64 off,
237 struct packet_description *pckt)
238{
239 struct icmp6hdr *icmp_hdr;
240 struct ipv6hdr *ip6h;
241
242 icmp_hdr = data + off;
243 if (icmp_hdr + 1 > data_end)
244 return TC_ACT_SHOT;
245 if (icmp_hdr->icmp6_type != ICMPV6_PKT_TOOBIG)
246 return TC_ACT_OK;
247 off += sizeof(struct icmp6hdr);
248 ip6h = data + off;
249 if (ip6h + 1 > data_end)
250 return TC_ACT_SHOT;
251 pckt->proto = ip6h->nexthdr;
252 pckt->flags |= F_ICMP;
253 memcpy(pckt->srcv6, ip6h->daddr.s6_addr32, 16);
254 memcpy(pckt->dstv6, ip6h->saddr.s6_addr32, 16);
255 return TC_ACT_UNSPEC;
256}
257
258static int parse_icmp(void *data, void *data_end, __u64 off,
259 struct packet_description *pckt)
260{
261 struct icmphdr *icmp_hdr;
262 struct iphdr *iph;
263
264 icmp_hdr = data + off;
265 if (icmp_hdr + 1 > data_end)
266 return TC_ACT_SHOT;
267 if (icmp_hdr->type != ICMP_DEST_UNREACH ||
268 icmp_hdr->code != ICMP_FRAG_NEEDED)
269 return TC_ACT_OK;
270 off += sizeof(struct icmphdr);
271 iph = data + off;
272 if (iph + 1 > data_end)
273 return TC_ACT_SHOT;
274 if (iph->ihl != 5)
275 return TC_ACT_SHOT;
276 pckt->proto = iph->protocol;
277 pckt->flags |= F_ICMP;
278 pckt->src = iph->daddr;
279 pckt->dst = iph->saddr;
280 return TC_ACT_UNSPEC;
281}
282
283static bool parse_udp(void *data, __u64 off, void *data_end,
284 struct packet_description *pckt)
285{
286 struct udphdr *udp;
287 udp = data + off;
288
289 if (udp + 1 > data_end)
290 return false;
291
292 if (!(pckt->flags & F_ICMP)) {
293 pckt->port16[0] = udp->source;
294 pckt->port16[1] = udp->dest;
295 } else {
296 pckt->port16[0] = udp->dest;
297 pckt->port16[1] = udp->source;
298 }
299 return true;
300}
301
302static bool parse_tcp(void *data, __u64 off, void *data_end,
303 struct packet_description *pckt)
304{
305 struct tcphdr *tcp;
306
307 tcp = data + off;
308 if (tcp + 1 > data_end)
309 return false;
310
311 if (tcp->syn)
312 pckt->flags |= F_SYN_SET;
313
314 if (!(pckt->flags & F_ICMP)) {
315 pckt->port16[0] = tcp->source;
316 pckt->port16[1] = tcp->dest;
317 } else {
318 pckt->port16[0] = tcp->dest;
319 pckt->port16[1] = tcp->source;
320 }
321 return true;
322}
323
324static int process_packet(void *data, __u64 off, void *data_end,
325 bool is_ipv6, struct __sk_buff *skb)
326{
327 void *pkt_start = (void *)(long)skb->data;
328 struct packet_description pckt = {};
329 struct eth_hdr *eth = pkt_start;
330 struct bpf_tunnel_key tkey = {};
331 struct vip_stats *data_stats;
332 struct real_definition *dst;
333 struct vip_meta *vip_info;
334 struct ctl_value *cval;
335 __u32 v4_intf_pos = 1;
336 __u32 v6_intf_pos = 2;
337 struct ipv6hdr *ip6h;
338 struct vip vip = {};
339 struct iphdr *iph;
340 int tun_flag = 0;
341 __u16 pkt_bytes;
342 __u64 iph_len;
343 __u32 ifindex;
344 __u8 protocol;
345 __u32 vip_num;
346 int action;
347
348 tkey.tunnel_ttl = 64;
349 if (is_ipv6) {
350 ip6h = data + off;
351 if (ip6h + 1 > data_end)
352 return TC_ACT_SHOT;
353
354 iph_len = sizeof(struct ipv6hdr);
355 protocol = ip6h->nexthdr;
356 pckt.proto = protocol;
357 pkt_bytes = bpf_ntohs(ip6h->payload_len);
358 off += iph_len;
359 if (protocol == IPPROTO_FRAGMENT) {
360 return TC_ACT_SHOT;
361 } else if (protocol == IPPROTO_ICMPV6) {
362 action = parse_icmpv6(data, data_end, off, &pckt);
363 if (action >= 0)
364 return action;
365 off += IPV6_PLUS_ICMP_HDR;
366 } else {
367 memcpy(pckt.srcv6, ip6h->saddr.s6_addr32, 16);
368 memcpy(pckt.dstv6, ip6h->daddr.s6_addr32, 16);
369 }
370 } else {
371 iph = data + off;
372 if (iph + 1 > data_end)
373 return TC_ACT_SHOT;
374 if (iph->ihl != 5)
375 return TC_ACT_SHOT;
376
377 protocol = iph->protocol;
378 pckt.proto = protocol;
379 pkt_bytes = bpf_ntohs(iph->tot_len);
380 off += IPV4_HDR_LEN_NO_OPT;
381
382 if (iph->frag_off & PCKT_FRAGMENTED)
383 return TC_ACT_SHOT;
384 if (protocol == IPPROTO_ICMP) {
385 action = parse_icmp(data, data_end, off, &pckt);
386 if (action >= 0)
387 return action;
388 off += IPV4_PLUS_ICMP_HDR;
389 } else {
390 pckt.src = iph->saddr;
391 pckt.dst = iph->daddr;
392 }
393 }
394 protocol = pckt.proto;
395
396 if (protocol == IPPROTO_TCP) {
397 if (!parse_tcp(data, off, data_end, &pckt))
398 return TC_ACT_SHOT;
399 } else if (protocol == IPPROTO_UDP) {
400 if (!parse_udp(data, off, data_end, &pckt))
401 return TC_ACT_SHOT;
402 } else {
403 return TC_ACT_SHOT;
404 }
405
406 if (is_ipv6)
407 memcpy(vip.daddr.v6, pckt.dstv6, 16);
408 else
409 vip.daddr.v4 = pckt.dst;
410
411 vip.dport = pckt.port16[1];
412 vip.protocol = pckt.proto;
413 vip_info = bpf_map_lookup_elem(&vip_map, &vip);
414 if (!vip_info) {
415 vip.dport = 0;
416 vip_info = bpf_map_lookup_elem(&vip_map, &vip);
417 if (!vip_info)
418 return TC_ACT_SHOT;
419 pckt.port16[1] = 0;
420 }
421
422 if (vip_info->flags & F_HASH_NO_SRC_PORT)
423 pckt.port16[0] = 0;
424
425 if (!get_packet_dst(&dst, &pckt, vip_info, is_ipv6))
426 return TC_ACT_SHOT;
427
428 if (dst->flags & F_IPV6) {
429 cval = bpf_map_lookup_elem(&ctl_array, &v6_intf_pos);
430 if (!cval)
431 return TC_ACT_SHOT;
432 ifindex = cval->ifindex;
433 memcpy(tkey.remote_ipv6, dst->dstv6, 16);
434 tun_flag = BPF_F_TUNINFO_IPV6;
435 } else {
436 cval = bpf_map_lookup_elem(&ctl_array, &v4_intf_pos);
437 if (!cval)
438 return TC_ACT_SHOT;
439 ifindex = cval->ifindex;
440 tkey.remote_ipv4 = dst->dst;
441 }
442 vip_num = vip_info->vip_num;
443 data_stats = bpf_map_lookup_elem(&stats, &vip_num);
444 if (!data_stats)
445 return TC_ACT_SHOT;
446 data_stats->pkts++;
447 data_stats->bytes += pkt_bytes;
448 bpf_skb_set_tunnel_key(skb, &tkey, sizeof(tkey), tun_flag);
449 *(u32 *)eth->eth_dest = tkey.remote_ipv4;
450 return bpf_redirect(ifindex, 0);
451}
452
453SEC("l4lb-demo")
454int balancer_ingress(struct __sk_buff *ctx)
455{
456 void *data_end = (void *)(long)ctx->data_end;
457 void *data = (void *)(long)ctx->data;
458 struct eth_hdr *eth = data;
459 __u32 eth_proto;
460 __u32 nh_off;
461
462 nh_off = sizeof(struct eth_hdr);
463 if (data + nh_off > data_end)
464 return TC_ACT_SHOT;
465 eth_proto = eth->eth_proto;
466 if (eth_proto == bpf_htons(ETH_P_IP))
467 return process_packet(data, nh_off, data_end, false, ctx);
468 else if (eth_proto == bpf_htons(ETH_P_IPV6))
469 return process_packet(data, nh_off, data_end, true, ctx);
470 else
471 return TC_ACT_SHOT;
472}
473char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/test_lpm_map.c b/tools/testing/selftests/bpf/test_lpm_map.c
index f61480641b6e..2be87e9ee28d 100644
--- a/tools/testing/selftests/bpf/test_lpm_map.c
+++ b/tools/testing/selftests/bpf/test_lpm_map.c
@@ -14,6 +14,7 @@
14#include <errno.h> 14#include <errno.h>
15#include <inttypes.h> 15#include <inttypes.h>
16#include <linux/bpf.h> 16#include <linux/bpf.h>
17#include <pthread.h>
17#include <stdio.h> 18#include <stdio.h>
18#include <stdlib.h> 19#include <stdlib.h>
19#include <string.h> 20#include <string.h>
@@ -521,6 +522,218 @@ static void test_lpm_delete(void)
521 close(map_fd); 522 close(map_fd);
522} 523}
523 524
525static void test_lpm_get_next_key(void)
526{
527 struct bpf_lpm_trie_key *key_p, *next_key_p;
528 size_t key_size;
529 __u32 value = 0;
530 int map_fd;
531
532 key_size = sizeof(*key_p) + sizeof(__u32);
533 key_p = alloca(key_size);
534 next_key_p = alloca(key_size);
535
536 map_fd = bpf_create_map(BPF_MAP_TYPE_LPM_TRIE, key_size, sizeof(value),
537 100, BPF_F_NO_PREALLOC);
538 assert(map_fd >= 0);
539
540 /* empty tree. get_next_key should return ENOENT */
541 assert(bpf_map_get_next_key(map_fd, NULL, key_p) == -1 &&
542 errno == ENOENT);
543
544 /* get and verify the first key, get the second one should fail. */
545 key_p->prefixlen = 16;
546 inet_pton(AF_INET, "192.168.0.0", key_p->data);
547 assert(bpf_map_update_elem(map_fd, key_p, &value, 0) == 0);
548
549 memset(key_p, 0, key_size);
550 assert(bpf_map_get_next_key(map_fd, NULL, key_p) == 0);
551 assert(key_p->prefixlen == 16 && key_p->data[0] == 192 &&
552 key_p->data[1] == 168);
553
554 assert(bpf_map_get_next_key(map_fd, key_p, next_key_p) == -1 &&
555 errno == ENOENT);
556
557 /* no exact matching key should get the first one in post order. */
558 key_p->prefixlen = 8;
559 assert(bpf_map_get_next_key(map_fd, NULL, key_p) == 0);
560 assert(key_p->prefixlen == 16 && key_p->data[0] == 192 &&
561 key_p->data[1] == 168);
562
563 /* add one more element (total two) */
564 key_p->prefixlen = 24;
565 inet_pton(AF_INET, "192.168.0.0", key_p->data);
566 assert(bpf_map_update_elem(map_fd, key_p, &value, 0) == 0);
567
568 memset(key_p, 0, key_size);
569 assert(bpf_map_get_next_key(map_fd, NULL, key_p) == 0);
570 assert(key_p->prefixlen == 24 && key_p->data[0] == 192 &&
571 key_p->data[1] == 168 && key_p->data[2] == 0);
572
573 memset(next_key_p, 0, key_size);
574 assert(bpf_map_get_next_key(map_fd, key_p, next_key_p) == 0);
575 assert(next_key_p->prefixlen == 16 && next_key_p->data[0] == 192 &&
576 next_key_p->data[1] == 168);
577
578 memcpy(key_p, next_key_p, key_size);
579 assert(bpf_map_get_next_key(map_fd, key_p, next_key_p) == -1 &&
580 errno == ENOENT);
581
582 /* Add one more element (total three) */
583 key_p->prefixlen = 24;
584 inet_pton(AF_INET, "192.168.128.0", key_p->data);
585 assert(bpf_map_update_elem(map_fd, key_p, &value, 0) == 0);
586
587 memset(key_p, 0, key_size);
588 assert(bpf_map_get_next_key(map_fd, NULL, key_p) == 0);
589 assert(key_p->prefixlen == 24 && key_p->data[0] == 192 &&
590 key_p->data[1] == 168 && key_p->data[2] == 0);
591
592 memset(next_key_p, 0, key_size);
593 assert(bpf_map_get_next_key(map_fd, key_p, next_key_p) == 0);
594 assert(next_key_p->prefixlen == 24 && next_key_p->data[0] == 192 &&
595 next_key_p->data[1] == 168 && next_key_p->data[2] == 128);
596
597 memcpy(key_p, next_key_p, key_size);
598 assert(bpf_map_get_next_key(map_fd, key_p, next_key_p) == 0);
599 assert(next_key_p->prefixlen == 16 && next_key_p->data[0] == 192 &&
600 next_key_p->data[1] == 168);
601
602 memcpy(key_p, next_key_p, key_size);
603 assert(bpf_map_get_next_key(map_fd, key_p, next_key_p) == -1 &&
604 errno == ENOENT);
605
606 /* Add one more element (total four) */
607 key_p->prefixlen = 24;
608 inet_pton(AF_INET, "192.168.1.0", key_p->data);
609 assert(bpf_map_update_elem(map_fd, key_p, &value, 0) == 0);
610
611 memset(key_p, 0, key_size);
612 assert(bpf_map_get_next_key(map_fd, NULL, key_p) == 0);
613 assert(key_p->prefixlen == 24 && key_p->data[0] == 192 &&
614 key_p->data[1] == 168 && key_p->data[2] == 0);
615
616 memset(next_key_p, 0, key_size);
617 assert(bpf_map_get_next_key(map_fd, key_p, next_key_p) == 0);
618 assert(next_key_p->prefixlen == 24 && next_key_p->data[0] == 192 &&
619 next_key_p->data[1] == 168 && next_key_p->data[2] == 1);
620
621 memcpy(key_p, next_key_p, key_size);
622 assert(bpf_map_get_next_key(map_fd, key_p, next_key_p) == 0);
623 assert(next_key_p->prefixlen == 24 && next_key_p->data[0] == 192 &&
624 next_key_p->data[1] == 168 && next_key_p->data[2] == 128);
625
626 memcpy(key_p, next_key_p, key_size);
627 assert(bpf_map_get_next_key(map_fd, key_p, next_key_p) == 0);
628 assert(next_key_p->prefixlen == 16 && next_key_p->data[0] == 192 &&
629 next_key_p->data[1] == 168);
630
631 memcpy(key_p, next_key_p, key_size);
632 assert(bpf_map_get_next_key(map_fd, key_p, next_key_p) == -1 &&
633 errno == ENOENT);
634
635 /* no exact matching key should return the first one in post order */
636 key_p->prefixlen = 22;
637 inet_pton(AF_INET, "192.168.1.0", key_p->data);
638 assert(bpf_map_get_next_key(map_fd, key_p, next_key_p) == 0);
639 assert(next_key_p->prefixlen == 24 && next_key_p->data[0] == 192 &&
640 next_key_p->data[1] == 168 && next_key_p->data[2] == 0);
641
642 close(map_fd);
643}
644
645#define MAX_TEST_KEYS 4
646struct lpm_mt_test_info {
647 int cmd; /* 0: update, 1: delete, 2: lookup, 3: get_next_key */
648 int iter;
649 int map_fd;
650 struct {
651 __u32 prefixlen;
652 __u32 data;
653 } key[MAX_TEST_KEYS];
654};
655
656static void *lpm_test_command(void *arg)
657{
658 int i, j, ret, iter, key_size;
659 struct lpm_mt_test_info *info = arg;
660 struct bpf_lpm_trie_key *key_p;
661
662 key_size = sizeof(struct bpf_lpm_trie_key) + sizeof(__u32);
663 key_p = alloca(key_size);
664 for (iter = 0; iter < info->iter; iter++)
665 for (i = 0; i < MAX_TEST_KEYS; i++) {
666 /* first half of iterations in forward order,
667 * and second half in backward order.
668 */
669 j = (iter < (info->iter / 2)) ? i : MAX_TEST_KEYS - i - 1;
670 key_p->prefixlen = info->key[j].prefixlen;
671 memcpy(key_p->data, &info->key[j].data, sizeof(__u32));
672 if (info->cmd == 0) {
673 __u32 value = j;
674 /* update must succeed */
675 assert(bpf_map_update_elem(info->map_fd, key_p, &value, 0) == 0);
676 } else if (info->cmd == 1) {
677 ret = bpf_map_delete_elem(info->map_fd, key_p);
678 assert(ret == 0 || errno == ENOENT);
679 } else if (info->cmd == 2) {
680 __u32 value;
681 ret = bpf_map_lookup_elem(info->map_fd, key_p, &value);
682 assert(ret == 0 || errno == ENOENT);
683 } else {
684 struct bpf_lpm_trie_key *next_key_p = alloca(key_size);
685 ret = bpf_map_get_next_key(info->map_fd, key_p, next_key_p);
686 assert(ret == 0 || errno == ENOENT || errno == ENOMEM);
687 }
688 }
689
690 // Pass successful exit info back to the main thread
691 pthread_exit((void *)info);
692}
693
694static void setup_lpm_mt_test_info(struct lpm_mt_test_info *info, int map_fd)
695{
696 info->iter = 2000;
697 info->map_fd = map_fd;
698 info->key[0].prefixlen = 16;
699 inet_pton(AF_INET, "192.168.0.0", &info->key[0].data);
700 info->key[1].prefixlen = 24;
701 inet_pton(AF_INET, "192.168.0.0", &info->key[1].data);
702 info->key[2].prefixlen = 24;
703 inet_pton(AF_INET, "192.168.128.0", &info->key[2].data);
704 info->key[3].prefixlen = 24;
705 inet_pton(AF_INET, "192.168.1.0", &info->key[3].data);
706}
707
708static void test_lpm_multi_thread(void)
709{
710 struct lpm_mt_test_info info[4];
711 size_t key_size, value_size;
712 pthread_t thread_id[4];
713 int i, map_fd;
714 void *ret;
715
716 /* create a trie */
717 value_size = sizeof(__u32);
718 key_size = sizeof(struct bpf_lpm_trie_key) + value_size;
719 map_fd = bpf_create_map(BPF_MAP_TYPE_LPM_TRIE, key_size, value_size,
720 100, BPF_F_NO_PREALLOC);
721
722 /* create 4 threads to test update, delete, lookup and get_next_key */
723 setup_lpm_mt_test_info(&info[0], map_fd);
724 for (i = 0; i < 4; i++) {
725 if (i != 0)
726 memcpy(&info[i], &info[0], sizeof(info[i]));
727 info[i].cmd = i;
728 assert(pthread_create(&thread_id[i], NULL, &lpm_test_command, &info[i]) == 0);
729 }
730
731 for (i = 0; i < 4; i++)
732 assert(pthread_join(thread_id[i], &ret) == 0 && ret == (void *)&info[i]);
733
734 close(map_fd);
735}
736
524int main(void) 737int main(void)
525{ 738{
526 struct rlimit limit = { RLIM_INFINITY, RLIM_INFINITY }; 739 struct rlimit limit = { RLIM_INFINITY, RLIM_INFINITY };
@@ -545,6 +758,10 @@ int main(void)
545 758
546 test_lpm_delete(); 759 test_lpm_delete();
547 760
761 test_lpm_get_next_key();
762
763 test_lpm_multi_thread();
764
548 printf("test_lpm: OK\n"); 765 printf("test_lpm: OK\n");
549 return 0; 766 return 0;
550} 767}
diff --git a/tools/testing/selftests/bpf/test_maps.c b/tools/testing/selftests/bpf/test_maps.c
index 040356ecc862..436c4c72414f 100644
--- a/tools/testing/selftests/bpf/test_maps.c
+++ b/tools/testing/selftests/bpf/test_maps.c
@@ -242,7 +242,7 @@ static void test_hashmap_percpu(int task, void *data)
242 242
243static void test_hashmap_walk(int task, void *data) 243static void test_hashmap_walk(int task, void *data)
244{ 244{
245 int fd, i, max_entries = 100000; 245 int fd, i, max_entries = 1000;
246 long long key, value, next_key; 246 long long key, value, next_key;
247 bool next_key_valid = true; 247 bool next_key_valid = true;
248 248
@@ -463,7 +463,7 @@ static void test_devmap(int task, void *data)
463#define SOCKMAP_VERDICT_PROG "./sockmap_verdict_prog.o" 463#define SOCKMAP_VERDICT_PROG "./sockmap_verdict_prog.o"
464static void test_sockmap(int tasks, void *data) 464static void test_sockmap(int tasks, void *data)
465{ 465{
466 int one = 1, map_fd_rx, map_fd_tx, map_fd_break, s, sc, rc; 466 int one = 1, map_fd_rx = 0, map_fd_tx = 0, map_fd_break, s, sc, rc;
467 struct bpf_map *bpf_map_rx, *bpf_map_tx, *bpf_map_break; 467 struct bpf_map *bpf_map_rx, *bpf_map_tx, *bpf_map_break;
468 int ports[] = {50200, 50201, 50202, 50204}; 468 int ports[] = {50200, 50201, 50202, 50204};
469 int err, i, fd, udp, sfd[6] = {0xdeadbeef}; 469 int err, i, fd, udp, sfd[6] = {0xdeadbeef};
@@ -868,9 +868,12 @@ static void test_sockmap(int tasks, void *data)
868 goto out_sockmap; 868 goto out_sockmap;
869 } 869 }
870 870
871 /* Test map close sockets */ 871 /* Test map close sockets and empty maps */
872 for (i = 0; i < 6; i++) 872 for (i = 0; i < 6; i++) {
873 bpf_map_delete_elem(map_fd_tx, &i);
874 bpf_map_delete_elem(map_fd_rx, &i);
873 close(sfd[i]); 875 close(sfd[i]);
876 }
874 close(fd); 877 close(fd);
875 close(map_fd_rx); 878 close(map_fd_rx);
876 bpf_object__close(obj); 879 bpf_object__close(obj);
@@ -881,8 +884,13 @@ out:
881 printf("Failed to create sockmap '%i:%s'!\n", i, strerror(errno)); 884 printf("Failed to create sockmap '%i:%s'!\n", i, strerror(errno));
882 exit(1); 885 exit(1);
883out_sockmap: 886out_sockmap:
884 for (i = 0; i < 6; i++) 887 for (i = 0; i < 6; i++) {
888 if (map_fd_tx)
889 bpf_map_delete_elem(map_fd_tx, &i);
890 if (map_fd_rx)
891 bpf_map_delete_elem(map_fd_rx, &i);
885 close(sfd[i]); 892 close(sfd[i]);
893 }
886 close(fd); 894 close(fd);
887 exit(1); 895 exit(1);
888} 896}
@@ -931,8 +939,12 @@ static void test_map_large(void)
931 close(fd); 939 close(fd);
932} 940}
933 941
934static void run_parallel(int tasks, void (*fn)(int task, void *data), 942#define run_parallel(N, FN, DATA) \
935 void *data) 943 printf("Fork %d tasks to '" #FN "'\n", N); \
944 __run_parallel(N, FN, DATA)
945
946static void __run_parallel(int tasks, void (*fn)(int task, void *data),
947 void *data)
936{ 948{
937 pid_t pid[tasks]; 949 pid_t pid[tasks];
938 int i; 950 int i;
@@ -972,7 +984,7 @@ static void test_map_stress(void)
972#define DO_UPDATE 1 984#define DO_UPDATE 1
973#define DO_DELETE 0 985#define DO_DELETE 0
974 986
975static void do_work(int fn, void *data) 987static void test_update_delete(int fn, void *data)
976{ 988{
977 int do_update = ((int *)data)[1]; 989 int do_update = ((int *)data)[1];
978 int fd = ((int *)data)[0]; 990 int fd = ((int *)data)[0];
@@ -1012,7 +1024,7 @@ static void test_map_parallel(void)
1012 */ 1024 */
1013 data[0] = fd; 1025 data[0] = fd;
1014 data[1] = DO_UPDATE; 1026 data[1] = DO_UPDATE;
1015 run_parallel(TASKS, do_work, data); 1027 run_parallel(TASKS, test_update_delete, data);
1016 1028
1017 /* Check that key=0 is already there. */ 1029 /* Check that key=0 is already there. */
1018 assert(bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST) == -1 && 1030 assert(bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST) == -1 &&
@@ -1035,7 +1047,7 @@ static void test_map_parallel(void)
1035 1047
1036 /* Now let's delete all elemenets in parallel. */ 1048 /* Now let's delete all elemenets in parallel. */
1037 data[1] = DO_DELETE; 1049 data[1] = DO_DELETE;
1038 run_parallel(TASKS, do_work, data); 1050 run_parallel(TASKS, test_update_delete, data);
1039 1051
1040 /* Nothing should be left. */ 1052 /* Nothing should be left. */
1041 key = -1; 1053 key = -1;
diff --git a/tools/testing/selftests/bpf/test_offload.py b/tools/testing/selftests/bpf/test_offload.py
new file mode 100755
index 000000000000..e78aad0a68bb
--- /dev/null
+++ b/tools/testing/selftests/bpf/test_offload.py
@@ -0,0 +1,1085 @@
1#!/usr/bin/python3
2
3# Copyright (C) 2017 Netronome Systems, Inc.
4#
5# This software is licensed under the GNU General License Version 2,
6# June 1991 as shown in the file COPYING in the top-level directory of this
7# source tree.
8#
9# THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS"
10# WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING,
11# BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
12# FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE
13# OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME
14# THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
15
16from datetime import datetime
17import argparse
18import json
19import os
20import pprint
21import random
22import string
23import struct
24import subprocess
25import time
26
27logfile = None
28log_level = 1
29skip_extack = False
30bpf_test_dir = os.path.dirname(os.path.realpath(__file__))
31pp = pprint.PrettyPrinter()
32devs = [] # devices we created for clean up
33files = [] # files to be removed
34netns = [] # net namespaces to be removed
35
36def log_get_sec(level=0):
37 return "*" * (log_level + level)
38
39def log_level_inc(add=1):
40 global log_level
41 log_level += add
42
43def log_level_dec(sub=1):
44 global log_level
45 log_level -= sub
46
47def log_level_set(level):
48 global log_level
49 log_level = level
50
51def log(header, data, level=None):
52 """
53 Output to an optional log.
54 """
55 if logfile is None:
56 return
57 if level is not None:
58 log_level_set(level)
59
60 if not isinstance(data, str):
61 data = pp.pformat(data)
62
63 if len(header):
64 logfile.write("\n" + log_get_sec() + " ")
65 logfile.write(header)
66 if len(header) and len(data.strip()):
67 logfile.write("\n")
68 logfile.write(data)
69
70def skip(cond, msg):
71 if not cond:
72 return
73 print("SKIP: " + msg)
74 log("SKIP: " + msg, "", level=1)
75 os.sys.exit(0)
76
77def fail(cond, msg):
78 if not cond:
79 return
80 print("FAIL: " + msg)
81 log("FAIL: " + msg, "", level=1)
82 os.sys.exit(1)
83
84def start_test(msg):
85 log(msg, "", level=1)
86 log_level_inc()
87 print(msg)
88
89def cmd(cmd, shell=True, include_stderr=False, background=False, fail=True):
90 """
91 Run a command in subprocess and return tuple of (retval, stdout);
92 optionally return stderr as well as third value.
93 """
94 proc = subprocess.Popen(cmd, shell=shell, stdout=subprocess.PIPE,
95 stderr=subprocess.PIPE)
96 if background:
97 msg = "%s START: %s" % (log_get_sec(1),
98 datetime.now().strftime("%H:%M:%S.%f"))
99 log("BKG " + proc.args, msg)
100 return proc
101
102 return cmd_result(proc, include_stderr=include_stderr, fail=fail)
103
104def cmd_result(proc, include_stderr=False, fail=False):
105 stdout, stderr = proc.communicate()
106 stdout = stdout.decode("utf-8")
107 stderr = stderr.decode("utf-8")
108 proc.stdout.close()
109 proc.stderr.close()
110
111 stderr = "\n" + stderr
112 if stderr[-1] == "\n":
113 stderr = stderr[:-1]
114
115 sec = log_get_sec(1)
116 log("CMD " + proc.args,
117 "RETCODE: %d\n%s STDOUT:\n%s%s STDERR:%s\n%s END: %s" %
118 (proc.returncode, sec, stdout, sec, stderr,
119 sec, datetime.now().strftime("%H:%M:%S.%f")))
120
121 if proc.returncode != 0 and fail:
122 if len(stderr) > 0 and stderr[-1] == "\n":
123 stderr = stderr[:-1]
124 raise Exception("Command failed: %s\n%s" % (proc.args, stderr))
125
126 if include_stderr:
127 return proc.returncode, stdout, stderr
128 else:
129 return proc.returncode, stdout
130
131def rm(f):
132 cmd("rm -f %s" % (f))
133 if f in files:
134 files.remove(f)
135
136def tool(name, args, flags, JSON=True, ns="", fail=True, include_stderr=False):
137 params = ""
138 if JSON:
139 params += "%s " % (flags["json"])
140
141 if ns != "":
142 ns = "ip netns exec %s " % (ns)
143
144 if include_stderr:
145 ret, stdout, stderr = cmd(ns + name + " " + params + args,
146 fail=fail, include_stderr=True)
147 else:
148 ret, stdout = cmd(ns + name + " " + params + args,
149 fail=fail, include_stderr=False)
150
151 if JSON and len(stdout.strip()) != 0:
152 out = json.loads(stdout)
153 else:
154 out = stdout
155
156 if include_stderr:
157 return ret, out, stderr
158 else:
159 return ret, out
160
161def bpftool(args, JSON=True, ns="", fail=True):
162 return tool("bpftool", args, {"json":"-p"}, JSON=JSON, ns=ns, fail=fail)
163
164def bpftool_prog_list(expected=None, ns=""):
165 _, progs = bpftool("prog show", JSON=True, ns=ns, fail=True)
166 if expected is not None:
167 if len(progs) != expected:
168 fail(True, "%d BPF programs loaded, expected %d" %
169 (len(progs), expected))
170 return progs
171
172def bpftool_map_list(expected=None, ns=""):
173 _, maps = bpftool("map show", JSON=True, ns=ns, fail=True)
174 if expected is not None:
175 if len(maps) != expected:
176 fail(True, "%d BPF maps loaded, expected %d" %
177 (len(maps), expected))
178 return maps
179
180def bpftool_prog_list_wait(expected=0, n_retry=20):
181 for i in range(n_retry):
182 nprogs = len(bpftool_prog_list())
183 if nprogs == expected:
184 return
185 time.sleep(0.05)
186 raise Exception("Time out waiting for program counts to stabilize want %d, have %d" % (expected, nprogs))
187
188def bpftool_map_list_wait(expected=0, n_retry=20):
189 for i in range(n_retry):
190 nmaps = len(bpftool_map_list())
191 if nmaps == expected:
192 return
193 time.sleep(0.05)
194 raise Exception("Time out waiting for map counts to stabilize want %d, have %d" % (expected, nmaps))
195
196def ip(args, force=False, JSON=True, ns="", fail=True, include_stderr=False):
197 if force:
198 args = "-force " + args
199 return tool("ip", args, {"json":"-j"}, JSON=JSON, ns=ns,
200 fail=fail, include_stderr=include_stderr)
201
202def tc(args, JSON=True, ns="", fail=True, include_stderr=False):
203 return tool("tc", args, {"json":"-p"}, JSON=JSON, ns=ns,
204 fail=fail, include_stderr=include_stderr)
205
206def ethtool(dev, opt, args, fail=True):
207 return cmd("ethtool %s %s %s" % (opt, dev["ifname"], args), fail=fail)
208
209def bpf_obj(name, sec=".text", path=bpf_test_dir,):
210 return "obj %s sec %s" % (os.path.join(path, name), sec)
211
212def bpf_pinned(name):
213 return "pinned %s" % (name)
214
215def bpf_bytecode(bytecode):
216 return "bytecode \"%s\"" % (bytecode)
217
218def mknetns(n_retry=10):
219 for i in range(n_retry):
220 name = ''.join([random.choice(string.ascii_letters) for i in range(8)])
221 ret, _ = ip("netns add %s" % (name), fail=False)
222 if ret == 0:
223 netns.append(name)
224 return name
225 return None
226
227def int2str(fmt, val):
228 ret = []
229 for b in struct.pack(fmt, val):
230 ret.append(int(b))
231 return " ".join(map(lambda x: str(x), ret))
232
233def str2int(strtab):
234 inttab = []
235 for i in strtab:
236 inttab.append(int(i, 16))
237 ba = bytearray(inttab)
238 if len(strtab) == 4:
239 fmt = "I"
240 elif len(strtab) == 8:
241 fmt = "Q"
242 else:
243 raise Exception("String array of len %d can't be unpacked to an int" %
244 (len(strtab)))
245 return struct.unpack(fmt, ba)[0]
246
247class DebugfsDir:
248 """
249 Class for accessing DebugFS directories as a dictionary.
250 """
251
252 def __init__(self, path):
253 self.path = path
254 self._dict = self._debugfs_dir_read(path)
255
256 def __len__(self):
257 return len(self._dict.keys())
258
259 def __getitem__(self, key):
260 if type(key) is int:
261 key = list(self._dict.keys())[key]
262 return self._dict[key]
263
264 def __setitem__(self, key, value):
265 log("DebugFS set %s = %s" % (key, value), "")
266 log_level_inc()
267
268 cmd("echo '%s' > %s/%s" % (value, self.path, key))
269 log_level_dec()
270
271 _, out = cmd('cat %s/%s' % (self.path, key))
272 self._dict[key] = out.strip()
273
274 def _debugfs_dir_read(self, path):
275 dfs = {}
276
277 log("DebugFS state for %s" % (path), "")
278 log_level_inc(add=2)
279
280 _, out = cmd('ls ' + path)
281 for f in out.split():
282 p = os.path.join(path, f)
283 if os.path.isfile(p):
284 _, out = cmd('cat %s/%s' % (path, f))
285 dfs[f] = out.strip()
286 elif os.path.isdir(p):
287 dfs[f] = DebugfsDir(p)
288 else:
289 raise Exception("%s is neither file nor directory" % (p))
290
291 log_level_dec()
292 log("DebugFS state", dfs)
293 log_level_dec()
294
295 return dfs
296
297class NetdevSim:
298 """
299 Class for netdevsim netdevice and its attributes.
300 """
301
302 def __init__(self):
303 self.dev = self._netdevsim_create()
304 devs.append(self)
305
306 self.ns = ""
307
308 self.dfs_dir = '/sys/kernel/debug/netdevsim/%s' % (self.dev['ifname'])
309 self.dfs_refresh()
310
311 def __getitem__(self, key):
312 return self.dev[key]
313
314 def _netdevsim_create(self):
315 _, old = ip("link show")
316 ip("link add sim%d type netdevsim")
317 _, new = ip("link show")
318
319 for dev in new:
320 f = filter(lambda x: x["ifname"] == dev["ifname"], old)
321 if len(list(f)) == 0:
322 return dev
323
324 raise Exception("failed to create netdevsim device")
325
326 def remove(self):
327 devs.remove(self)
328 ip("link del dev %s" % (self.dev["ifname"]), ns=self.ns)
329
330 def dfs_refresh(self):
331 self.dfs = DebugfsDir(self.dfs_dir)
332 return self.dfs
333
334 def dfs_num_bound_progs(self):
335 path = os.path.join(self.dfs_dir, "bpf_bound_progs")
336 _, progs = cmd('ls %s' % (path))
337 return len(progs.split())
338
339 def dfs_get_bound_progs(self, expected):
340 progs = DebugfsDir(os.path.join(self.dfs_dir, "bpf_bound_progs"))
341 if expected is not None:
342 if len(progs) != expected:
343 fail(True, "%d BPF programs bound, expected %d" %
344 (len(progs), expected))
345 return progs
346
347 def wait_for_flush(self, bound=0, total=0, n_retry=20):
348 for i in range(n_retry):
349 nbound = self.dfs_num_bound_progs()
350 nprogs = len(bpftool_prog_list())
351 if nbound == bound and nprogs == total:
352 return
353 time.sleep(0.05)
354 raise Exception("Time out waiting for program counts to stabilize want %d/%d, have %d bound, %d loaded" % (bound, total, nbound, nprogs))
355
356 def set_ns(self, ns):
357 name = "1" if ns == "" else ns
358 ip("link set dev %s netns %s" % (self.dev["ifname"], name), ns=self.ns)
359 self.ns = ns
360
361 def set_mtu(self, mtu, fail=True):
362 return ip("link set dev %s mtu %d" % (self.dev["ifname"], mtu),
363 fail=fail)
364
365 def set_xdp(self, bpf, mode, force=False, JSON=True, verbose=False,
366 fail=True, include_stderr=False):
367 if verbose:
368 bpf += " verbose"
369 return ip("link set dev %s xdp%s %s" % (self.dev["ifname"], mode, bpf),
370 force=force, JSON=JSON,
371 fail=fail, include_stderr=include_stderr)
372
373 def unset_xdp(self, mode, force=False, JSON=True,
374 fail=True, include_stderr=False):
375 return ip("link set dev %s xdp%s off" % (self.dev["ifname"], mode),
376 force=force, JSON=JSON,
377 fail=fail, include_stderr=include_stderr)
378
379 def ip_link_show(self, xdp):
380 _, link = ip("link show dev %s" % (self['ifname']))
381 if len(link) > 1:
382 raise Exception("Multiple objects on ip link show")
383 if len(link) < 1:
384 return {}
385 fail(xdp != "xdp" in link,
386 "XDP program not reporting in iplink (reported %s, expected %s)" %
387 ("xdp" in link, xdp))
388 return link[0]
389
390 def tc_add_ingress(self):
391 tc("qdisc add dev %s ingress" % (self['ifname']))
392
393 def tc_del_ingress(self):
394 tc("qdisc del dev %s ingress" % (self['ifname']))
395
396 def tc_flush_filters(self, bound=0, total=0):
397 self.tc_del_ingress()
398 self.tc_add_ingress()
399 self.wait_for_flush(bound=bound, total=total)
400
401 def tc_show_ingress(self, expected=None):
402 # No JSON support, oh well...
403 flags = ["skip_sw", "skip_hw", "in_hw"]
404 named = ["protocol", "pref", "chain", "handle", "id", "tag"]
405
406 args = "-s filter show dev %s ingress" % (self['ifname'])
407 _, out = tc(args, JSON=False)
408
409 filters = []
410 lines = out.split('\n')
411 for line in lines:
412 words = line.split()
413 if "handle" not in words:
414 continue
415 fltr = {}
416 for flag in flags:
417 fltr[flag] = flag in words
418 for name in named:
419 try:
420 idx = words.index(name)
421 fltr[name] = words[idx + 1]
422 except ValueError:
423 pass
424 filters.append(fltr)
425
426 if expected is not None:
427 fail(len(filters) != expected,
428 "%d ingress filters loaded, expected %d" %
429 (len(filters), expected))
430 return filters
431
432 def cls_filter_op(self, op, qdisc="ingress", prio=None, handle=None,
433 chain=None, cls="", params="",
434 fail=True, include_stderr=False):
435 spec = ""
436 if prio is not None:
437 spec += " prio %d" % (prio)
438 if handle:
439 spec += " handle %s" % (handle)
440 if chain is not None:
441 spec += " chain %d" % (chain)
442
443 return tc("filter {op} dev {dev} {qdisc} {spec} {cls} {params}"\
444 .format(op=op, dev=self['ifname'], qdisc=qdisc, spec=spec,
445 cls=cls, params=params),
446 fail=fail, include_stderr=include_stderr)
447
448 def cls_bpf_add_filter(self, bpf, op="add", prio=None, handle=None,
449 chain=None, da=False, verbose=False,
450 skip_sw=False, skip_hw=False,
451 fail=True, include_stderr=False):
452 cls = "bpf " + bpf
453
454 params = ""
455 if da:
456 params += " da"
457 if verbose:
458 params += " verbose"
459 if skip_sw:
460 params += " skip_sw"
461 if skip_hw:
462 params += " skip_hw"
463
464 return self.cls_filter_op(op=op, prio=prio, handle=handle, cls=cls,
465 chain=chain, params=params,
466 fail=fail, include_stderr=include_stderr)
467
468 def set_ethtool_tc_offloads(self, enable, fail=True):
469 args = "hw-tc-offload %s" % ("on" if enable else "off")
470 return ethtool(self, "-K", args, fail=fail)
471
472################################################################################
473def clean_up():
474 global files, netns, devs
475
476 for dev in devs:
477 dev.remove()
478 for f in files:
479 cmd("rm -f %s" % (f))
480 for ns in netns:
481 cmd("ip netns delete %s" % (ns))
482 files = []
483 netns = []
484
485def pin_prog(file_name, idx=0):
486 progs = bpftool_prog_list(expected=(idx + 1))
487 prog = progs[idx]
488 bpftool("prog pin id %d %s" % (prog["id"], file_name))
489 files.append(file_name)
490
491 return file_name, bpf_pinned(file_name)
492
493def pin_map(file_name, idx=0, expected=1):
494 maps = bpftool_map_list(expected=expected)
495 m = maps[idx]
496 bpftool("map pin id %d %s" % (m["id"], file_name))
497 files.append(file_name)
498
499 return file_name, bpf_pinned(file_name)
500
501def check_dev_info_removed(prog_file=None, map_file=None):
502 bpftool_prog_list(expected=0)
503 ret, err = bpftool("prog show pin %s" % (prog_file), fail=False)
504 fail(ret == 0, "Showing prog with removed device did not fail")
505 fail(err["error"].find("No such device") == -1,
506 "Showing prog with removed device expected ENODEV, error is %s" %
507 (err["error"]))
508
509 bpftool_map_list(expected=0)
510 ret, err = bpftool("map show pin %s" % (map_file), fail=False)
511 fail(ret == 0, "Showing map with removed device did not fail")
512 fail(err["error"].find("No such device") == -1,
513 "Showing map with removed device expected ENODEV, error is %s" %
514 (err["error"]))
515
516def check_dev_info(other_ns, ns, prog_file=None, map_file=None, removed=False):
517 progs = bpftool_prog_list(expected=1, ns=ns)
518 prog = progs[0]
519
520 fail("dev" not in prog.keys(), "Device parameters not reported")
521 dev = prog["dev"]
522 fail("ifindex" not in dev.keys(), "Device parameters not reported")
523 fail("ns_dev" not in dev.keys(), "Device parameters not reported")
524 fail("ns_inode" not in dev.keys(), "Device parameters not reported")
525
526 if not other_ns:
527 fail("ifname" not in dev.keys(), "Ifname not reported")
528 fail(dev["ifname"] != sim["ifname"],
529 "Ifname incorrect %s vs %s" % (dev["ifname"], sim["ifname"]))
530 else:
531 fail("ifname" in dev.keys(), "Ifname is reported for other ns")
532
533 maps = bpftool_map_list(expected=2, ns=ns)
534 for m in maps:
535 fail("dev" not in m.keys(), "Device parameters not reported")
536 fail(dev != m["dev"], "Map's device different than program's")
537
538def check_extack(output, reference, args):
539 if skip_extack:
540 return
541 lines = output.split("\n")
542 comp = len(lines) >= 2 and lines[1] == reference
543 fail(not comp, "Missing or incorrect netlink extack message")
544
545def check_extack_nsim(output, reference, args):
546 check_extack(output, "Error: netdevsim: " + reference, args)
547
548def check_no_extack(res, needle):
549 fail((res[1] + res[2]).count(needle) or (res[1] + res[2]).count("Warning:"),
550 "Found '%s' in command output, leaky extack?" % (needle))
551
552def check_verifier_log(output, reference):
553 lines = output.split("\n")
554 for l in reversed(lines):
555 if l == reference:
556 return
557 fail(True, "Missing or incorrect message from netdevsim in verifier log")
558
559def test_spurios_extack(sim, obj, skip_hw, needle):
560 res = sim.cls_bpf_add_filter(obj, prio=1, handle=1, skip_hw=skip_hw,
561 include_stderr=True)
562 check_no_extack(res, needle)
563 res = sim.cls_bpf_add_filter(obj, op="replace", prio=1, handle=1,
564 skip_hw=skip_hw, include_stderr=True)
565 check_no_extack(res, needle)
566 res = sim.cls_filter_op(op="delete", prio=1, handle=1, cls="bpf",
567 include_stderr=True)
568 check_no_extack(res, needle)
569
570
571# Parse command line
572parser = argparse.ArgumentParser()
573parser.add_argument("--log", help="output verbose log to given file")
574args = parser.parse_args()
575if args.log:
576 logfile = open(args.log, 'w+')
577 logfile.write("# -*-Org-*-")
578
579log("Prepare...", "", level=1)
580log_level_inc()
581
582# Check permissions
583skip(os.getuid() != 0, "test must be run as root")
584
585# Check tools
586ret, progs = bpftool("prog", fail=False)
587skip(ret != 0, "bpftool not installed")
588# Check no BPF programs are loaded
589skip(len(progs) != 0, "BPF programs already loaded on the system")
590
591# Check netdevsim
592ret, out = cmd("modprobe netdevsim", fail=False)
593skip(ret != 0, "netdevsim module could not be loaded")
594
595# Check debugfs
596_, out = cmd("mount")
597if out.find("/sys/kernel/debug type debugfs") == -1:
598 cmd("mount -t debugfs none /sys/kernel/debug")
599
600# Check samples are compiled
601samples = ["sample_ret0.o", "sample_map_ret0.o"]
602for s in samples:
603 ret, out = cmd("ls %s/%s" % (bpf_test_dir, s), fail=False)
604 skip(ret != 0, "sample %s/%s not found, please compile it" %
605 (bpf_test_dir, s))
606
607# Check if iproute2 is built with libmnl (needed by extack support)
608_, _, err = cmd("tc qdisc delete dev lo handle 0",
609 fail=False, include_stderr=True)
610if err.find("Error: Failed to find qdisc with specified handle.") == -1:
611 print("Warning: no extack message in iproute2 output, libmnl missing?")
612 log("Warning: no extack message in iproute2 output, libmnl missing?", "")
613 skip_extack = True
614
615# Check if net namespaces seem to work
616ns = mknetns()
617skip(ns is None, "Could not create a net namespace")
618cmd("ip netns delete %s" % (ns))
619netns = []
620
621try:
622 obj = bpf_obj("sample_ret0.o")
623 bytecode = bpf_bytecode("1,6 0 0 4294967295,")
624
625 start_test("Test destruction of generic XDP...")
626 sim = NetdevSim()
627 sim.set_xdp(obj, "generic")
628 sim.remove()
629 bpftool_prog_list_wait(expected=0)
630
631 sim = NetdevSim()
632 sim.tc_add_ingress()
633
634 start_test("Test TC non-offloaded...")
635 ret, _ = sim.cls_bpf_add_filter(obj, skip_hw=True, fail=False)
636 fail(ret != 0, "Software TC filter did not load")
637
638 start_test("Test TC non-offloaded isn't getting bound...")
639 ret, _ = sim.cls_bpf_add_filter(obj, fail=False)
640 fail(ret != 0, "Software TC filter did not load")
641 sim.dfs_get_bound_progs(expected=0)
642
643 sim.tc_flush_filters()
644
645 start_test("Test TC offloads are off by default...")
646 ret, _, err = sim.cls_bpf_add_filter(obj, skip_sw=True,
647 fail=False, include_stderr=True)
648 fail(ret == 0, "TC filter loaded without enabling TC offloads")
649 check_extack(err, "Error: TC offload is disabled on net device.", args)
650 sim.wait_for_flush()
651
652 sim.set_ethtool_tc_offloads(True)
653 sim.dfs["bpf_tc_non_bound_accept"] = "Y"
654
655 start_test("Test TC offload by default...")
656 ret, _ = sim.cls_bpf_add_filter(obj, fail=False)
657 fail(ret != 0, "Software TC filter did not load")
658 sim.dfs_get_bound_progs(expected=0)
659 ingress = sim.tc_show_ingress(expected=1)
660 fltr = ingress[0]
661 fail(not fltr["in_hw"], "Filter not offloaded by default")
662
663 sim.tc_flush_filters()
664
665 start_test("Test TC cBPF bytcode tries offload by default...")
666 ret, _ = sim.cls_bpf_add_filter(bytecode, fail=False)
667 fail(ret != 0, "Software TC filter did not load")
668 sim.dfs_get_bound_progs(expected=0)
669 ingress = sim.tc_show_ingress(expected=1)
670 fltr = ingress[0]
671 fail(not fltr["in_hw"], "Bytecode not offloaded by default")
672
673 sim.tc_flush_filters()
674 sim.dfs["bpf_tc_non_bound_accept"] = "N"
675
676 start_test("Test TC cBPF unbound bytecode doesn't offload...")
677 ret, _, err = sim.cls_bpf_add_filter(bytecode, skip_sw=True,
678 fail=False, include_stderr=True)
679 fail(ret == 0, "TC bytecode loaded for offload")
680 check_extack_nsim(err, "netdevsim configured to reject unbound programs.",
681 args)
682 sim.wait_for_flush()
683
684 start_test("Test non-0 chain offload...")
685 ret, _, err = sim.cls_bpf_add_filter(obj, chain=1, prio=1, handle=1,
686 skip_sw=True,
687 fail=False, include_stderr=True)
688 fail(ret == 0, "Offloaded a filter to chain other than 0")
689 check_extack(err, "Error: Driver supports only offload of chain 0.", args)
690 sim.tc_flush_filters()
691
692 start_test("Test TC replace...")
693 sim.cls_bpf_add_filter(obj, prio=1, handle=1)
694 sim.cls_bpf_add_filter(obj, op="replace", prio=1, handle=1)
695 sim.cls_filter_op(op="delete", prio=1, handle=1, cls="bpf")
696
697 sim.cls_bpf_add_filter(obj, prio=1, handle=1, skip_sw=True)
698 sim.cls_bpf_add_filter(obj, op="replace", prio=1, handle=1, skip_sw=True)
699 sim.cls_filter_op(op="delete", prio=1, handle=1, cls="bpf")
700
701 sim.cls_bpf_add_filter(obj, prio=1, handle=1, skip_hw=True)
702 sim.cls_bpf_add_filter(obj, op="replace", prio=1, handle=1, skip_hw=True)
703 sim.cls_filter_op(op="delete", prio=1, handle=1, cls="bpf")
704
705 start_test("Test TC replace bad flags...")
706 for i in range(3):
707 for j in range(3):
708 ret, _ = sim.cls_bpf_add_filter(obj, op="replace", prio=1, handle=1,
709 skip_sw=(j == 1), skip_hw=(j == 2),
710 fail=False)
711 fail(bool(ret) != bool(j),
712 "Software TC incorrect load in replace test, iteration %d" %
713 (j))
714 sim.cls_filter_op(op="delete", prio=1, handle=1, cls="bpf")
715
716 start_test("Test spurious extack from the driver...")
717 test_spurios_extack(sim, obj, False, "netdevsim")
718 test_spurios_extack(sim, obj, True, "netdevsim")
719
720 sim.set_ethtool_tc_offloads(False)
721
722 test_spurios_extack(sim, obj, False, "TC offload is disabled")
723 test_spurios_extack(sim, obj, True, "TC offload is disabled")
724
725 sim.set_ethtool_tc_offloads(True)
726
727 sim.tc_flush_filters()
728
729 start_test("Test TC offloads work...")
730 ret, _, err = sim.cls_bpf_add_filter(obj, verbose=True, skip_sw=True,
731 fail=False, include_stderr=True)
732 fail(ret != 0, "TC filter did not load with TC offloads enabled")
733 check_verifier_log(err, "[netdevsim] Hello from netdevsim!")
734
735 start_test("Test TC offload basics...")
736 dfs = sim.dfs_get_bound_progs(expected=1)
737 progs = bpftool_prog_list(expected=1)
738 ingress = sim.tc_show_ingress(expected=1)
739
740 dprog = dfs[0]
741 prog = progs[0]
742 fltr = ingress[0]
743 fail(fltr["skip_hw"], "TC does reports 'skip_hw' on offloaded filter")
744 fail(not fltr["in_hw"], "TC does not report 'in_hw' for offloaded filter")
745 fail(not fltr["skip_sw"], "TC does not report 'skip_sw' back")
746
747 start_test("Test TC offload is device-bound...")
748 fail(str(prog["id"]) != fltr["id"], "Program IDs don't match")
749 fail(prog["tag"] != fltr["tag"], "Program tags don't match")
750 fail(fltr["id"] != dprog["id"], "Program IDs don't match")
751 fail(dprog["state"] != "xlated", "Offloaded program state not translated")
752 fail(dprog["loaded"] != "Y", "Offloaded program is not loaded")
753
754 start_test("Test disabling TC offloads is rejected while filters installed...")
755 ret, _ = sim.set_ethtool_tc_offloads(False, fail=False)
756 fail(ret == 0, "Driver should refuse to disable TC offloads with filters installed...")
757
758 start_test("Test qdisc removal frees things...")
759 sim.tc_flush_filters()
760 sim.tc_show_ingress(expected=0)
761
762 start_test("Test disabling TC offloads is OK without filters...")
763 ret, _ = sim.set_ethtool_tc_offloads(False, fail=False)
764 fail(ret != 0,
765 "Driver refused to disable TC offloads without filters installed...")
766
767 sim.set_ethtool_tc_offloads(True)
768
769 start_test("Test destroying device gets rid of TC filters...")
770 sim.cls_bpf_add_filter(obj, skip_sw=True)
771 sim.remove()
772 bpftool_prog_list_wait(expected=0)
773
774 sim = NetdevSim()
775 sim.set_ethtool_tc_offloads(True)
776
777 start_test("Test destroying device gets rid of XDP...")
778 sim.set_xdp(obj, "offload")
779 sim.remove()
780 bpftool_prog_list_wait(expected=0)
781
782 sim = NetdevSim()
783 sim.set_ethtool_tc_offloads(True)
784
785 start_test("Test XDP prog reporting...")
786 sim.set_xdp(obj, "drv")
787 ipl = sim.ip_link_show(xdp=True)
788 progs = bpftool_prog_list(expected=1)
789 fail(ipl["xdp"]["prog"]["id"] != progs[0]["id"],
790 "Loaded program has wrong ID")
791
792 start_test("Test XDP prog replace without force...")
793 ret, _ = sim.set_xdp(obj, "drv", fail=False)
794 fail(ret == 0, "Replaced XDP program without -force")
795 sim.wait_for_flush(total=1)
796
797 start_test("Test XDP prog replace with force...")
798 ret, _ = sim.set_xdp(obj, "drv", force=True, fail=False)
799 fail(ret != 0, "Could not replace XDP program with -force")
800 bpftool_prog_list_wait(expected=1)
801 ipl = sim.ip_link_show(xdp=True)
802 progs = bpftool_prog_list(expected=1)
803 fail(ipl["xdp"]["prog"]["id"] != progs[0]["id"],
804 "Loaded program has wrong ID")
805 fail("dev" in progs[0].keys(),
806 "Device parameters reported for non-offloaded program")
807
808 start_test("Test XDP prog replace with bad flags...")
809 ret, _, err = sim.set_xdp(obj, "offload", force=True,
810 fail=False, include_stderr=True)
811 fail(ret == 0, "Replaced XDP program with a program in different mode")
812 check_extack_nsim(err, "program loaded with different flags.", args)
813 ret, _, err = sim.set_xdp(obj, "", force=True,
814 fail=False, include_stderr=True)
815 fail(ret == 0, "Replaced XDP program with a program in different mode")
816 check_extack_nsim(err, "program loaded with different flags.", args)
817
818 start_test("Test XDP prog remove with bad flags...")
819 ret, _, err = sim.unset_xdp("offload", force=True,
820 fail=False, include_stderr=True)
821 fail(ret == 0, "Removed program with a bad mode mode")
822 check_extack_nsim(err, "program loaded with different flags.", args)
823 ret, _, err = sim.unset_xdp("", force=True,
824 fail=False, include_stderr=True)
825 fail(ret == 0, "Removed program with a bad mode mode")
826 check_extack_nsim(err, "program loaded with different flags.", args)
827
828 start_test("Test MTU restrictions...")
829 ret, _ = sim.set_mtu(9000, fail=False)
830 fail(ret == 0,
831 "Driver should refuse to increase MTU to 9000 with XDP loaded...")
832 sim.unset_xdp("drv")
833 bpftool_prog_list_wait(expected=0)
834 sim.set_mtu(9000)
835 ret, _, err = sim.set_xdp(obj, "drv", fail=False, include_stderr=True)
836 fail(ret == 0, "Driver should refuse to load program with MTU of 9000...")
837 check_extack_nsim(err, "MTU too large w/ XDP enabled.", args)
838 sim.set_mtu(1500)
839
840 sim.wait_for_flush()
841 start_test("Test XDP offload...")
842 _, _, err = sim.set_xdp(obj, "offload", verbose=True, include_stderr=True)
843 ipl = sim.ip_link_show(xdp=True)
844 link_xdp = ipl["xdp"]["prog"]
845 progs = bpftool_prog_list(expected=1)
846 prog = progs[0]
847 fail(link_xdp["id"] != prog["id"], "Loaded program has wrong ID")
848 check_verifier_log(err, "[netdevsim] Hello from netdevsim!")
849
850 start_test("Test XDP offload is device bound...")
851 dfs = sim.dfs_get_bound_progs(expected=1)
852 dprog = dfs[0]
853
854 fail(prog["id"] != link_xdp["id"], "Program IDs don't match")
855 fail(prog["tag"] != link_xdp["tag"], "Program tags don't match")
856 fail(str(link_xdp["id"]) != dprog["id"], "Program IDs don't match")
857 fail(dprog["state"] != "xlated", "Offloaded program state not translated")
858 fail(dprog["loaded"] != "Y", "Offloaded program is not loaded")
859
860 start_test("Test removing XDP program many times...")
861 sim.unset_xdp("offload")
862 sim.unset_xdp("offload")
863 sim.unset_xdp("drv")
864 sim.unset_xdp("drv")
865 sim.unset_xdp("")
866 sim.unset_xdp("")
867 bpftool_prog_list_wait(expected=0)
868
869 start_test("Test attempt to use a program for a wrong device...")
870 sim2 = NetdevSim()
871 sim2.set_xdp(obj, "offload")
872 pin_file, pinned = pin_prog("/sys/fs/bpf/tmp")
873
874 ret, _, err = sim.set_xdp(pinned, "offload",
875 fail=False, include_stderr=True)
876 fail(ret == 0, "Pinned program loaded for a different device accepted")
877 check_extack_nsim(err, "program bound to different dev.", args)
878 sim2.remove()
879 ret, _, err = sim.set_xdp(pinned, "offload",
880 fail=False, include_stderr=True)
881 fail(ret == 0, "Pinned program loaded for a removed device accepted")
882 check_extack_nsim(err, "xdpoffload of non-bound program.", args)
883 rm(pin_file)
884 bpftool_prog_list_wait(expected=0)
885
886 start_test("Test mixing of TC and XDP...")
887 sim.tc_add_ingress()
888 sim.set_xdp(obj, "offload")
889 ret, _, err = sim.cls_bpf_add_filter(obj, skip_sw=True,
890 fail=False, include_stderr=True)
891 fail(ret == 0, "Loading TC when XDP active should fail")
892 check_extack_nsim(err, "driver and netdev offload states mismatch.", args)
893 sim.unset_xdp("offload")
894 sim.wait_for_flush()
895
896 sim.cls_bpf_add_filter(obj, skip_sw=True)
897 ret, _, err = sim.set_xdp(obj, "offload", fail=False, include_stderr=True)
898 fail(ret == 0, "Loading XDP when TC active should fail")
899 check_extack_nsim(err, "TC program is already loaded.", args)
900
901 start_test("Test binding TC from pinned...")
902 pin_file, pinned = pin_prog("/sys/fs/bpf/tmp")
903 sim.tc_flush_filters(bound=1, total=1)
904 sim.cls_bpf_add_filter(pinned, da=True, skip_sw=True)
905 sim.tc_flush_filters(bound=1, total=1)
906
907 start_test("Test binding XDP from pinned...")
908 sim.set_xdp(obj, "offload")
909 pin_file, pinned = pin_prog("/sys/fs/bpf/tmp2", idx=1)
910
911 sim.set_xdp(pinned, "offload", force=True)
912 sim.unset_xdp("offload")
913 sim.set_xdp(pinned, "offload", force=True)
914 sim.unset_xdp("offload")
915
916 start_test("Test offload of wrong type fails...")
917 ret, _ = sim.cls_bpf_add_filter(pinned, da=True, skip_sw=True, fail=False)
918 fail(ret == 0, "Managed to attach XDP program to TC")
919
920 start_test("Test asking for TC offload of two filters...")
921 sim.cls_bpf_add_filter(obj, da=True, skip_sw=True)
922 ret, _, err = sim.cls_bpf_add_filter(obj, da=True, skip_sw=True,
923 fail=False, include_stderr=True)
924 fail(ret == 0, "Managed to offload two TC filters at the same time")
925 check_extack_nsim(err, "driver and netdev offload states mismatch.", args)
926
927 sim.tc_flush_filters(bound=2, total=2)
928
929 start_test("Test if netdev removal waits for translation...")
930 delay_msec = 500
931 sim.dfs["bpf_bind_verifier_delay"] = delay_msec
932 start = time.time()
933 cmd_line = "tc filter add dev %s ingress bpf %s da skip_sw" % \
934 (sim['ifname'], obj)
935 tc_proc = cmd(cmd_line, background=True, fail=False)
936 # Wait for the verifier to start
937 while sim.dfs_num_bound_progs() <= 2:
938 pass
939 sim.remove()
940 end = time.time()
941 ret, _ = cmd_result(tc_proc, fail=False)
942 time_diff = end - start
943 log("Time", "start:\t%s\nend:\t%s\ndiff:\t%s" % (start, end, time_diff))
944
945 fail(ret == 0, "Managed to load TC filter on a unregistering device")
946 delay_sec = delay_msec * 0.001
947 fail(time_diff < delay_sec, "Removal process took %s, expected %s" %
948 (time_diff, delay_sec))
949
950 # Remove all pinned files and reinstantiate the netdev
951 clean_up()
952 bpftool_prog_list_wait(expected=0)
953
954 sim = NetdevSim()
955 map_obj = bpf_obj("sample_map_ret0.o")
956 start_test("Test loading program with maps...")
957 sim.set_xdp(map_obj, "offload", JSON=False) # map fixup msg breaks JSON
958
959 start_test("Test bpftool bound info reporting (own ns)...")
960 check_dev_info(False, "")
961
962 start_test("Test bpftool bound info reporting (other ns)...")
963 ns = mknetns()
964 sim.set_ns(ns)
965 check_dev_info(True, "")
966
967 start_test("Test bpftool bound info reporting (remote ns)...")
968 check_dev_info(False, ns)
969
970 start_test("Test bpftool bound info reporting (back to own ns)...")
971 sim.set_ns("")
972 check_dev_info(False, "")
973
974 prog_file, _ = pin_prog("/sys/fs/bpf/tmp_prog")
975 map_file, _ = pin_map("/sys/fs/bpf/tmp_map", idx=1, expected=2)
976 sim.remove()
977
978 start_test("Test bpftool bound info reporting (removed dev)...")
979 check_dev_info_removed(prog_file=prog_file, map_file=map_file)
980
981 # Remove all pinned files and reinstantiate the netdev
982 clean_up()
983 bpftool_prog_list_wait(expected=0)
984
985 sim = NetdevSim()
986
987 start_test("Test map update (no flags)...")
988 sim.set_xdp(map_obj, "offload", JSON=False) # map fixup msg breaks JSON
989 maps = bpftool_map_list(expected=2)
990 array = maps[0] if maps[0]["type"] == "array" else maps[1]
991 htab = maps[0] if maps[0]["type"] == "hash" else maps[1]
992 for m in maps:
993 for i in range(2):
994 bpftool("map update id %d key %s value %s" %
995 (m["id"], int2str("I", i), int2str("Q", i * 3)))
996
997 for m in maps:
998 ret, _ = bpftool("map update id %d key %s value %s" %
999 (m["id"], int2str("I", 3), int2str("Q", 3 * 3)),
1000 fail=False)
1001 fail(ret == 0, "added too many entries")
1002
1003 start_test("Test map update (exists)...")
1004 for m in maps:
1005 for i in range(2):
1006 bpftool("map update id %d key %s value %s exist" %
1007 (m["id"], int2str("I", i), int2str("Q", i * 3)))
1008
1009 for m in maps:
1010 ret, err = bpftool("map update id %d key %s value %s exist" %
1011 (m["id"], int2str("I", 3), int2str("Q", 3 * 3)),
1012 fail=False)
1013 fail(ret == 0, "updated non-existing key")
1014 fail(err["error"].find("No such file or directory") == -1,
1015 "expected ENOENT, error is '%s'" % (err["error"]))
1016
1017 start_test("Test map update (noexist)...")
1018 for m in maps:
1019 for i in range(2):
1020 ret, err = bpftool("map update id %d key %s value %s noexist" %
1021 (m["id"], int2str("I", i), int2str("Q", i * 3)),
1022 fail=False)
1023 fail(ret == 0, "updated existing key")
1024 fail(err["error"].find("File exists") == -1,
1025 "expected EEXIST, error is '%s'" % (err["error"]))
1026
1027 start_test("Test map dump...")
1028 for m in maps:
1029 _, entries = bpftool("map dump id %d" % (m["id"]))
1030 for i in range(2):
1031 key = str2int(entries[i]["key"])
1032 fail(key != i, "expected key %d, got %d" % (key, i))
1033 val = str2int(entries[i]["value"])
1034 fail(val != i * 3, "expected value %d, got %d" % (val, i * 3))
1035
1036 start_test("Test map getnext...")
1037 for m in maps:
1038 _, entry = bpftool("map getnext id %d" % (m["id"]))
1039 key = str2int(entry["next_key"])
1040 fail(key != 0, "next key %d, expected %d" % (key, 0))
1041 _, entry = bpftool("map getnext id %d key %s" %
1042 (m["id"], int2str("I", 0)))
1043 key = str2int(entry["next_key"])
1044 fail(key != 1, "next key %d, expected %d" % (key, 1))
1045 ret, err = bpftool("map getnext id %d key %s" %
1046 (m["id"], int2str("I", 1)), fail=False)
1047 fail(ret == 0, "got next key past the end of map")
1048 fail(err["error"].find("No such file or directory") == -1,
1049 "expected ENOENT, error is '%s'" % (err["error"]))
1050
1051 start_test("Test map delete (htab)...")
1052 for i in range(2):
1053 bpftool("map delete id %d key %s" % (htab["id"], int2str("I", i)))
1054
1055 start_test("Test map delete (array)...")
1056 for i in range(2):
1057 ret, err = bpftool("map delete id %d key %s" %
1058 (htab["id"], int2str("I", i)), fail=False)
1059 fail(ret == 0, "removed entry from an array")
1060 fail(err["error"].find("No such file or directory") == -1,
1061 "expected ENOENT, error is '%s'" % (err["error"]))
1062
1063 start_test("Test map remove...")
1064 sim.unset_xdp("offload")
1065 bpftool_map_list_wait(expected=0)
1066 sim.remove()
1067
1068 sim = NetdevSim()
1069 sim.set_xdp(map_obj, "offload", JSON=False) # map fixup msg breaks JSON
1070 sim.remove()
1071 bpftool_map_list_wait(expected=0)
1072
1073 start_test("Test map creation fail path...")
1074 sim = NetdevSim()
1075 sim.dfs["bpf_map_accept"] = "N"
1076 ret, _ = sim.set_xdp(map_obj, "offload", JSON=False, fail=False)
1077 fail(ret == 0,
1078 "netdevsim didn't refuse to create a map with offload disabled")
1079
1080 print("%s: OK" % (os.path.basename(__file__)))
1081
1082finally:
1083 log("Clean up...", "", level=1)
1084 log_level_inc()
1085 clean_up()
diff --git a/tools/testing/selftests/bpf/test_progs.c b/tools/testing/selftests/bpf/test_progs.c
index 6761be18a91f..b549308abd19 100644
--- a/tools/testing/selftests/bpf/test_progs.c
+++ b/tools/testing/selftests/bpf/test_progs.c
@@ -21,8 +21,10 @@ typedef __u16 __sum16;
21#include <linux/ipv6.h> 21#include <linux/ipv6.h>
22#include <linux/tcp.h> 22#include <linux/tcp.h>
23#include <linux/filter.h> 23#include <linux/filter.h>
24#include <linux/perf_event.h>
24#include <linux/unistd.h> 25#include <linux/unistd.h>
25 26
27#include <sys/ioctl.h>
26#include <sys/wait.h> 28#include <sys/wait.h>
27#include <sys/resource.h> 29#include <sys/resource.h>
28#include <sys/types.h> 30#include <sys/types.h>
@@ -167,10 +169,9 @@ out:
167#define NUM_ITER 100000 169#define NUM_ITER 100000
168#define VIP_NUM 5 170#define VIP_NUM 5
169 171
170static void test_l4lb(void) 172static void test_l4lb(const char *file)
171{ 173{
172 unsigned int nr_cpus = bpf_num_possible_cpus(); 174 unsigned int nr_cpus = bpf_num_possible_cpus();
173 const char *file = "./test_l4lb.o";
174 struct vip key = {.protocol = 6}; 175 struct vip key = {.protocol = 6};
175 struct vip_meta { 176 struct vip_meta {
176 __u32 flags; 177 __u32 flags;
@@ -247,6 +248,95 @@ out:
247 bpf_object__close(obj); 248 bpf_object__close(obj);
248} 249}
249 250
251static void test_l4lb_all(void)
252{
253 const char *file1 = "./test_l4lb.o";
254 const char *file2 = "./test_l4lb_noinline.o";
255
256 test_l4lb(file1);
257 test_l4lb(file2);
258}
259
260static void test_xdp_noinline(void)
261{
262 const char *file = "./test_xdp_noinline.o";
263 unsigned int nr_cpus = bpf_num_possible_cpus();
264 struct vip key = {.protocol = 6};
265 struct vip_meta {
266 __u32 flags;
267 __u32 vip_num;
268 } value = {.vip_num = VIP_NUM};
269 __u32 stats_key = VIP_NUM;
270 struct vip_stats {
271 __u64 bytes;
272 __u64 pkts;
273 } stats[nr_cpus];
274 struct real_definition {
275 union {
276 __be32 dst;
277 __be32 dstv6[4];
278 };
279 __u8 flags;
280 } real_def = {.dst = MAGIC_VAL};
281 __u32 ch_key = 11, real_num = 3;
282 __u32 duration, retval, size;
283 int err, i, prog_fd, map_fd;
284 __u64 bytes = 0, pkts = 0;
285 struct bpf_object *obj;
286 char buf[128];
287 u32 *magic = (u32 *)buf;
288
289 err = bpf_prog_load(file, BPF_PROG_TYPE_XDP, &obj, &prog_fd);
290 if (err) {
291 error_cnt++;
292 return;
293 }
294
295 map_fd = bpf_find_map(__func__, obj, "vip_map");
296 if (map_fd < 0)
297 goto out;
298 bpf_map_update_elem(map_fd, &key, &value, 0);
299
300 map_fd = bpf_find_map(__func__, obj, "ch_rings");
301 if (map_fd < 0)
302 goto out;
303 bpf_map_update_elem(map_fd, &ch_key, &real_num, 0);
304
305 map_fd = bpf_find_map(__func__, obj, "reals");
306 if (map_fd < 0)
307 goto out;
308 bpf_map_update_elem(map_fd, &real_num, &real_def, 0);
309
310 err = bpf_prog_test_run(prog_fd, NUM_ITER, &pkt_v4, sizeof(pkt_v4),
311 buf, &size, &retval, &duration);
312 CHECK(err || errno || retval != 1 || size != 54 ||
313 *magic != MAGIC_VAL, "ipv4",
314 "err %d errno %d retval %d size %d magic %x\n",
315 err, errno, retval, size, *magic);
316
317 err = bpf_prog_test_run(prog_fd, NUM_ITER, &pkt_v6, sizeof(pkt_v6),
318 buf, &size, &retval, &duration);
319 CHECK(err || errno || retval != 1 || size != 74 ||
320 *magic != MAGIC_VAL, "ipv6",
321 "err %d errno %d retval %d size %d magic %x\n",
322 err, errno, retval, size, *magic);
323
324 map_fd = bpf_find_map(__func__, obj, "stats");
325 if (map_fd < 0)
326 goto out;
327 bpf_map_lookup_elem(map_fd, &stats_key, stats);
328 for (i = 0; i < nr_cpus; i++) {
329 bytes += stats[i].bytes;
330 pkts += stats[i].pkts;
331 }
332 if (bytes != MAGIC_BYTES * NUM_ITER * 2 || pkts != NUM_ITER * 2) {
333 error_cnt++;
334 printf("test_xdp_noinline:FAIL:stats %lld %lld\n", bytes, pkts);
335 }
336out:
337 bpf_object__close(obj);
338}
339
250static void test_tcp_estats(void) 340static void test_tcp_estats(void)
251{ 341{
252 const char *file = "./test_tcp_estats.o"; 342 const char *file = "./test_tcp_estats.o";
@@ -617,6 +707,262 @@ static void test_obj_name(void)
617 } 707 }
618} 708}
619 709
710static void test_tp_attach_query(void)
711{
712 const int num_progs = 3;
713 int i, j, bytes, efd, err, prog_fd[num_progs], pmu_fd[num_progs];
714 __u32 duration = 0, info_len, saved_prog_ids[num_progs];
715 const char *file = "./test_tracepoint.o";
716 struct perf_event_query_bpf *query;
717 struct perf_event_attr attr = {};
718 struct bpf_object *obj[num_progs];
719 struct bpf_prog_info prog_info;
720 char buf[256];
721
722 snprintf(buf, sizeof(buf),
723 "/sys/kernel/debug/tracing/events/sched/sched_switch/id");
724 efd = open(buf, O_RDONLY, 0);
725 if (CHECK(efd < 0, "open", "err %d errno %d\n", efd, errno))
726 return;
727 bytes = read(efd, buf, sizeof(buf));
728 close(efd);
729 if (CHECK(bytes <= 0 || bytes >= sizeof(buf),
730 "read", "bytes %d errno %d\n", bytes, errno))
731 return;
732
733 attr.config = strtol(buf, NULL, 0);
734 attr.type = PERF_TYPE_TRACEPOINT;
735 attr.sample_type = PERF_SAMPLE_RAW | PERF_SAMPLE_CALLCHAIN;
736 attr.sample_period = 1;
737 attr.wakeup_events = 1;
738
739 query = malloc(sizeof(*query) + sizeof(__u32) * num_progs);
740 for (i = 0; i < num_progs; i++) {
741 err = bpf_prog_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj[i],
742 &prog_fd[i]);
743 if (CHECK(err, "prog_load", "err %d errno %d\n", err, errno))
744 goto cleanup1;
745
746 bzero(&prog_info, sizeof(prog_info));
747 prog_info.jited_prog_len = 0;
748 prog_info.xlated_prog_len = 0;
749 prog_info.nr_map_ids = 0;
750 info_len = sizeof(prog_info);
751 err = bpf_obj_get_info_by_fd(prog_fd[i], &prog_info, &info_len);
752 if (CHECK(err, "bpf_obj_get_info_by_fd", "err %d errno %d\n",
753 err, errno))
754 goto cleanup1;
755 saved_prog_ids[i] = prog_info.id;
756
757 pmu_fd[i] = syscall(__NR_perf_event_open, &attr, -1 /* pid */,
758 0 /* cpu 0 */, -1 /* group id */,
759 0 /* flags */);
760 if (CHECK(pmu_fd[i] < 0, "perf_event_open", "err %d errno %d\n",
761 pmu_fd[i], errno))
762 goto cleanup2;
763 err = ioctl(pmu_fd[i], PERF_EVENT_IOC_ENABLE, 0);
764 if (CHECK(err, "perf_event_ioc_enable", "err %d errno %d\n",
765 err, errno))
766 goto cleanup3;
767
768 if (i == 0) {
769 /* check NULL prog array query */
770 query->ids_len = num_progs;
771 err = ioctl(pmu_fd[i], PERF_EVENT_IOC_QUERY_BPF, query);
772 if (CHECK(err || query->prog_cnt != 0,
773 "perf_event_ioc_query_bpf",
774 "err %d errno %d query->prog_cnt %u\n",
775 err, errno, query->prog_cnt))
776 goto cleanup3;
777 }
778
779 err = ioctl(pmu_fd[i], PERF_EVENT_IOC_SET_BPF, prog_fd[i]);
780 if (CHECK(err, "perf_event_ioc_set_bpf", "err %d errno %d\n",
781 err, errno))
782 goto cleanup3;
783
784 if (i == 1) {
785 /* try to get # of programs only */
786 query->ids_len = 0;
787 err = ioctl(pmu_fd[i], PERF_EVENT_IOC_QUERY_BPF, query);
788 if (CHECK(err || query->prog_cnt != 2,
789 "perf_event_ioc_query_bpf",
790 "err %d errno %d query->prog_cnt %u\n",
791 err, errno, query->prog_cnt))
792 goto cleanup3;
793
794 /* try a few negative tests */
795 /* invalid query pointer */
796 err = ioctl(pmu_fd[i], PERF_EVENT_IOC_QUERY_BPF,
797 (struct perf_event_query_bpf *)0x1);
798 if (CHECK(!err || errno != EFAULT,
799 "perf_event_ioc_query_bpf",
800 "err %d errno %d\n", err, errno))
801 goto cleanup3;
802
803 /* no enough space */
804 query->ids_len = 1;
805 err = ioctl(pmu_fd[i], PERF_EVENT_IOC_QUERY_BPF, query);
806 if (CHECK(!err || errno != ENOSPC || query->prog_cnt != 2,
807 "perf_event_ioc_query_bpf",
808 "err %d errno %d query->prog_cnt %u\n",
809 err, errno, query->prog_cnt))
810 goto cleanup3;
811 }
812
813 query->ids_len = num_progs;
814 err = ioctl(pmu_fd[i], PERF_EVENT_IOC_QUERY_BPF, query);
815 if (CHECK(err || query->prog_cnt != (i + 1),
816 "perf_event_ioc_query_bpf",
817 "err %d errno %d query->prog_cnt %u\n",
818 err, errno, query->prog_cnt))
819 goto cleanup3;
820 for (j = 0; j < i + 1; j++)
821 if (CHECK(saved_prog_ids[j] != query->ids[j],
822 "perf_event_ioc_query_bpf",
823 "#%d saved_prog_id %x query prog_id %x\n",
824 j, saved_prog_ids[j], query->ids[j]))
825 goto cleanup3;
826 }
827
828 i = num_progs - 1;
829 for (; i >= 0; i--) {
830 cleanup3:
831 ioctl(pmu_fd[i], PERF_EVENT_IOC_DISABLE);
832 cleanup2:
833 close(pmu_fd[i]);
834 cleanup1:
835 bpf_object__close(obj[i]);
836 }
837 free(query);
838}
839
840static int compare_map_keys(int map1_fd, int map2_fd)
841{
842 __u32 key, next_key;
843 char val_buf[PERF_MAX_STACK_DEPTH * sizeof(__u64)];
844 int err;
845
846 err = bpf_map_get_next_key(map1_fd, NULL, &key);
847 if (err)
848 return err;
849 err = bpf_map_lookup_elem(map2_fd, &key, val_buf);
850 if (err)
851 return err;
852
853 while (bpf_map_get_next_key(map1_fd, &key, &next_key) == 0) {
854 err = bpf_map_lookup_elem(map2_fd, &next_key, val_buf);
855 if (err)
856 return err;
857
858 key = next_key;
859 }
860 if (errno != ENOENT)
861 return -1;
862
863 return 0;
864}
865
866static void test_stacktrace_map()
867{
868 int control_map_fd, stackid_hmap_fd, stackmap_fd;
869 const char *file = "./test_stacktrace_map.o";
870 int bytes, efd, err, pmu_fd, prog_fd;
871 struct perf_event_attr attr = {};
872 __u32 key, val, duration = 0;
873 struct bpf_object *obj;
874 char buf[256];
875
876 err = bpf_prog_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj, &prog_fd);
877 if (CHECK(err, "prog_load", "err %d errno %d\n", err, errno))
878 goto out;
879
880 /* Get the ID for the sched/sched_switch tracepoint */
881 snprintf(buf, sizeof(buf),
882 "/sys/kernel/debug/tracing/events/sched/sched_switch/id");
883 efd = open(buf, O_RDONLY, 0);
884 if (CHECK(efd < 0, "open", "err %d errno %d\n", efd, errno))
885 goto close_prog;
886
887 bytes = read(efd, buf, sizeof(buf));
888 close(efd);
889 if (CHECK(bytes <= 0 || bytes >= sizeof(buf),
890 "read", "bytes %d errno %d\n", bytes, errno))
891 goto close_prog;
892
893 /* Open the perf event and attach bpf progrram */
894 attr.config = strtol(buf, NULL, 0);
895 attr.type = PERF_TYPE_TRACEPOINT;
896 attr.sample_type = PERF_SAMPLE_RAW | PERF_SAMPLE_CALLCHAIN;
897 attr.sample_period = 1;
898 attr.wakeup_events = 1;
899 pmu_fd = syscall(__NR_perf_event_open, &attr, -1 /* pid */,
900 0 /* cpu 0 */, -1 /* group id */,
901 0 /* flags */);
902 if (CHECK(pmu_fd < 0, "perf_event_open", "err %d errno %d\n",
903 pmu_fd, errno))
904 goto close_prog;
905
906 err = ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0);
907 if (CHECK(err, "perf_event_ioc_enable", "err %d errno %d\n",
908 err, errno))
909 goto close_pmu;
910
911 err = ioctl(pmu_fd, PERF_EVENT_IOC_SET_BPF, prog_fd);
912 if (CHECK(err, "perf_event_ioc_set_bpf", "err %d errno %d\n",
913 err, errno))
914 goto disable_pmu;
915
916 /* find map fds */
917 control_map_fd = bpf_find_map(__func__, obj, "control_map");
918 if (CHECK(control_map_fd < 0, "bpf_find_map control_map",
919 "err %d errno %d\n", err, errno))
920 goto disable_pmu;
921
922 stackid_hmap_fd = bpf_find_map(__func__, obj, "stackid_hmap");
923 if (CHECK(stackid_hmap_fd < 0, "bpf_find_map stackid_hmap",
924 "err %d errno %d\n", err, errno))
925 goto disable_pmu;
926
927 stackmap_fd = bpf_find_map(__func__, obj, "stackmap");
928 if (CHECK(stackmap_fd < 0, "bpf_find_map stackmap", "err %d errno %d\n",
929 err, errno))
930 goto disable_pmu;
931
932 /* give some time for bpf program run */
933 sleep(1);
934
935 /* disable stack trace collection */
936 key = 0;
937 val = 1;
938 bpf_map_update_elem(control_map_fd, &key, &val, 0);
939
940 /* for every element in stackid_hmap, we can find a corresponding one
941 * in stackmap, and vise versa.
942 */
943 err = compare_map_keys(stackid_hmap_fd, stackmap_fd);
944 if (CHECK(err, "compare_map_keys stackid_hmap vs. stackmap",
945 "err %d errno %d\n", err, errno))
946 goto disable_pmu;
947
948 err = compare_map_keys(stackmap_fd, stackid_hmap_fd);
949 if (CHECK(err, "compare_map_keys stackmap vs. stackid_hmap",
950 "err %d errno %d\n", err, errno))
951 ; /* fall through */
952
953disable_pmu:
954 ioctl(pmu_fd, PERF_EVENT_IOC_DISABLE);
955
956close_pmu:
957 close(pmu_fd);
958
959close_prog:
960 bpf_object__close(obj);
961
962out:
963 return;
964}
965
620int main(void) 966int main(void)
621{ 967{
622 struct rlimit rinf = { RLIM_INFINITY, RLIM_INFINITY }; 968 struct rlimit rinf = { RLIM_INFINITY, RLIM_INFINITY };
@@ -625,11 +971,14 @@ int main(void)
625 971
626 test_pkt_access(); 972 test_pkt_access();
627 test_xdp(); 973 test_xdp();
628 test_l4lb(); 974 test_l4lb_all();
975 test_xdp_noinline();
629 test_tcp_estats(); 976 test_tcp_estats();
630 test_bpf_obj_id(); 977 test_bpf_obj_id();
631 test_pkt_md_access(); 978 test_pkt_md_access();
632 test_obj_name(); 979 test_obj_name();
980 test_tp_attach_query();
981 test_stacktrace_map();
633 982
634 printf("Summary: %d PASSED, %d FAILED\n", pass_cnt, error_cnt); 983 printf("Summary: %d PASSED, %d FAILED\n", pass_cnt, error_cnt);
635 return error_cnt ? EXIT_FAILURE : EXIT_SUCCESS; 984 return error_cnt ? EXIT_FAILURE : EXIT_SUCCESS;
diff --git a/tools/testing/selftests/bpf/test_stacktrace_map.c b/tools/testing/selftests/bpf/test_stacktrace_map.c
new file mode 100644
index 000000000000..76d85c5d08bd
--- /dev/null
+++ b/tools/testing/selftests/bpf/test_stacktrace_map.c
@@ -0,0 +1,62 @@
1// SPDX-License-Identifier: GPL-2.0
2// Copyright (c) 2018 Facebook
3
4#include <linux/bpf.h>
5#include "bpf_helpers.h"
6
7#ifndef PERF_MAX_STACK_DEPTH
8#define PERF_MAX_STACK_DEPTH 127
9#endif
10
11struct bpf_map_def SEC("maps") control_map = {
12 .type = BPF_MAP_TYPE_ARRAY,
13 .key_size = sizeof(__u32),
14 .value_size = sizeof(__u32),
15 .max_entries = 1,
16};
17
18struct bpf_map_def SEC("maps") stackid_hmap = {
19 .type = BPF_MAP_TYPE_HASH,
20 .key_size = sizeof(__u32),
21 .value_size = sizeof(__u32),
22 .max_entries = 10000,
23};
24
25struct bpf_map_def SEC("maps") stackmap = {
26 .type = BPF_MAP_TYPE_STACK_TRACE,
27 .key_size = sizeof(__u32),
28 .value_size = sizeof(__u64) * PERF_MAX_STACK_DEPTH,
29 .max_entries = 10000,
30};
31
32/* taken from /sys/kernel/debug/tracing/events/sched/sched_switch/format */
33struct sched_switch_args {
34 unsigned long long pad;
35 char prev_comm[16];
36 int prev_pid;
37 int prev_prio;
38 long long prev_state;
39 char next_comm[16];
40 int next_pid;
41 int next_prio;
42};
43
44SEC("tracepoint/sched/sched_switch")
45int oncpu(struct sched_switch_args *ctx)
46{
47 __u32 key = 0, val = 0, *value_p;
48
49 value_p = bpf_map_lookup_elem(&control_map, &key);
50 if (value_p && *value_p)
51 return 0; /* skip if non-zero *value_p */
52
53 /* The size of stackmap and stackid_hmap should be the same */
54 key = bpf_get_stackid(ctx, &stackmap, 0);
55 if ((int)key >= 0)
56 bpf_map_update_elem(&stackid_hmap, &key, &val, 0);
57
58 return 0;
59}
60
61char _license[] SEC("license") = "GPL";
62__u32 _version SEC("version") = 1; /* ignored by tracepoints, required by libbpf.a */
diff --git a/tools/testing/selftests/bpf/test_tcpbpf.h b/tools/testing/selftests/bpf/test_tcpbpf.h
new file mode 100644
index 000000000000..2fe43289943c
--- /dev/null
+++ b/tools/testing/selftests/bpf/test_tcpbpf.h
@@ -0,0 +1,16 @@
1// SPDX-License-Identifier: GPL-2.0
2
3#ifndef _TEST_TCPBPF_H
4#define _TEST_TCPBPF_H
5
6struct tcpbpf_globals {
7 __u32 event_map;
8 __u32 total_retrans;
9 __u32 data_segs_in;
10 __u32 data_segs_out;
11 __u32 bad_cb_test_rv;
12 __u32 good_cb_test_rv;
13 __u64 bytes_received;
14 __u64 bytes_acked;
15};
16#endif
diff --git a/tools/testing/selftests/bpf/test_tcpbpf_kern.c b/tools/testing/selftests/bpf/test_tcpbpf_kern.c
new file mode 100644
index 000000000000..57119ad57a3f
--- /dev/null
+++ b/tools/testing/selftests/bpf/test_tcpbpf_kern.c
@@ -0,0 +1,115 @@
1// SPDX-License-Identifier: GPL-2.0
2#include <stddef.h>
3#include <string.h>
4#include <linux/bpf.h>
5#include <linux/if_ether.h>
6#include <linux/if_packet.h>
7#include <linux/ip.h>
8#include <linux/in6.h>
9#include <linux/types.h>
10#include <linux/socket.h>
11#include <linux/tcp.h>
12#include <netinet/in.h>
13#include "bpf_helpers.h"
14#include "bpf_endian.h"
15#include "test_tcpbpf.h"
16
17struct bpf_map_def SEC("maps") global_map = {
18 .type = BPF_MAP_TYPE_ARRAY,
19 .key_size = sizeof(__u32),
20 .value_size = sizeof(struct tcpbpf_globals),
21 .max_entries = 2,
22};
23
24static inline void update_event_map(int event)
25{
26 __u32 key = 0;
27 struct tcpbpf_globals g, *gp;
28
29 gp = bpf_map_lookup_elem(&global_map, &key);
30 if (gp == NULL) {
31 struct tcpbpf_globals g = {0};
32
33 g.event_map |= (1 << event);
34 bpf_map_update_elem(&global_map, &key, &g,
35 BPF_ANY);
36 } else {
37 g = *gp;
38 g.event_map |= (1 << event);
39 bpf_map_update_elem(&global_map, &key, &g,
40 BPF_ANY);
41 }
42}
43
44int _version SEC("version") = 1;
45
46SEC("sockops")
47int bpf_testcb(struct bpf_sock_ops *skops)
48{
49 int rv = -1;
50 int bad_call_rv = 0;
51 int good_call_rv = 0;
52 int op;
53 int v = 0;
54
55 op = (int) skops->op;
56
57 update_event_map(op);
58
59 switch (op) {
60 case BPF_SOCK_OPS_ACTIVE_ESTABLISHED_CB:
61 /* Test failure to set largest cb flag (assumes not defined) */
62 bad_call_rv = bpf_sock_ops_cb_flags_set(skops, 0x80);
63 /* Set callback */
64 good_call_rv = bpf_sock_ops_cb_flags_set(skops,
65 BPF_SOCK_OPS_STATE_CB_FLAG);
66 /* Update results */
67 {
68 __u32 key = 0;
69 struct tcpbpf_globals g, *gp;
70
71 gp = bpf_map_lookup_elem(&global_map, &key);
72 if (!gp)
73 break;
74 g = *gp;
75 g.bad_cb_test_rv = bad_call_rv;
76 g.good_cb_test_rv = good_call_rv;
77 bpf_map_update_elem(&global_map, &key, &g,
78 BPF_ANY);
79 }
80 break;
81 case BPF_SOCK_OPS_PASSIVE_ESTABLISHED_CB:
82 skops->sk_txhash = 0x12345f;
83 v = 0xff;
84 rv = bpf_setsockopt(skops, SOL_IPV6, IPV6_TCLASS, &v,
85 sizeof(v));
86 break;
87 case BPF_SOCK_OPS_RTO_CB:
88 break;
89 case BPF_SOCK_OPS_RETRANS_CB:
90 break;
91 case BPF_SOCK_OPS_STATE_CB:
92 if (skops->args[1] == BPF_TCP_CLOSE) {
93 __u32 key = 0;
94 struct tcpbpf_globals g, *gp;
95
96 gp = bpf_map_lookup_elem(&global_map, &key);
97 if (!gp)
98 break;
99 g = *gp;
100 g.total_retrans = skops->total_retrans;
101 g.data_segs_in = skops->data_segs_in;
102 g.data_segs_out = skops->data_segs_out;
103 g.bytes_received = skops->bytes_received;
104 g.bytes_acked = skops->bytes_acked;
105 bpf_map_update_elem(&global_map, &key, &g,
106 BPF_ANY);
107 }
108 break;
109 default:
110 rv = -1;
111 }
112 skops->reply = rv;
113 return 1;
114}
115char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/test_tcpbpf_user.c b/tools/testing/selftests/bpf/test_tcpbpf_user.c
new file mode 100644
index 000000000000..95a370f3d378
--- /dev/null
+++ b/tools/testing/selftests/bpf/test_tcpbpf_user.c
@@ -0,0 +1,126 @@
1// SPDX-License-Identifier: GPL-2.0
2#include <stdio.h>
3#include <stdlib.h>
4#include <stdio.h>
5#include <unistd.h>
6#include <errno.h>
7#include <signal.h>
8#include <string.h>
9#include <assert.h>
10#include <linux/perf_event.h>
11#include <linux/ptrace.h>
12#include <linux/bpf.h>
13#include <sys/ioctl.h>
14#include <sys/types.h>
15#include <sys/stat.h>
16#include <fcntl.h>
17#include <bpf/bpf.h>
18#include <bpf/libbpf.h>
19#include "bpf_util.h"
20#include <linux/perf_event.h>
21#include "test_tcpbpf.h"
22
23static int bpf_find_map(const char *test, struct bpf_object *obj,
24 const char *name)
25{
26 struct bpf_map *map;
27
28 map = bpf_object__find_map_by_name(obj, name);
29 if (!map) {
30 printf("%s:FAIL:map '%s' not found\n", test, name);
31 return -1;
32 }
33 return bpf_map__fd(map);
34}
35
36#define SYSTEM(CMD) \
37 do { \
38 if (system(CMD)) { \
39 printf("system(%s) FAILS!\n", CMD); \
40 } \
41 } while (0)
42
43int main(int argc, char **argv)
44{
45 const char *file = "test_tcpbpf_kern.o";
46 struct tcpbpf_globals g = {0};
47 int cg_fd, prog_fd, map_fd;
48 bool debug_flag = false;
49 int error = EXIT_FAILURE;
50 struct bpf_object *obj;
51 char cmd[100], *dir;
52 struct stat buffer;
53 __u32 key = 0;
54 int pid;
55 int rv;
56
57 if (argc > 1 && strcmp(argv[1], "-d") == 0)
58 debug_flag = true;
59
60 dir = "/tmp/cgroupv2/foo";
61
62 if (stat(dir, &buffer) != 0) {
63 SYSTEM("mkdir -p /tmp/cgroupv2");
64 SYSTEM("mount -t cgroup2 none /tmp/cgroupv2");
65 SYSTEM("mkdir -p /tmp/cgroupv2/foo");
66 }
67 pid = (int) getpid();
68 sprintf(cmd, "echo %d >> /tmp/cgroupv2/foo/cgroup.procs", pid);
69 SYSTEM(cmd);
70
71 cg_fd = open(dir, O_DIRECTORY, O_RDONLY);
72 if (bpf_prog_load(file, BPF_PROG_TYPE_SOCK_OPS, &obj, &prog_fd)) {
73 printf("FAILED: load_bpf_file failed for: %s\n", file);
74 goto err;
75 }
76
77 rv = bpf_prog_attach(prog_fd, cg_fd, BPF_CGROUP_SOCK_OPS, 0);
78 if (rv) {
79 printf("FAILED: bpf_prog_attach: %d (%s)\n",
80 error, strerror(errno));
81 goto err;
82 }
83
84 SYSTEM("./tcp_server.py");
85
86 map_fd = bpf_find_map(__func__, obj, "global_map");
87 if (map_fd < 0)
88 goto err;
89
90 rv = bpf_map_lookup_elem(map_fd, &key, &g);
91 if (rv != 0) {
92 printf("FAILED: bpf_map_lookup_elem returns %d\n", rv);
93 goto err;
94 }
95
96 if (g.bytes_received != 501 || g.bytes_acked != 1002 ||
97 g.data_segs_in != 1 || g.data_segs_out != 1 ||
98 (g.event_map ^ 0x47e) != 0 || g.bad_cb_test_rv != 0x80 ||
99 g.good_cb_test_rv != 0) {
100 printf("FAILED: Wrong stats\n");
101 if (debug_flag) {
102 printf("\n");
103 printf("bytes_received: %d (expecting 501)\n",
104 (int)g.bytes_received);
105 printf("bytes_acked: %d (expecting 1002)\n",
106 (int)g.bytes_acked);
107 printf("data_segs_in: %d (expecting 1)\n",
108 g.data_segs_in);
109 printf("data_segs_out: %d (expecting 1)\n",
110 g.data_segs_out);
111 printf("event_map: 0x%x (at least 0x47e)\n",
112 g.event_map);
113 printf("bad_cb_test_rv: 0x%x (expecting 0x80)\n",
114 g.bad_cb_test_rv);
115 printf("good_cb_test_rv:0x%x (expecting 0)\n",
116 g.good_cb_test_rv);
117 }
118 goto err;
119 }
120 printf("PASSED!\n");
121 error = 0;
122err:
123 bpf_prog_detach(cg_fd, BPF_CGROUP_SOCK_OPS);
124 return error;
125
126}
diff --git a/tools/testing/selftests/bpf/test_tracepoint.c b/tools/testing/selftests/bpf/test_tracepoint.c
new file mode 100644
index 000000000000..04bf084517e0
--- /dev/null
+++ b/tools/testing/selftests/bpf/test_tracepoint.c
@@ -0,0 +1,26 @@
1// SPDX-License-Identifier: GPL-2.0
2// Copyright (c) 2017 Facebook
3
4#include <linux/bpf.h>
5#include "bpf_helpers.h"
6
7/* taken from /sys/kernel/debug/tracing/events/sched/sched_switch/format */
8struct sched_switch_args {
9 unsigned long long pad;
10 char prev_comm[16];
11 int prev_pid;
12 int prev_prio;
13 long long prev_state;
14 char next_comm[16];
15 int next_pid;
16 int next_prio;
17};
18
19SEC("tracepoint/sched/sched_switch")
20int oncpu(struct sched_switch_args *ctx)
21{
22 return 0;
23}
24
25char _license[] SEC("license") = "GPL";
26__u32 _version SEC("version") = 1; /* ignored by tracepoints, required by libbpf.a */
diff --git a/tools/testing/selftests/bpf/test_verifier.c b/tools/testing/selftests/bpf/test_verifier.c
index 5ed4175c4ff8..697bd83de295 100644
--- a/tools/testing/selftests/bpf/test_verifier.c
+++ b/tools/testing/selftests/bpf/test_verifier.c
@@ -2,6 +2,7 @@
2 * Testsuite for eBPF verifier 2 * Testsuite for eBPF verifier
3 * 3 *
4 * Copyright (c) 2014 PLUMgrid, http://plumgrid.com 4 * Copyright (c) 2014 PLUMgrid, http://plumgrid.com
5 * Copyright (c) 2017 Facebook
5 * 6 *
6 * This program is free software; you can redistribute it and/or 7 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of version 2 of the GNU General Public 8 * modify it under the terms of version 2 of the GNU General Public
@@ -20,6 +21,7 @@
20#include <stddef.h> 21#include <stddef.h>
21#include <stdbool.h> 22#include <stdbool.h>
22#include <sched.h> 23#include <sched.h>
24#include <limits.h>
23 25
24#include <sys/capability.h> 26#include <sys/capability.h>
25#include <sys/resource.h> 27#include <sys/resource.h>
@@ -28,6 +30,7 @@
28#include <linux/filter.h> 30#include <linux/filter.h>
29#include <linux/bpf_perf_event.h> 31#include <linux/bpf_perf_event.h>
30#include <linux/bpf.h> 32#include <linux/bpf.h>
33#include <linux/if_ether.h>
31 34
32#include <bpf/bpf.h> 35#include <bpf/bpf.h>
33 36
@@ -48,6 +51,8 @@
48#define MAX_INSNS 512 51#define MAX_INSNS 512
49#define MAX_FIXUPS 8 52#define MAX_FIXUPS 8
50#define MAX_NR_MAPS 4 53#define MAX_NR_MAPS 4
54#define POINTER_VALUE 0xcafe4all
55#define TEST_DATA_LEN 64
51 56
52#define F_NEEDS_EFFICIENT_UNALIGNED_ACCESS (1 << 0) 57#define F_NEEDS_EFFICIENT_UNALIGNED_ACCESS (1 << 0)
53#define F_LOAD_WITH_STRICT_ALIGNMENT (1 << 1) 58#define F_LOAD_WITH_STRICT_ALIGNMENT (1 << 1)
@@ -61,6 +66,7 @@ struct bpf_test {
61 int fixup_map_in_map[MAX_FIXUPS]; 66 int fixup_map_in_map[MAX_FIXUPS];
62 const char *errstr; 67 const char *errstr;
63 const char *errstr_unpriv; 68 const char *errstr_unpriv;
69 uint32_t retval;
64 enum { 70 enum {
65 UNDEF, 71 UNDEF,
66 ACCEPT, 72 ACCEPT,
@@ -94,6 +100,326 @@ static struct bpf_test tests[] = {
94 BPF_EXIT_INSN(), 100 BPF_EXIT_INSN(),
95 }, 101 },
96 .result = ACCEPT, 102 .result = ACCEPT,
103 .retval = -3,
104 },
105 {
106 "DIV32 by 0, zero check 1",
107 .insns = {
108 BPF_MOV32_IMM(BPF_REG_0, 42),
109 BPF_MOV32_IMM(BPF_REG_1, 0),
110 BPF_MOV32_IMM(BPF_REG_2, 1),
111 BPF_ALU32_REG(BPF_DIV, BPF_REG_2, BPF_REG_1),
112 BPF_EXIT_INSN(),
113 },
114 .result = ACCEPT,
115 .retval = 42,
116 },
117 {
118 "DIV32 by 0, zero check 2",
119 .insns = {
120 BPF_MOV32_IMM(BPF_REG_0, 42),
121 BPF_LD_IMM64(BPF_REG_1, 0xffffffff00000000LL),
122 BPF_MOV32_IMM(BPF_REG_2, 1),
123 BPF_ALU32_REG(BPF_DIV, BPF_REG_2, BPF_REG_1),
124 BPF_EXIT_INSN(),
125 },
126 .result = ACCEPT,
127 .retval = 42,
128 },
129 {
130 "DIV64 by 0, zero check",
131 .insns = {
132 BPF_MOV32_IMM(BPF_REG_0, 42),
133 BPF_MOV32_IMM(BPF_REG_1, 0),
134 BPF_MOV32_IMM(BPF_REG_2, 1),
135 BPF_ALU64_REG(BPF_DIV, BPF_REG_2, BPF_REG_1),
136 BPF_EXIT_INSN(),
137 },
138 .result = ACCEPT,
139 .retval = 42,
140 },
141 {
142 "MOD32 by 0, zero check 1",
143 .insns = {
144 BPF_MOV32_IMM(BPF_REG_0, 42),
145 BPF_MOV32_IMM(BPF_REG_1, 0),
146 BPF_MOV32_IMM(BPF_REG_2, 1),
147 BPF_ALU32_REG(BPF_MOD, BPF_REG_2, BPF_REG_1),
148 BPF_EXIT_INSN(),
149 },
150 .result = ACCEPT,
151 .retval = 42,
152 },
153 {
154 "MOD32 by 0, zero check 2",
155 .insns = {
156 BPF_MOV32_IMM(BPF_REG_0, 42),
157 BPF_LD_IMM64(BPF_REG_1, 0xffffffff00000000LL),
158 BPF_MOV32_IMM(BPF_REG_2, 1),
159 BPF_ALU32_REG(BPF_MOD, BPF_REG_2, BPF_REG_1),
160 BPF_EXIT_INSN(),
161 },
162 .result = ACCEPT,
163 .retval = 42,
164 },
165 {
166 "MOD64 by 0, zero check",
167 .insns = {
168 BPF_MOV32_IMM(BPF_REG_0, 42),
169 BPF_MOV32_IMM(BPF_REG_1, 0),
170 BPF_MOV32_IMM(BPF_REG_2, 1),
171 BPF_ALU64_REG(BPF_MOD, BPF_REG_2, BPF_REG_1),
172 BPF_EXIT_INSN(),
173 },
174 .result = ACCEPT,
175 .retval = 42,
176 },
177 {
178 "DIV32 by 0, zero check ok, cls",
179 .insns = {
180 BPF_MOV32_IMM(BPF_REG_0, 42),
181 BPF_MOV32_IMM(BPF_REG_1, 2),
182 BPF_MOV32_IMM(BPF_REG_2, 16),
183 BPF_ALU32_REG(BPF_DIV, BPF_REG_2, BPF_REG_1),
184 BPF_MOV64_REG(BPF_REG_0, BPF_REG_2),
185 BPF_EXIT_INSN(),
186 },
187 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
188 .result = ACCEPT,
189 .retval = 8,
190 },
191 {
192 "DIV32 by 0, zero check 1, cls",
193 .insns = {
194 BPF_MOV32_IMM(BPF_REG_1, 0),
195 BPF_MOV32_IMM(BPF_REG_0, 1),
196 BPF_ALU32_REG(BPF_DIV, BPF_REG_0, BPF_REG_1),
197 BPF_EXIT_INSN(),
198 },
199 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
200 .result = ACCEPT,
201 .retval = 0,
202 },
203 {
204 "DIV32 by 0, zero check 2, cls",
205 .insns = {
206 BPF_LD_IMM64(BPF_REG_1, 0xffffffff00000000LL),
207 BPF_MOV32_IMM(BPF_REG_0, 1),
208 BPF_ALU32_REG(BPF_DIV, BPF_REG_0, BPF_REG_1),
209 BPF_EXIT_INSN(),
210 },
211 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
212 .result = ACCEPT,
213 .retval = 0,
214 },
215 {
216 "DIV64 by 0, zero check, cls",
217 .insns = {
218 BPF_MOV32_IMM(BPF_REG_1, 0),
219 BPF_MOV32_IMM(BPF_REG_0, 1),
220 BPF_ALU64_REG(BPF_DIV, BPF_REG_0, BPF_REG_1),
221 BPF_EXIT_INSN(),
222 },
223 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
224 .result = ACCEPT,
225 .retval = 0,
226 },
227 {
228 "MOD32 by 0, zero check ok, cls",
229 .insns = {
230 BPF_MOV32_IMM(BPF_REG_0, 42),
231 BPF_MOV32_IMM(BPF_REG_1, 3),
232 BPF_MOV32_IMM(BPF_REG_2, 5),
233 BPF_ALU32_REG(BPF_MOD, BPF_REG_2, BPF_REG_1),
234 BPF_MOV64_REG(BPF_REG_0, BPF_REG_2),
235 BPF_EXIT_INSN(),
236 },
237 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
238 .result = ACCEPT,
239 .retval = 2,
240 },
241 {
242 "MOD32 by 0, zero check 1, cls",
243 .insns = {
244 BPF_MOV32_IMM(BPF_REG_1, 0),
245 BPF_MOV32_IMM(BPF_REG_0, 1),
246 BPF_ALU32_REG(BPF_MOD, BPF_REG_0, BPF_REG_1),
247 BPF_EXIT_INSN(),
248 },
249 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
250 .result = ACCEPT,
251 .retval = 1,
252 },
253 {
254 "MOD32 by 0, zero check 2, cls",
255 .insns = {
256 BPF_LD_IMM64(BPF_REG_1, 0xffffffff00000000LL),
257 BPF_MOV32_IMM(BPF_REG_0, 1),
258 BPF_ALU32_REG(BPF_MOD, BPF_REG_0, BPF_REG_1),
259 BPF_EXIT_INSN(),
260 },
261 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
262 .result = ACCEPT,
263 .retval = 1,
264 },
265 {
266 "MOD64 by 0, zero check 1, cls",
267 .insns = {
268 BPF_MOV32_IMM(BPF_REG_1, 0),
269 BPF_MOV32_IMM(BPF_REG_0, 2),
270 BPF_ALU64_REG(BPF_MOD, BPF_REG_0, BPF_REG_1),
271 BPF_EXIT_INSN(),
272 },
273 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
274 .result = ACCEPT,
275 .retval = 2,
276 },
277 {
278 "MOD64 by 0, zero check 2, cls",
279 .insns = {
280 BPF_MOV32_IMM(BPF_REG_1, 0),
281 BPF_MOV32_IMM(BPF_REG_0, -1),
282 BPF_ALU64_REG(BPF_MOD, BPF_REG_0, BPF_REG_1),
283 BPF_EXIT_INSN(),
284 },
285 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
286 .result = ACCEPT,
287 .retval = -1,
288 },
289 /* Just make sure that JITs used udiv/umod as otherwise we get
290 * an exception from INT_MIN/-1 overflow similarly as with div
291 * by zero.
292 */
293 {
294 "DIV32 overflow, check 1",
295 .insns = {
296 BPF_MOV32_IMM(BPF_REG_1, -1),
297 BPF_MOV32_IMM(BPF_REG_0, INT_MIN),
298 BPF_ALU32_REG(BPF_DIV, BPF_REG_0, BPF_REG_1),
299 BPF_EXIT_INSN(),
300 },
301 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
302 .result = ACCEPT,
303 .retval = 0,
304 },
305 {
306 "DIV32 overflow, check 2",
307 .insns = {
308 BPF_MOV32_IMM(BPF_REG_0, INT_MIN),
309 BPF_ALU32_IMM(BPF_DIV, BPF_REG_0, -1),
310 BPF_EXIT_INSN(),
311 },
312 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
313 .result = ACCEPT,
314 .retval = 0,
315 },
316 {
317 "DIV64 overflow, check 1",
318 .insns = {
319 BPF_MOV64_IMM(BPF_REG_1, -1),
320 BPF_LD_IMM64(BPF_REG_0, LLONG_MIN),
321 BPF_ALU64_REG(BPF_DIV, BPF_REG_0, BPF_REG_1),
322 BPF_EXIT_INSN(),
323 },
324 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
325 .result = ACCEPT,
326 .retval = 0,
327 },
328 {
329 "DIV64 overflow, check 2",
330 .insns = {
331 BPF_LD_IMM64(BPF_REG_0, LLONG_MIN),
332 BPF_ALU64_IMM(BPF_DIV, BPF_REG_0, -1),
333 BPF_EXIT_INSN(),
334 },
335 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
336 .result = ACCEPT,
337 .retval = 0,
338 },
339 {
340 "MOD32 overflow, check 1",
341 .insns = {
342 BPF_MOV32_IMM(BPF_REG_1, -1),
343 BPF_MOV32_IMM(BPF_REG_0, INT_MIN),
344 BPF_ALU32_REG(BPF_MOD, BPF_REG_0, BPF_REG_1),
345 BPF_EXIT_INSN(),
346 },
347 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
348 .result = ACCEPT,
349 .retval = INT_MIN,
350 },
351 {
352 "MOD32 overflow, check 2",
353 .insns = {
354 BPF_MOV32_IMM(BPF_REG_0, INT_MIN),
355 BPF_ALU32_IMM(BPF_MOD, BPF_REG_0, -1),
356 BPF_EXIT_INSN(),
357 },
358 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
359 .result = ACCEPT,
360 .retval = INT_MIN,
361 },
362 {
363 "MOD64 overflow, check 1",
364 .insns = {
365 BPF_MOV64_IMM(BPF_REG_1, -1),
366 BPF_LD_IMM64(BPF_REG_2, LLONG_MIN),
367 BPF_MOV64_REG(BPF_REG_3, BPF_REG_2),
368 BPF_ALU64_REG(BPF_MOD, BPF_REG_2, BPF_REG_1),
369 BPF_MOV32_IMM(BPF_REG_0, 0),
370 BPF_JMP_REG(BPF_JNE, BPF_REG_3, BPF_REG_2, 1),
371 BPF_MOV32_IMM(BPF_REG_0, 1),
372 BPF_EXIT_INSN(),
373 },
374 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
375 .result = ACCEPT,
376 .retval = 1,
377 },
378 {
379 "MOD64 overflow, check 2",
380 .insns = {
381 BPF_LD_IMM64(BPF_REG_2, LLONG_MIN),
382 BPF_MOV64_REG(BPF_REG_3, BPF_REG_2),
383 BPF_ALU64_IMM(BPF_MOD, BPF_REG_2, -1),
384 BPF_MOV32_IMM(BPF_REG_0, 0),
385 BPF_JMP_REG(BPF_JNE, BPF_REG_3, BPF_REG_2, 1),
386 BPF_MOV32_IMM(BPF_REG_0, 1),
387 BPF_EXIT_INSN(),
388 },
389 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
390 .result = ACCEPT,
391 .retval = 1,
392 },
393 {
394 "xor32 zero extend check",
395 .insns = {
396 BPF_MOV32_IMM(BPF_REG_2, -1),
397 BPF_ALU64_IMM(BPF_LSH, BPF_REG_2, 32),
398 BPF_ALU64_IMM(BPF_OR, BPF_REG_2, 0xffff),
399 BPF_ALU32_REG(BPF_XOR, BPF_REG_2, BPF_REG_2),
400 BPF_MOV32_IMM(BPF_REG_0, 2),
401 BPF_JMP_IMM(BPF_JNE, BPF_REG_2, 0, 1),
402 BPF_MOV32_IMM(BPF_REG_0, 1),
403 BPF_EXIT_INSN(),
404 },
405 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
406 .result = ACCEPT,
407 .retval = 1,
408 },
409 {
410 "empty prog",
411 .insns = {
412 },
413 .errstr = "unknown opcode 00",
414 .result = REJECT,
415 },
416 {
417 "only exit insn",
418 .insns = {
419 BPF_EXIT_INSN(),
420 },
421 .errstr = "R0 !read_ok",
422 .result = REJECT,
97 }, 423 },
98 { 424 {
99 "unreachable", 425 "unreachable",
@@ -209,6 +535,7 @@ static struct bpf_test tests[] = {
209 BPF_EXIT_INSN(), 535 BPF_EXIT_INSN(),
210 }, 536 },
211 .result = ACCEPT, 537 .result = ACCEPT,
538 .retval = 1,
212 }, 539 },
213 { 540 {
214 "test8 ld_imm64", 541 "test8 ld_imm64",
@@ -280,7 +607,7 @@ static struct bpf_test tests[] = {
280 BPF_EXIT_INSN(), 607 BPF_EXIT_INSN(),
281 }, 608 },
282 .result = REJECT, 609 .result = REJECT,
283 .errstr = "BPF_ARSH not supported for 32 bit ALU", 610 .errstr = "unknown opcode c4",
284 }, 611 },
285 { 612 {
286 "arsh32 on reg", 613 "arsh32 on reg",
@@ -291,7 +618,7 @@ static struct bpf_test tests[] = {
291 BPF_EXIT_INSN(), 618 BPF_EXIT_INSN(),
292 }, 619 },
293 .result = REJECT, 620 .result = REJECT,
294 .errstr = "BPF_ARSH not supported for 32 bit ALU", 621 .errstr = "unknown opcode cc",
295 }, 622 },
296 { 623 {
297 "arsh64 on imm", 624 "arsh64 on imm",
@@ -317,7 +644,7 @@ static struct bpf_test tests[] = {
317 .insns = { 644 .insns = {
318 BPF_ALU64_REG(BPF_MOV, BPF_REG_0, BPF_REG_2), 645 BPF_ALU64_REG(BPF_MOV, BPF_REG_0, BPF_REG_2),
319 }, 646 },
320 .errstr = "jump out of range", 647 .errstr = "not an exit",
321 .result = REJECT, 648 .result = REJECT,
322 }, 649 },
323 { 650 {
@@ -407,7 +734,7 @@ static struct bpf_test tests[] = {
407 BPF_RAW_INSN(BPF_JMP | BPF_CALL | BPF_X, 0, 0, 0, 0), 734 BPF_RAW_INSN(BPF_JMP | BPF_CALL | BPF_X, 0, 0, 0, 0),
408 BPF_EXIT_INSN(), 735 BPF_EXIT_INSN(),
409 }, 736 },
410 .errstr = "BPF_CALL uses reserved", 737 .errstr = "unknown opcode 8d",
411 .result = REJECT, 738 .result = REJECT,
412 }, 739 },
413 { 740 {
@@ -516,6 +843,7 @@ static struct bpf_test tests[] = {
516 .errstr_unpriv = "R0 leaks addr", 843 .errstr_unpriv = "R0 leaks addr",
517 .result = ACCEPT, 844 .result = ACCEPT,
518 .result_unpriv = REJECT, 845 .result_unpriv = REJECT,
846 .retval = POINTER_VALUE,
519 }, 847 },
520 { 848 {
521 "check valid spill/fill, skb mark", 849 "check valid spill/fill, skb mark",
@@ -596,7 +924,7 @@ static struct bpf_test tests[] = {
596 BPF_RAW_INSN(0, 0, 0, 0, 0), 924 BPF_RAW_INSN(0, 0, 0, 0, 0),
597 BPF_EXIT_INSN(), 925 BPF_EXIT_INSN(),
598 }, 926 },
599 .errstr = "invalid BPF_LD_IMM", 927 .errstr = "unknown opcode 00",
600 .result = REJECT, 928 .result = REJECT,
601 }, 929 },
602 { 930 {
@@ -614,7 +942,7 @@ static struct bpf_test tests[] = {
614 BPF_RAW_INSN(-1, 0, 0, 0, 0), 942 BPF_RAW_INSN(-1, 0, 0, 0, 0),
615 BPF_EXIT_INSN(), 943 BPF_EXIT_INSN(),
616 }, 944 },
617 .errstr = "invalid BPF_ALU opcode f0", 945 .errstr = "unknown opcode ff",
618 .result = REJECT, 946 .result = REJECT,
619 }, 947 },
620 { 948 {
@@ -623,7 +951,7 @@ static struct bpf_test tests[] = {
623 BPF_RAW_INSN(-1, -1, -1, -1, -1), 951 BPF_RAW_INSN(-1, -1, -1, -1, -1),
624 BPF_EXIT_INSN(), 952 BPF_EXIT_INSN(),
625 }, 953 },
626 .errstr = "invalid BPF_ALU opcode f0", 954 .errstr = "unknown opcode ff",
627 .result = REJECT, 955 .result = REJECT,
628 }, 956 },
629 { 957 {
@@ -802,6 +1130,7 @@ static struct bpf_test tests[] = {
802 .errstr_unpriv = "R1 pointer comparison", 1130 .errstr_unpriv = "R1 pointer comparison",
803 .result_unpriv = REJECT, 1131 .result_unpriv = REJECT,
804 .result = ACCEPT, 1132 .result = ACCEPT,
1133 .retval = -ENOENT,
805 }, 1134 },
806 { 1135 {
807 "jump test 4", 1136 "jump test 4",
@@ -1822,6 +2151,7 @@ static struct bpf_test tests[] = {
1822 BPF_EXIT_INSN(), 2151 BPF_EXIT_INSN(),
1823 }, 2152 },
1824 .result = ACCEPT, 2153 .result = ACCEPT,
2154 .retval = 0xfaceb00c,
1825 }, 2155 },
1826 { 2156 {
1827 "PTR_TO_STACK store/load - bad alignment on off", 2157 "PTR_TO_STACK store/load - bad alignment on off",
@@ -1880,6 +2210,7 @@ static struct bpf_test tests[] = {
1880 .result = ACCEPT, 2210 .result = ACCEPT,
1881 .result_unpriv = REJECT, 2211 .result_unpriv = REJECT,
1882 .errstr_unpriv = "R0 leaks addr", 2212 .errstr_unpriv = "R0 leaks addr",
2213 .retval = POINTER_VALUE,
1883 }, 2214 },
1884 { 2215 {
1885 "unpriv: add const to pointer", 2216 "unpriv: add const to pointer",
@@ -2053,6 +2384,7 @@ static struct bpf_test tests[] = {
2053 BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_6, 0), 2384 BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_6, 0),
2054 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, 2385 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
2055 BPF_FUNC_get_hash_recalc), 2386 BPF_FUNC_get_hash_recalc),
2387 BPF_MOV64_IMM(BPF_REG_0, 0),
2056 BPF_EXIT_INSN(), 2388 BPF_EXIT_INSN(),
2057 }, 2389 },
2058 .result = ACCEPT, 2390 .result = ACCEPT,
@@ -2840,6 +3172,7 @@ static struct bpf_test tests[] = {
2840 }, 3172 },
2841 .result = ACCEPT, 3173 .result = ACCEPT,
2842 .prog_type = BPF_PROG_TYPE_SCHED_CLS, 3174 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
3175 .retval = 1,
2843 }, 3176 },
2844 { 3177 {
2845 "direct packet access: test12 (and, good access)", 3178 "direct packet access: test12 (and, good access)",
@@ -2864,6 +3197,7 @@ static struct bpf_test tests[] = {
2864 }, 3197 },
2865 .result = ACCEPT, 3198 .result = ACCEPT,
2866 .prog_type = BPF_PROG_TYPE_SCHED_CLS, 3199 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
3200 .retval = 1,
2867 }, 3201 },
2868 { 3202 {
2869 "direct packet access: test13 (branches, good access)", 3203 "direct packet access: test13 (branches, good access)",
@@ -2894,6 +3228,7 @@ static struct bpf_test tests[] = {
2894 }, 3228 },
2895 .result = ACCEPT, 3229 .result = ACCEPT,
2896 .prog_type = BPF_PROG_TYPE_SCHED_CLS, 3230 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
3231 .retval = 1,
2897 }, 3232 },
2898 { 3233 {
2899 "direct packet access: test14 (pkt_ptr += 0, CONST_IMM, good access)", 3234 "direct packet access: test14 (pkt_ptr += 0, CONST_IMM, good access)",
@@ -2917,6 +3252,7 @@ static struct bpf_test tests[] = {
2917 }, 3252 },
2918 .result = ACCEPT, 3253 .result = ACCEPT,
2919 .prog_type = BPF_PROG_TYPE_SCHED_CLS, 3254 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
3255 .retval = 1,
2920 }, 3256 },
2921 { 3257 {
2922 "direct packet access: test15 (spill with xadd)", 3258 "direct packet access: test15 (spill with xadd)",
@@ -3203,6 +3539,7 @@ static struct bpf_test tests[] = {
3203 }, 3539 },
3204 .result = ACCEPT, 3540 .result = ACCEPT,
3205 .prog_type = BPF_PROG_TYPE_SCHED_CLS, 3541 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
3542 .retval = 1,
3206 }, 3543 },
3207 { 3544 {
3208 "direct packet access: test28 (marking on <=, bad access)", 3545 "direct packet access: test28 (marking on <=, bad access)",
@@ -5700,7 +6037,7 @@ static struct bpf_test tests[] = {
5700 "helper access to variable memory: size > 0 not allowed on NULL (ARG_PTR_TO_MEM_OR_NULL)", 6037 "helper access to variable memory: size > 0 not allowed on NULL (ARG_PTR_TO_MEM_OR_NULL)",
5701 .insns = { 6038 .insns = {
5702 BPF_MOV64_IMM(BPF_REG_1, 0), 6039 BPF_MOV64_IMM(BPF_REG_1, 0),
5703 BPF_MOV64_IMM(BPF_REG_2, 0), 6040 BPF_MOV64_IMM(BPF_REG_2, 1),
5704 BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_2, -128), 6041 BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_2, -128),
5705 BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_10, -128), 6042 BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_10, -128),
5706 BPF_ALU64_IMM(BPF_AND, BPF_REG_2, 64), 6043 BPF_ALU64_IMM(BPF_AND, BPF_REG_2, 64),
@@ -5822,6 +6159,7 @@ static struct bpf_test tests[] = {
5822 }, 6159 },
5823 .result = ACCEPT, 6160 .result = ACCEPT,
5824 .prog_type = BPF_PROG_TYPE_SCHED_CLS, 6161 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
6162 .retval = 0 /* csum_diff of 64-byte packet */,
5825 }, 6163 },
5826 { 6164 {
5827 "helper access to variable memory: size = 0 not allowed on NULL (!ARG_PTR_TO_MEM_OR_NULL)", 6165 "helper access to variable memory: size = 0 not allowed on NULL (!ARG_PTR_TO_MEM_OR_NULL)",
@@ -5935,7 +6273,7 @@ static struct bpf_test tests[] = {
5935 BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, -24), 6273 BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, -24),
5936 BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, -16), 6274 BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, -16),
5937 BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, -8), 6275 BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, -8),
5938 BPF_MOV64_IMM(BPF_REG_2, 0), 6276 BPF_MOV64_IMM(BPF_REG_2, 1),
5939 BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_2, -128), 6277 BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_2, -128),
5940 BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_10, -128), 6278 BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_10, -128),
5941 BPF_ALU64_IMM(BPF_AND, BPF_REG_2, 63), 6279 BPF_ALU64_IMM(BPF_AND, BPF_REG_2, 63),
@@ -6190,6 +6528,7 @@ static struct bpf_test tests[] = {
6190 }, 6528 },
6191 .prog_type = BPF_PROG_TYPE_SCHED_CLS, 6529 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
6192 .result = ACCEPT, 6530 .result = ACCEPT,
6531 .retval = 42 /* ultimate return value */,
6193 }, 6532 },
6194 { 6533 {
6195 "ld_ind: check calling conv, r1", 6534 "ld_ind: check calling conv, r1",
@@ -6261,6 +6600,7 @@ static struct bpf_test tests[] = {
6261 BPF_EXIT_INSN(), 6600 BPF_EXIT_INSN(),
6262 }, 6601 },
6263 .result = ACCEPT, 6602 .result = ACCEPT,
6603 .retval = 1,
6264 }, 6604 },
6265 { 6605 {
6266 "check bpf_perf_event_data->sample_period byte load permitted", 6606 "check bpf_perf_event_data->sample_period byte load permitted",
@@ -7248,6 +7588,7 @@ static struct bpf_test tests[] = {
7248 }, 7588 },
7249 .fixup_map1 = { 3 }, 7589 .fixup_map1 = { 3 },
7250 .result = ACCEPT, 7590 .result = ACCEPT,
7591 .retval = POINTER_VALUE,
7251 .result_unpriv = REJECT, 7592 .result_unpriv = REJECT,
7252 .errstr_unpriv = "R0 leaks addr as return value" 7593 .errstr_unpriv = "R0 leaks addr as return value"
7253 }, 7594 },
@@ -7268,6 +7609,7 @@ static struct bpf_test tests[] = {
7268 }, 7609 },
7269 .fixup_map1 = { 3 }, 7610 .fixup_map1 = { 3 },
7270 .result = ACCEPT, 7611 .result = ACCEPT,
7612 .retval = POINTER_VALUE,
7271 .result_unpriv = REJECT, 7613 .result_unpriv = REJECT,
7272 .errstr_unpriv = "R0 leaks addr as return value" 7614 .errstr_unpriv = "R0 leaks addr as return value"
7273 }, 7615 },
@@ -7434,7 +7776,7 @@ static struct bpf_test tests[] = {
7434 }, 7776 },
7435 BPF_EXIT_INSN(), 7777 BPF_EXIT_INSN(),
7436 }, 7778 },
7437 .errstr = "BPF_END uses reserved fields", 7779 .errstr = "unknown opcode d7",
7438 .result = REJECT, 7780 .result = REJECT,
7439 }, 7781 },
7440 { 7782 {
@@ -7709,6 +8051,7 @@ static struct bpf_test tests[] = {
7709 BPF_EXIT_INSN(), 8051 BPF_EXIT_INSN(),
7710 }, 8052 },
7711 .result = ACCEPT, 8053 .result = ACCEPT,
8054 .retval = TEST_DATA_LEN,
7712 .prog_type = BPF_PROG_TYPE_SCHED_CLS, 8055 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
7713 }, 8056 },
7714 { 8057 {
@@ -8656,6 +8999,7 @@ static struct bpf_test tests[] = {
8656 BPF_EXIT_INSN(), 8999 BPF_EXIT_INSN(),
8657 }, 9000 },
8658 .result = ACCEPT, 9001 .result = ACCEPT,
9002 .retval = 1,
8659 }, 9003 },
8660 { 9004 {
8661 "check deducing bounds from const, 3", 9005 "check deducing bounds from const, 3",
@@ -8826,6 +9170,1959 @@ static struct bpf_test tests[] = {
8826 .result = REJECT, 9170 .result = REJECT,
8827 .prog_type = BPF_PROG_TYPE_CGROUP_SOCK, 9171 .prog_type = BPF_PROG_TYPE_CGROUP_SOCK,
8828 }, 9172 },
9173 {
9174 "calls: basic sanity",
9175 .insns = {
9176 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 2),
9177 BPF_MOV64_IMM(BPF_REG_0, 1),
9178 BPF_EXIT_INSN(),
9179 BPF_MOV64_IMM(BPF_REG_0, 2),
9180 BPF_EXIT_INSN(),
9181 },
9182 .prog_type = BPF_PROG_TYPE_TRACEPOINT,
9183 .result = ACCEPT,
9184 },
9185 {
9186 "calls: not on unpriviledged",
9187 .insns = {
9188 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 2),
9189 BPF_MOV64_IMM(BPF_REG_0, 1),
9190 BPF_EXIT_INSN(),
9191 BPF_MOV64_IMM(BPF_REG_0, 2),
9192 BPF_EXIT_INSN(),
9193 },
9194 .errstr_unpriv = "function calls to other bpf functions are allowed for root only",
9195 .result_unpriv = REJECT,
9196 .result = ACCEPT,
9197 .retval = 1,
9198 },
9199 {
9200 "calls: div by 0 in subprog",
9201 .insns = {
9202 BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
9203 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 8),
9204 BPF_MOV64_REG(BPF_REG_1, BPF_REG_6),
9205 BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_1,
9206 offsetof(struct __sk_buff, data_end)),
9207 BPF_MOV64_REG(BPF_REG_2, BPF_REG_0),
9208 BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, 8),
9209 BPF_JMP_REG(BPF_JGT, BPF_REG_2, BPF_REG_1, 1),
9210 BPF_LDX_MEM(BPF_B, BPF_REG_0, BPF_REG_0, 0),
9211 BPF_MOV64_IMM(BPF_REG_0, 1),
9212 BPF_EXIT_INSN(),
9213 BPF_MOV32_IMM(BPF_REG_2, 0),
9214 BPF_MOV32_IMM(BPF_REG_3, 1),
9215 BPF_ALU32_REG(BPF_DIV, BPF_REG_3, BPF_REG_2),
9216 BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_1,
9217 offsetof(struct __sk_buff, data)),
9218 BPF_EXIT_INSN(),
9219 },
9220 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
9221 .result = ACCEPT,
9222 .retval = 1,
9223 },
9224 {
9225 "calls: multiple ret types in subprog 1",
9226 .insns = {
9227 BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
9228 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 8),
9229 BPF_MOV64_REG(BPF_REG_1, BPF_REG_6),
9230 BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_1,
9231 offsetof(struct __sk_buff, data_end)),
9232 BPF_MOV64_REG(BPF_REG_2, BPF_REG_0),
9233 BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, 8),
9234 BPF_JMP_REG(BPF_JGT, BPF_REG_2, BPF_REG_1, 1),
9235 BPF_LDX_MEM(BPF_B, BPF_REG_0, BPF_REG_0, 0),
9236 BPF_MOV64_IMM(BPF_REG_0, 1),
9237 BPF_EXIT_INSN(),
9238 BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_1,
9239 offsetof(struct __sk_buff, data)),
9240 BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 1),
9241 BPF_MOV32_IMM(BPF_REG_0, 42),
9242 BPF_EXIT_INSN(),
9243 },
9244 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
9245 .result = REJECT,
9246 .errstr = "R0 invalid mem access 'inv'",
9247 },
9248 {
9249 "calls: multiple ret types in subprog 2",
9250 .insns = {
9251 BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
9252 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 8),
9253 BPF_MOV64_REG(BPF_REG_1, BPF_REG_6),
9254 BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_1,
9255 offsetof(struct __sk_buff, data_end)),
9256 BPF_MOV64_REG(BPF_REG_2, BPF_REG_0),
9257 BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, 8),
9258 BPF_JMP_REG(BPF_JGT, BPF_REG_2, BPF_REG_1, 1),
9259 BPF_LDX_MEM(BPF_B, BPF_REG_0, BPF_REG_0, 0),
9260 BPF_MOV64_IMM(BPF_REG_0, 1),
9261 BPF_EXIT_INSN(),
9262 BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_1,
9263 offsetof(struct __sk_buff, data)),
9264 BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
9265 BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 9),
9266 BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0),
9267 BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
9268 BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
9269 BPF_LD_MAP_FD(BPF_REG_1, 0),
9270 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
9271 BPF_FUNC_map_lookup_elem),
9272 BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 1),
9273 BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_6,
9274 offsetof(struct __sk_buff, data)),
9275 BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 64),
9276 BPF_EXIT_INSN(),
9277 },
9278 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
9279 .fixup_map1 = { 16 },
9280 .result = REJECT,
9281 .errstr = "R0 min value is outside of the array range",
9282 },
9283 {
9284 "calls: overlapping caller/callee",
9285 .insns = {
9286 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 0),
9287 BPF_MOV64_IMM(BPF_REG_0, 1),
9288 BPF_EXIT_INSN(),
9289 },
9290 .prog_type = BPF_PROG_TYPE_TRACEPOINT,
9291 .errstr = "last insn is not an exit or jmp",
9292 .result = REJECT,
9293 },
9294 {
9295 "calls: wrong recursive calls",
9296 .insns = {
9297 BPF_JMP_IMM(BPF_JA, 0, 0, 4),
9298 BPF_JMP_IMM(BPF_JA, 0, 0, 4),
9299 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, -2),
9300 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, -2),
9301 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, -2),
9302 BPF_MOV64_IMM(BPF_REG_0, 1),
9303 BPF_EXIT_INSN(),
9304 },
9305 .prog_type = BPF_PROG_TYPE_TRACEPOINT,
9306 .errstr = "jump out of range",
9307 .result = REJECT,
9308 },
9309 {
9310 "calls: wrong src reg",
9311 .insns = {
9312 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 2, 0, 0),
9313 BPF_MOV64_IMM(BPF_REG_0, 1),
9314 BPF_EXIT_INSN(),
9315 },
9316 .prog_type = BPF_PROG_TYPE_TRACEPOINT,
9317 .errstr = "BPF_CALL uses reserved fields",
9318 .result = REJECT,
9319 },
9320 {
9321 "calls: wrong off value",
9322 .insns = {
9323 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, -1, 2),
9324 BPF_MOV64_IMM(BPF_REG_0, 1),
9325 BPF_EXIT_INSN(),
9326 BPF_MOV64_IMM(BPF_REG_0, 2),
9327 BPF_EXIT_INSN(),
9328 },
9329 .prog_type = BPF_PROG_TYPE_TRACEPOINT,
9330 .errstr = "BPF_CALL uses reserved fields",
9331 .result = REJECT,
9332 },
9333 {
9334 "calls: jump back loop",
9335 .insns = {
9336 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, -1),
9337 BPF_MOV64_IMM(BPF_REG_0, 1),
9338 BPF_EXIT_INSN(),
9339 },
9340 .prog_type = BPF_PROG_TYPE_TRACEPOINT,
9341 .errstr = "back-edge from insn 0 to 0",
9342 .result = REJECT,
9343 },
9344 {
9345 "calls: conditional call",
9346 .insns = {
9347 BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_1,
9348 offsetof(struct __sk_buff, mark)),
9349 BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 3),
9350 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 2),
9351 BPF_MOV64_IMM(BPF_REG_0, 1),
9352 BPF_EXIT_INSN(),
9353 BPF_MOV64_IMM(BPF_REG_0, 2),
9354 BPF_EXIT_INSN(),
9355 },
9356 .prog_type = BPF_PROG_TYPE_TRACEPOINT,
9357 .errstr = "jump out of range",
9358 .result = REJECT,
9359 },
9360 {
9361 "calls: conditional call 2",
9362 .insns = {
9363 BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_1,
9364 offsetof(struct __sk_buff, mark)),
9365 BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 3),
9366 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 4),
9367 BPF_MOV64_IMM(BPF_REG_0, 1),
9368 BPF_EXIT_INSN(),
9369 BPF_MOV64_IMM(BPF_REG_0, 2),
9370 BPF_EXIT_INSN(),
9371 BPF_MOV64_IMM(BPF_REG_0, 3),
9372 BPF_EXIT_INSN(),
9373 },
9374 .prog_type = BPF_PROG_TYPE_TRACEPOINT,
9375 .result = ACCEPT,
9376 },
9377 {
9378 "calls: conditional call 3",
9379 .insns = {
9380 BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_1,
9381 offsetof(struct __sk_buff, mark)),
9382 BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 3),
9383 BPF_JMP_IMM(BPF_JA, 0, 0, 4),
9384 BPF_MOV64_IMM(BPF_REG_0, 1),
9385 BPF_EXIT_INSN(),
9386 BPF_MOV64_IMM(BPF_REG_0, 1),
9387 BPF_JMP_IMM(BPF_JA, 0, 0, -6),
9388 BPF_MOV64_IMM(BPF_REG_0, 3),
9389 BPF_JMP_IMM(BPF_JA, 0, 0, -6),
9390 },
9391 .prog_type = BPF_PROG_TYPE_TRACEPOINT,
9392 .errstr = "back-edge from insn",
9393 .result = REJECT,
9394 },
9395 {
9396 "calls: conditional call 4",
9397 .insns = {
9398 BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_1,
9399 offsetof(struct __sk_buff, mark)),
9400 BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 3),
9401 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 4),
9402 BPF_MOV64_IMM(BPF_REG_0, 1),
9403 BPF_EXIT_INSN(),
9404 BPF_MOV64_IMM(BPF_REG_0, 1),
9405 BPF_JMP_IMM(BPF_JA, 0, 0, -5),
9406 BPF_MOV64_IMM(BPF_REG_0, 3),
9407 BPF_EXIT_INSN(),
9408 },
9409 .prog_type = BPF_PROG_TYPE_TRACEPOINT,
9410 .result = ACCEPT,
9411 },
9412 {
9413 "calls: conditional call 5",
9414 .insns = {
9415 BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_1,
9416 offsetof(struct __sk_buff, mark)),
9417 BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 3),
9418 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 4),
9419 BPF_MOV64_IMM(BPF_REG_0, 1),
9420 BPF_EXIT_INSN(),
9421 BPF_MOV64_IMM(BPF_REG_0, 1),
9422 BPF_JMP_IMM(BPF_JA, 0, 0, -6),
9423 BPF_MOV64_IMM(BPF_REG_0, 3),
9424 BPF_EXIT_INSN(),
9425 },
9426 .prog_type = BPF_PROG_TYPE_TRACEPOINT,
9427 .errstr = "back-edge from insn",
9428 .result = REJECT,
9429 },
9430 {
9431 "calls: conditional call 6",
9432 .insns = {
9433 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 2),
9434 BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, -2),
9435 BPF_EXIT_INSN(),
9436 BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_1,
9437 offsetof(struct __sk_buff, mark)),
9438 BPF_EXIT_INSN(),
9439 },
9440 .prog_type = BPF_PROG_TYPE_TRACEPOINT,
9441 .errstr = "back-edge from insn",
9442 .result = REJECT,
9443 },
9444 {
9445 "calls: using r0 returned by callee",
9446 .insns = {
9447 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 1),
9448 BPF_EXIT_INSN(),
9449 BPF_MOV64_IMM(BPF_REG_0, 2),
9450 BPF_EXIT_INSN(),
9451 },
9452 .prog_type = BPF_PROG_TYPE_TRACEPOINT,
9453 .result = ACCEPT,
9454 },
9455 {
9456 "calls: using uninit r0 from callee",
9457 .insns = {
9458 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 1),
9459 BPF_EXIT_INSN(),
9460 BPF_EXIT_INSN(),
9461 },
9462 .prog_type = BPF_PROG_TYPE_TRACEPOINT,
9463 .errstr = "!read_ok",
9464 .result = REJECT,
9465 },
9466 {
9467 "calls: callee is using r1",
9468 .insns = {
9469 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 1),
9470 BPF_EXIT_INSN(),
9471 BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_1,
9472 offsetof(struct __sk_buff, len)),
9473 BPF_EXIT_INSN(),
9474 },
9475 .prog_type = BPF_PROG_TYPE_SCHED_ACT,
9476 .result = ACCEPT,
9477 .retval = TEST_DATA_LEN,
9478 },
9479 {
9480 "calls: callee using args1",
9481 .insns = {
9482 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 1),
9483 BPF_EXIT_INSN(),
9484 BPF_MOV64_REG(BPF_REG_0, BPF_REG_1),
9485 BPF_EXIT_INSN(),
9486 },
9487 .errstr_unpriv = "allowed for root only",
9488 .result_unpriv = REJECT,
9489 .result = ACCEPT,
9490 .retval = POINTER_VALUE,
9491 },
9492 {
9493 "calls: callee using wrong args2",
9494 .insns = {
9495 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 1),
9496 BPF_EXIT_INSN(),
9497 BPF_MOV64_REG(BPF_REG_0, BPF_REG_2),
9498 BPF_EXIT_INSN(),
9499 },
9500 .prog_type = BPF_PROG_TYPE_TRACEPOINT,
9501 .errstr = "R2 !read_ok",
9502 .result = REJECT,
9503 },
9504 {
9505 "calls: callee using two args",
9506 .insns = {
9507 BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
9508 BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_6,
9509 offsetof(struct __sk_buff, len)),
9510 BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_6,
9511 offsetof(struct __sk_buff, len)),
9512 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 1),
9513 BPF_EXIT_INSN(),
9514 BPF_MOV64_REG(BPF_REG_0, BPF_REG_1),
9515 BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_2),
9516 BPF_EXIT_INSN(),
9517 },
9518 .errstr_unpriv = "allowed for root only",
9519 .result_unpriv = REJECT,
9520 .result = ACCEPT,
9521 .retval = TEST_DATA_LEN + TEST_DATA_LEN - ETH_HLEN - ETH_HLEN,
9522 },
9523 {
9524 "calls: callee changing pkt pointers",
9525 .insns = {
9526 BPF_LDX_MEM(BPF_W, BPF_REG_6, BPF_REG_1,
9527 offsetof(struct xdp_md, data)),
9528 BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_1,
9529 offsetof(struct xdp_md, data_end)),
9530 BPF_MOV64_REG(BPF_REG_8, BPF_REG_6),
9531 BPF_ALU64_IMM(BPF_ADD, BPF_REG_8, 8),
9532 BPF_JMP_REG(BPF_JGT, BPF_REG_8, BPF_REG_7, 2),
9533 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 3),
9534 /* clear_all_pkt_pointers() has to walk all frames
9535 * to make sure that pkt pointers in the caller
9536 * are cleared when callee is calling a helper that
9537 * adjusts packet size
9538 */
9539 BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_6, 0),
9540 BPF_MOV32_IMM(BPF_REG_0, 0),
9541 BPF_EXIT_INSN(),
9542 BPF_MOV64_IMM(BPF_REG_2, 0),
9543 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
9544 BPF_FUNC_xdp_adjust_head),
9545 BPF_EXIT_INSN(),
9546 },
9547 .result = REJECT,
9548 .errstr = "R6 invalid mem access 'inv'",
9549 .prog_type = BPF_PROG_TYPE_XDP,
9550 },
9551 {
9552 "calls: two calls with args",
9553 .insns = {
9554 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 1),
9555 BPF_EXIT_INSN(),
9556 BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
9557 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 6),
9558 BPF_MOV64_REG(BPF_REG_7, BPF_REG_0),
9559 BPF_MOV64_REG(BPF_REG_1, BPF_REG_6),
9560 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 3),
9561 BPF_ALU64_REG(BPF_ADD, BPF_REG_7, BPF_REG_0),
9562 BPF_MOV64_REG(BPF_REG_0, BPF_REG_7),
9563 BPF_EXIT_INSN(),
9564 BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_1,
9565 offsetof(struct __sk_buff, len)),
9566 BPF_EXIT_INSN(),
9567 },
9568 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
9569 .result = ACCEPT,
9570 .retval = TEST_DATA_LEN + TEST_DATA_LEN,
9571 },
9572 {
9573 "calls: calls with stack arith",
9574 .insns = {
9575 BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
9576 BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -64),
9577 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 1),
9578 BPF_EXIT_INSN(),
9579 BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -64),
9580 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 1),
9581 BPF_EXIT_INSN(),
9582 BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -64),
9583 BPF_MOV64_IMM(BPF_REG_0, 42),
9584 BPF_STX_MEM(BPF_DW, BPF_REG_2, BPF_REG_0, 0),
9585 BPF_EXIT_INSN(),
9586 },
9587 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
9588 .result = ACCEPT,
9589 .retval = 42,
9590 },
9591 {
9592 "calls: calls with misaligned stack access",
9593 .insns = {
9594 BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
9595 BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -63),
9596 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 1),
9597 BPF_EXIT_INSN(),
9598 BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -61),
9599 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 1),
9600 BPF_EXIT_INSN(),
9601 BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -63),
9602 BPF_MOV64_IMM(BPF_REG_0, 42),
9603 BPF_STX_MEM(BPF_DW, BPF_REG_2, BPF_REG_0, 0),
9604 BPF_EXIT_INSN(),
9605 },
9606 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
9607 .flags = F_LOAD_WITH_STRICT_ALIGNMENT,
9608 .errstr = "misaligned stack access",
9609 .result = REJECT,
9610 },
9611 {
9612 "calls: calls control flow, jump test",
9613 .insns = {
9614 BPF_MOV64_IMM(BPF_REG_0, 42),
9615 BPF_JMP_IMM(BPF_JA, 0, 0, 2),
9616 BPF_MOV64_IMM(BPF_REG_0, 43),
9617 BPF_JMP_IMM(BPF_JA, 0, 0, 1),
9618 BPF_JMP_IMM(BPF_JA, 0, 0, -3),
9619 BPF_EXIT_INSN(),
9620 },
9621 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
9622 .result = ACCEPT,
9623 .retval = 43,
9624 },
9625 {
9626 "calls: calls control flow, jump test 2",
9627 .insns = {
9628 BPF_MOV64_IMM(BPF_REG_0, 42),
9629 BPF_JMP_IMM(BPF_JA, 0, 0, 2),
9630 BPF_MOV64_IMM(BPF_REG_0, 43),
9631 BPF_JMP_IMM(BPF_JA, 0, 0, 1),
9632 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, -3),
9633 BPF_EXIT_INSN(),
9634 },
9635 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
9636 .errstr = "jump out of range from insn 1 to 4",
9637 .result = REJECT,
9638 },
9639 {
9640 "calls: two calls with bad jump",
9641 .insns = {
9642 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 1),
9643 BPF_EXIT_INSN(),
9644 BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
9645 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 6),
9646 BPF_MOV64_REG(BPF_REG_7, BPF_REG_0),
9647 BPF_MOV64_REG(BPF_REG_1, BPF_REG_6),
9648 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 3),
9649 BPF_ALU64_REG(BPF_ADD, BPF_REG_7, BPF_REG_0),
9650 BPF_MOV64_REG(BPF_REG_0, BPF_REG_7),
9651 BPF_EXIT_INSN(),
9652 BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_1,
9653 offsetof(struct __sk_buff, len)),
9654 BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, -3),
9655 BPF_EXIT_INSN(),
9656 },
9657 .prog_type = BPF_PROG_TYPE_TRACEPOINT,
9658 .errstr = "jump out of range from insn 11 to 9",
9659 .result = REJECT,
9660 },
9661 {
9662 "calls: recursive call. test1",
9663 .insns = {
9664 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 1),
9665 BPF_EXIT_INSN(),
9666 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, -1),
9667 BPF_EXIT_INSN(),
9668 },
9669 .prog_type = BPF_PROG_TYPE_TRACEPOINT,
9670 .errstr = "back-edge",
9671 .result = REJECT,
9672 },
9673 {
9674 "calls: recursive call. test2",
9675 .insns = {
9676 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 1),
9677 BPF_EXIT_INSN(),
9678 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, -3),
9679 BPF_EXIT_INSN(),
9680 },
9681 .prog_type = BPF_PROG_TYPE_TRACEPOINT,
9682 .errstr = "back-edge",
9683 .result = REJECT,
9684 },
9685 {
9686 "calls: unreachable code",
9687 .insns = {
9688 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 1),
9689 BPF_EXIT_INSN(),
9690 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 1),
9691 BPF_EXIT_INSN(),
9692 BPF_MOV64_IMM(BPF_REG_0, 0),
9693 BPF_EXIT_INSN(),
9694 BPF_MOV64_IMM(BPF_REG_0, 0),
9695 BPF_EXIT_INSN(),
9696 },
9697 .prog_type = BPF_PROG_TYPE_TRACEPOINT,
9698 .errstr = "unreachable insn 6",
9699 .result = REJECT,
9700 },
9701 {
9702 "calls: invalid call",
9703 .insns = {
9704 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 1),
9705 BPF_EXIT_INSN(),
9706 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, -4),
9707 BPF_EXIT_INSN(),
9708 },
9709 .prog_type = BPF_PROG_TYPE_TRACEPOINT,
9710 .errstr = "invalid destination",
9711 .result = REJECT,
9712 },
9713 {
9714 "calls: invalid call 2",
9715 .insns = {
9716 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 1),
9717 BPF_EXIT_INSN(),
9718 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 0x7fffffff),
9719 BPF_EXIT_INSN(),
9720 },
9721 .prog_type = BPF_PROG_TYPE_TRACEPOINT,
9722 .errstr = "invalid destination",
9723 .result = REJECT,
9724 },
9725 {
9726 "calls: jumping across function bodies. test1",
9727 .insns = {
9728 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 2),
9729 BPF_MOV64_IMM(BPF_REG_0, 0),
9730 BPF_EXIT_INSN(),
9731 BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, 0, -3),
9732 BPF_EXIT_INSN(),
9733 },
9734 .prog_type = BPF_PROG_TYPE_TRACEPOINT,
9735 .errstr = "jump out of range",
9736 .result = REJECT,
9737 },
9738 {
9739 "calls: jumping across function bodies. test2",
9740 .insns = {
9741 BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, 0, 3),
9742 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 2),
9743 BPF_MOV64_IMM(BPF_REG_0, 0),
9744 BPF_EXIT_INSN(),
9745 BPF_EXIT_INSN(),
9746 },
9747 .prog_type = BPF_PROG_TYPE_TRACEPOINT,
9748 .errstr = "jump out of range",
9749 .result = REJECT,
9750 },
9751 {
9752 "calls: call without exit",
9753 .insns = {
9754 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 1),
9755 BPF_EXIT_INSN(),
9756 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 1),
9757 BPF_EXIT_INSN(),
9758 BPF_MOV64_IMM(BPF_REG_0, 0),
9759 BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, 0, -2),
9760 },
9761 .prog_type = BPF_PROG_TYPE_TRACEPOINT,
9762 .errstr = "not an exit",
9763 .result = REJECT,
9764 },
9765 {
9766 "calls: call into middle of ld_imm64",
9767 .insns = {
9768 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 3),
9769 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 3),
9770 BPF_MOV64_IMM(BPF_REG_0, 0),
9771 BPF_EXIT_INSN(),
9772 BPF_LD_IMM64(BPF_REG_0, 0),
9773 BPF_EXIT_INSN(),
9774 },
9775 .prog_type = BPF_PROG_TYPE_TRACEPOINT,
9776 .errstr = "last insn",
9777 .result = REJECT,
9778 },
9779 {
9780 "calls: call into middle of other call",
9781 .insns = {
9782 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 3),
9783 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 3),
9784 BPF_MOV64_IMM(BPF_REG_0, 0),
9785 BPF_EXIT_INSN(),
9786 BPF_MOV64_IMM(BPF_REG_0, 0),
9787 BPF_MOV64_IMM(BPF_REG_0, 0),
9788 BPF_EXIT_INSN(),
9789 },
9790 .prog_type = BPF_PROG_TYPE_TRACEPOINT,
9791 .errstr = "last insn",
9792 .result = REJECT,
9793 },
9794 {
9795 "calls: ld_abs with changing ctx data in callee",
9796 .insns = {
9797 BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
9798 BPF_LD_ABS(BPF_B, 0),
9799 BPF_LD_ABS(BPF_H, 0),
9800 BPF_LD_ABS(BPF_W, 0),
9801 BPF_MOV64_REG(BPF_REG_7, BPF_REG_6),
9802 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 5),
9803 BPF_MOV64_REG(BPF_REG_6, BPF_REG_7),
9804 BPF_LD_ABS(BPF_B, 0),
9805 BPF_LD_ABS(BPF_H, 0),
9806 BPF_LD_ABS(BPF_W, 0),
9807 BPF_EXIT_INSN(),
9808 BPF_MOV64_IMM(BPF_REG_2, 1),
9809 BPF_MOV64_IMM(BPF_REG_3, 2),
9810 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
9811 BPF_FUNC_skb_vlan_push),
9812 BPF_EXIT_INSN(),
9813 },
9814 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
9815 .errstr = "BPF_LD_[ABS|IND] instructions cannot be mixed",
9816 .result = REJECT,
9817 },
9818 {
9819 "calls: two calls with bad fallthrough",
9820 .insns = {
9821 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 1),
9822 BPF_EXIT_INSN(),
9823 BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
9824 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 6),
9825 BPF_MOV64_REG(BPF_REG_7, BPF_REG_0),
9826 BPF_MOV64_REG(BPF_REG_1, BPF_REG_6),
9827 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 3),
9828 BPF_ALU64_REG(BPF_ADD, BPF_REG_7, BPF_REG_0),
9829 BPF_MOV64_REG(BPF_REG_0, BPF_REG_7),
9830 BPF_MOV64_REG(BPF_REG_0, BPF_REG_0),
9831 BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_1,
9832 offsetof(struct __sk_buff, len)),
9833 BPF_EXIT_INSN(),
9834 },
9835 .prog_type = BPF_PROG_TYPE_TRACEPOINT,
9836 .errstr = "not an exit",
9837 .result = REJECT,
9838 },
9839 {
9840 "calls: two calls with stack read",
9841 .insns = {
9842 BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0),
9843 BPF_MOV64_REG(BPF_REG_1, BPF_REG_10),
9844 BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -8),
9845 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 1),
9846 BPF_EXIT_INSN(),
9847 BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
9848 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 6),
9849 BPF_MOV64_REG(BPF_REG_7, BPF_REG_0),
9850 BPF_MOV64_REG(BPF_REG_1, BPF_REG_6),
9851 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 3),
9852 BPF_ALU64_REG(BPF_ADD, BPF_REG_7, BPF_REG_0),
9853 BPF_MOV64_REG(BPF_REG_0, BPF_REG_7),
9854 BPF_EXIT_INSN(),
9855 BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_1, 0),
9856 BPF_EXIT_INSN(),
9857 },
9858 .prog_type = BPF_PROG_TYPE_XDP,
9859 .result = ACCEPT,
9860 },
9861 {
9862 "calls: two calls with stack write",
9863 .insns = {
9864 /* main prog */
9865 BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0),
9866 BPF_MOV64_REG(BPF_REG_1, BPF_REG_10),
9867 BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -8),
9868 BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
9869 BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -16),
9870 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 2),
9871 BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_10, -16),
9872 BPF_EXIT_INSN(),
9873
9874 /* subprog 1 */
9875 BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
9876 BPF_MOV64_REG(BPF_REG_7, BPF_REG_2),
9877 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 7),
9878 BPF_MOV64_REG(BPF_REG_8, BPF_REG_0),
9879 BPF_MOV64_REG(BPF_REG_1, BPF_REG_6),
9880 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 4),
9881 BPF_ALU64_REG(BPF_ADD, BPF_REG_8, BPF_REG_0),
9882 BPF_MOV64_REG(BPF_REG_0, BPF_REG_8),
9883 /* write into stack frame of main prog */
9884 BPF_STX_MEM(BPF_DW, BPF_REG_7, BPF_REG_0, 0),
9885 BPF_EXIT_INSN(),
9886
9887 /* subprog 2 */
9888 /* read from stack frame of main prog */
9889 BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_1, 0),
9890 BPF_EXIT_INSN(),
9891 },
9892 .prog_type = BPF_PROG_TYPE_XDP,
9893 .result = ACCEPT,
9894 },
9895 {
9896 "calls: stack overflow using two frames (pre-call access)",
9897 .insns = {
9898 /* prog 1 */
9899 BPF_ST_MEM(BPF_B, BPF_REG_10, -300, 0),
9900 BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1),
9901 BPF_EXIT_INSN(),
9902
9903 /* prog 2 */
9904 BPF_ST_MEM(BPF_B, BPF_REG_10, -300, 0),
9905 BPF_MOV64_IMM(BPF_REG_0, 0),
9906 BPF_EXIT_INSN(),
9907 },
9908 .prog_type = BPF_PROG_TYPE_XDP,
9909 .errstr = "combined stack size",
9910 .result = REJECT,
9911 },
9912 {
9913 "calls: stack overflow using two frames (post-call access)",
9914 .insns = {
9915 /* prog 1 */
9916 BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 2),
9917 BPF_ST_MEM(BPF_B, BPF_REG_10, -300, 0),
9918 BPF_EXIT_INSN(),
9919
9920 /* prog 2 */
9921 BPF_ST_MEM(BPF_B, BPF_REG_10, -300, 0),
9922 BPF_MOV64_IMM(BPF_REG_0, 0),
9923 BPF_EXIT_INSN(),
9924 },
9925 .prog_type = BPF_PROG_TYPE_XDP,
9926 .errstr = "combined stack size",
9927 .result = REJECT,
9928 },
9929 {
9930 "calls: stack depth check using three frames. test1",
9931 .insns = {
9932 /* main */
9933 BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 4), /* call A */
9934 BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 5), /* call B */
9935 BPF_ST_MEM(BPF_B, BPF_REG_10, -32, 0),
9936 BPF_MOV64_IMM(BPF_REG_0, 0),
9937 BPF_EXIT_INSN(),
9938 /* A */
9939 BPF_ST_MEM(BPF_B, BPF_REG_10, -256, 0),
9940 BPF_EXIT_INSN(),
9941 /* B */
9942 BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, -3), /* call A */
9943 BPF_ST_MEM(BPF_B, BPF_REG_10, -64, 0),
9944 BPF_EXIT_INSN(),
9945 },
9946 .prog_type = BPF_PROG_TYPE_XDP,
9947 /* stack_main=32, stack_A=256, stack_B=64
9948 * and max(main+A, main+A+B) < 512
9949 */
9950 .result = ACCEPT,
9951 },
9952 {
9953 "calls: stack depth check using three frames. test2",
9954 .insns = {
9955 /* main */
9956 BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 4), /* call A */
9957 BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 5), /* call B */
9958 BPF_ST_MEM(BPF_B, BPF_REG_10, -32, 0),
9959 BPF_MOV64_IMM(BPF_REG_0, 0),
9960 BPF_EXIT_INSN(),
9961 /* A */
9962 BPF_ST_MEM(BPF_B, BPF_REG_10, -64, 0),
9963 BPF_EXIT_INSN(),
9964 /* B */
9965 BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, -3), /* call A */
9966 BPF_ST_MEM(BPF_B, BPF_REG_10, -256, 0),
9967 BPF_EXIT_INSN(),
9968 },
9969 .prog_type = BPF_PROG_TYPE_XDP,
9970 /* stack_main=32, stack_A=64, stack_B=256
9971 * and max(main+A, main+A+B) < 512
9972 */
9973 .result = ACCEPT,
9974 },
9975 {
9976 "calls: stack depth check using three frames. test3",
9977 .insns = {
9978 /* main */
9979 BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
9980 BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 6), /* call A */
9981 BPF_MOV64_REG(BPF_REG_1, BPF_REG_6),
9982 BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 8), /* call B */
9983 BPF_JMP_IMM(BPF_JGE, BPF_REG_6, 0, 1),
9984 BPF_ST_MEM(BPF_B, BPF_REG_10, -64, 0),
9985 BPF_MOV64_IMM(BPF_REG_0, 0),
9986 BPF_EXIT_INSN(),
9987 /* A */
9988 BPF_JMP_IMM(BPF_JLT, BPF_REG_1, 10, 1),
9989 BPF_EXIT_INSN(),
9990 BPF_ST_MEM(BPF_B, BPF_REG_10, -224, 0),
9991 BPF_JMP_IMM(BPF_JA, 0, 0, -3),
9992 /* B */
9993 BPF_JMP_IMM(BPF_JGT, BPF_REG_1, 2, 1),
9994 BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, -6), /* call A */
9995 BPF_ST_MEM(BPF_B, BPF_REG_10, -256, 0),
9996 BPF_EXIT_INSN(),
9997 },
9998 .prog_type = BPF_PROG_TYPE_XDP,
9999 /* stack_main=64, stack_A=224, stack_B=256
10000 * and max(main+A, main+A+B) > 512
10001 */
10002 .errstr = "combined stack",
10003 .result = REJECT,
10004 },
10005 {
10006 "calls: stack depth check using three frames. test4",
10007 /* void main(void) {
10008 * func1(0);
10009 * func1(1);
10010 * func2(1);
10011 * }
10012 * void func1(int alloc_or_recurse) {
10013 * if (alloc_or_recurse) {
10014 * frame_pointer[-300] = 1;
10015 * } else {
10016 * func2(alloc_or_recurse);
10017 * }
10018 * }
10019 * void func2(int alloc_or_recurse) {
10020 * if (alloc_or_recurse) {
10021 * frame_pointer[-300] = 1;
10022 * }
10023 * }
10024 */
10025 .insns = {
10026 /* main */
10027 BPF_MOV64_IMM(BPF_REG_1, 0),
10028 BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 6), /* call A */
10029 BPF_MOV64_IMM(BPF_REG_1, 1),
10030 BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 4), /* call A */
10031 BPF_MOV64_IMM(BPF_REG_1, 1),
10032 BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 7), /* call B */
10033 BPF_MOV64_IMM(BPF_REG_0, 0),
10034 BPF_EXIT_INSN(),
10035 /* A */
10036 BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, 0, 2),
10037 BPF_ST_MEM(BPF_B, BPF_REG_10, -300, 0),
10038 BPF_EXIT_INSN(),
10039 BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call B */
10040 BPF_EXIT_INSN(),
10041 /* B */
10042 BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, 0, 1),
10043 BPF_ST_MEM(BPF_B, BPF_REG_10, -300, 0),
10044 BPF_EXIT_INSN(),
10045 },
10046 .prog_type = BPF_PROG_TYPE_XDP,
10047 .result = REJECT,
10048 .errstr = "combined stack",
10049 },
10050 {
10051 "calls: stack depth check using three frames. test5",
10052 .insns = {
10053 /* main */
10054 BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call A */
10055 BPF_EXIT_INSN(),
10056 /* A */
10057 BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call B */
10058 BPF_EXIT_INSN(),
10059 /* B */
10060 BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call C */
10061 BPF_EXIT_INSN(),
10062 /* C */
10063 BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call D */
10064 BPF_EXIT_INSN(),
10065 /* D */
10066 BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call E */
10067 BPF_EXIT_INSN(),
10068 /* E */
10069 BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call F */
10070 BPF_EXIT_INSN(),
10071 /* F */
10072 BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call G */
10073 BPF_EXIT_INSN(),
10074 /* G */
10075 BPF_RAW_INSN(BPF_JMP|BPF_CALL, 0, 1, 0, 1), /* call H */
10076 BPF_EXIT_INSN(),
10077 /* H */
10078 BPF_MOV64_IMM(BPF_REG_0, 0),
10079 BPF_EXIT_INSN(),
10080 },
10081 .prog_type = BPF_PROG_TYPE_XDP,
10082 .errstr = "call stack",
10083 .result = REJECT,
10084 },
10085 {
10086 "calls: spill into caller stack frame",
10087 .insns = {
10088 BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0),
10089 BPF_MOV64_REG(BPF_REG_1, BPF_REG_10),
10090 BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -8),
10091 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 1),
10092 BPF_EXIT_INSN(),
10093 BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0),
10094 BPF_MOV64_IMM(BPF_REG_0, 0),
10095 BPF_EXIT_INSN(),
10096 },
10097 .prog_type = BPF_PROG_TYPE_XDP,
10098 .errstr = "cannot spill",
10099 .result = REJECT,
10100 },
10101 {
10102 "calls: write into caller stack frame",
10103 .insns = {
10104 BPF_MOV64_REG(BPF_REG_1, BPF_REG_10),
10105 BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -8),
10106 BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
10107 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 2),
10108 BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_6, 0),
10109 BPF_EXIT_INSN(),
10110 BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 42),
10111 BPF_MOV64_IMM(BPF_REG_0, 0),
10112 BPF_EXIT_INSN(),
10113 },
10114 .prog_type = BPF_PROG_TYPE_XDP,
10115 .result = ACCEPT,
10116 .retval = 42,
10117 },
10118 {
10119 "calls: write into callee stack frame",
10120 .insns = {
10121 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 2),
10122 BPF_ST_MEM(BPF_DW, BPF_REG_0, 0, 42),
10123 BPF_EXIT_INSN(),
10124 BPF_MOV64_REG(BPF_REG_0, BPF_REG_10),
10125 BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, -8),
10126 BPF_EXIT_INSN(),
10127 },
10128 .prog_type = BPF_PROG_TYPE_XDP,
10129 .errstr = "cannot return stack pointer",
10130 .result = REJECT,
10131 },
10132 {
10133 "calls: two calls with stack write and void return",
10134 .insns = {
10135 /* main prog */
10136 BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0),
10137 BPF_MOV64_REG(BPF_REG_1, BPF_REG_10),
10138 BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -8),
10139 BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
10140 BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -16),
10141 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 2),
10142 BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_10, -16),
10143 BPF_EXIT_INSN(),
10144
10145 /* subprog 1 */
10146 BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
10147 BPF_MOV64_REG(BPF_REG_7, BPF_REG_2),
10148 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 3),
10149 BPF_MOV64_REG(BPF_REG_1, BPF_REG_7),
10150 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 1),
10151 BPF_EXIT_INSN(),
10152
10153 /* subprog 2 */
10154 /* write into stack frame of main prog */
10155 BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 0),
10156 BPF_EXIT_INSN(), /* void return */
10157 },
10158 .prog_type = BPF_PROG_TYPE_XDP,
10159 .result = ACCEPT,
10160 },
10161 {
10162 "calls: ambiguous return value",
10163 .insns = {
10164 BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
10165 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 5),
10166 BPF_MOV64_REG(BPF_REG_1, BPF_REG_0),
10167 BPF_MOV64_REG(BPF_REG_1, BPF_REG_6),
10168 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 2),
10169 BPF_MOV64_REG(BPF_REG_1, BPF_REG_0),
10170 BPF_EXIT_INSN(),
10171 BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, 0, 1),
10172 BPF_MOV64_IMM(BPF_REG_0, 0),
10173 BPF_EXIT_INSN(),
10174 },
10175 .errstr_unpriv = "allowed for root only",
10176 .result_unpriv = REJECT,
10177 .errstr = "R0 !read_ok",
10178 .result = REJECT,
10179 },
10180 {
10181 "calls: two calls that return map_value",
10182 .insns = {
10183 /* main prog */
10184 /* pass fp-16, fp-8 into a function */
10185 BPF_MOV64_REG(BPF_REG_1, BPF_REG_10),
10186 BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -8),
10187 BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
10188 BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -16),
10189 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 8),
10190
10191 /* fetch map_value_ptr from the stack of this function */
10192 BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_10, -8),
10193 BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 1),
10194 /* write into map value */
10195 BPF_ST_MEM(BPF_DW, BPF_REG_0, 0, 0),
10196 /* fetch secound map_value_ptr from the stack */
10197 BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_10, -16),
10198 BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 1),
10199 /* write into map value */
10200 BPF_ST_MEM(BPF_DW, BPF_REG_0, 0, 0),
10201 BPF_MOV64_IMM(BPF_REG_0, 0),
10202 BPF_EXIT_INSN(),
10203
10204 /* subprog 1 */
10205 /* call 3rd function twice */
10206 BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
10207 BPF_MOV64_REG(BPF_REG_7, BPF_REG_2),
10208 /* first time with fp-8 */
10209 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 3),
10210 BPF_MOV64_REG(BPF_REG_1, BPF_REG_7),
10211 /* second time with fp-16 */
10212 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 1),
10213 BPF_EXIT_INSN(),
10214
10215 /* subprog 2 */
10216 BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
10217 /* lookup from map */
10218 BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0),
10219 BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
10220 BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
10221 BPF_LD_MAP_FD(BPF_REG_1, 0),
10222 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
10223 BPF_FUNC_map_lookup_elem),
10224 /* write map_value_ptr into stack frame of main prog */
10225 BPF_STX_MEM(BPF_DW, BPF_REG_6, BPF_REG_0, 0),
10226 BPF_MOV64_IMM(BPF_REG_0, 0),
10227 BPF_EXIT_INSN(), /* return 0 */
10228 },
10229 .prog_type = BPF_PROG_TYPE_XDP,
10230 .fixup_map1 = { 23 },
10231 .result = ACCEPT,
10232 },
10233 {
10234 "calls: two calls that return map_value with bool condition",
10235 .insns = {
10236 /* main prog */
10237 /* pass fp-16, fp-8 into a function */
10238 BPF_MOV64_REG(BPF_REG_1, BPF_REG_10),
10239 BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -8),
10240 BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
10241 BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -16),
10242 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 2),
10243 BPF_MOV64_IMM(BPF_REG_0, 0),
10244 BPF_EXIT_INSN(),
10245
10246 /* subprog 1 */
10247 /* call 3rd function twice */
10248 BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
10249 BPF_MOV64_REG(BPF_REG_7, BPF_REG_2),
10250 /* first time with fp-8 */
10251 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 9),
10252 BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 1, 2),
10253 /* fetch map_value_ptr from the stack of this function */
10254 BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_6, 0),
10255 /* write into map value */
10256 BPF_ST_MEM(BPF_DW, BPF_REG_0, 0, 0),
10257 BPF_MOV64_REG(BPF_REG_1, BPF_REG_7),
10258 /* second time with fp-16 */
10259 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 4),
10260 BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 1, 2),
10261 /* fetch secound map_value_ptr from the stack */
10262 BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_7, 0),
10263 /* write into map value */
10264 BPF_ST_MEM(BPF_DW, BPF_REG_0, 0, 0),
10265 BPF_EXIT_INSN(),
10266
10267 /* subprog 2 */
10268 BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
10269 /* lookup from map */
10270 BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0),
10271 BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
10272 BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
10273 BPF_LD_MAP_FD(BPF_REG_1, 0),
10274 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
10275 BPF_FUNC_map_lookup_elem),
10276 BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 2),
10277 BPF_MOV64_IMM(BPF_REG_0, 0),
10278 BPF_EXIT_INSN(), /* return 0 */
10279 /* write map_value_ptr into stack frame of main prog */
10280 BPF_STX_MEM(BPF_DW, BPF_REG_6, BPF_REG_0, 0),
10281 BPF_MOV64_IMM(BPF_REG_0, 1),
10282 BPF_EXIT_INSN(), /* return 1 */
10283 },
10284 .prog_type = BPF_PROG_TYPE_XDP,
10285 .fixup_map1 = { 23 },
10286 .result = ACCEPT,
10287 },
10288 {
10289 "calls: two calls that return map_value with incorrect bool check",
10290 .insns = {
10291 /* main prog */
10292 /* pass fp-16, fp-8 into a function */
10293 BPF_MOV64_REG(BPF_REG_1, BPF_REG_10),
10294 BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -8),
10295 BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
10296 BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -16),
10297 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 2),
10298 BPF_MOV64_IMM(BPF_REG_0, 0),
10299 BPF_EXIT_INSN(),
10300
10301 /* subprog 1 */
10302 /* call 3rd function twice */
10303 BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
10304 BPF_MOV64_REG(BPF_REG_7, BPF_REG_2),
10305 /* first time with fp-8 */
10306 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 9),
10307 BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 1, 2),
10308 /* fetch map_value_ptr from the stack of this function */
10309 BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_6, 0),
10310 /* write into map value */
10311 BPF_ST_MEM(BPF_DW, BPF_REG_0, 0, 0),
10312 BPF_MOV64_REG(BPF_REG_1, BPF_REG_7),
10313 /* second time with fp-16 */
10314 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 4),
10315 BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 2),
10316 /* fetch secound map_value_ptr from the stack */
10317 BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_7, 0),
10318 /* write into map value */
10319 BPF_ST_MEM(BPF_DW, BPF_REG_0, 0, 0),
10320 BPF_EXIT_INSN(),
10321
10322 /* subprog 2 */
10323 BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
10324 /* lookup from map */
10325 BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0),
10326 BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
10327 BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
10328 BPF_LD_MAP_FD(BPF_REG_1, 0),
10329 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
10330 BPF_FUNC_map_lookup_elem),
10331 BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 2),
10332 BPF_MOV64_IMM(BPF_REG_0, 0),
10333 BPF_EXIT_INSN(), /* return 0 */
10334 /* write map_value_ptr into stack frame of main prog */
10335 BPF_STX_MEM(BPF_DW, BPF_REG_6, BPF_REG_0, 0),
10336 BPF_MOV64_IMM(BPF_REG_0, 1),
10337 BPF_EXIT_INSN(), /* return 1 */
10338 },
10339 .prog_type = BPF_PROG_TYPE_XDP,
10340 .fixup_map1 = { 23 },
10341 .result = REJECT,
10342 .errstr = "invalid read from stack off -16+0 size 8",
10343 },
10344 {
10345 "calls: two calls that receive map_value via arg=ptr_stack_of_caller. test1",
10346 .insns = {
10347 /* main prog */
10348 /* pass fp-16, fp-8 into a function */
10349 BPF_MOV64_REG(BPF_REG_1, BPF_REG_10),
10350 BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -8),
10351 BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
10352 BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -16),
10353 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 2),
10354 BPF_MOV64_IMM(BPF_REG_0, 0),
10355 BPF_EXIT_INSN(),
10356
10357 /* subprog 1 */
10358 BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
10359 BPF_MOV64_REG(BPF_REG_7, BPF_REG_2),
10360 /* 1st lookup from map */
10361 BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0),
10362 BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
10363 BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
10364 BPF_LD_MAP_FD(BPF_REG_1, 0),
10365 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
10366 BPF_FUNC_map_lookup_elem),
10367 BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 2),
10368 BPF_MOV64_IMM(BPF_REG_8, 0),
10369 BPF_JMP_IMM(BPF_JA, 0, 0, 2),
10370 /* write map_value_ptr into stack frame of main prog at fp-8 */
10371 BPF_STX_MEM(BPF_DW, BPF_REG_6, BPF_REG_0, 0),
10372 BPF_MOV64_IMM(BPF_REG_8, 1),
10373
10374 /* 2nd lookup from map */
10375 BPF_MOV64_REG(BPF_REG_2, BPF_REG_10), /* 20 */
10376 BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
10377 BPF_LD_MAP_FD(BPF_REG_1, 0),
10378 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, /* 24 */
10379 BPF_FUNC_map_lookup_elem),
10380 BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 2),
10381 BPF_MOV64_IMM(BPF_REG_9, 0),
10382 BPF_JMP_IMM(BPF_JA, 0, 0, 2),
10383 /* write map_value_ptr into stack frame of main prog at fp-16 */
10384 BPF_STX_MEM(BPF_DW, BPF_REG_7, BPF_REG_0, 0),
10385 BPF_MOV64_IMM(BPF_REG_9, 1),
10386
10387 /* call 3rd func with fp-8, 0|1, fp-16, 0|1 */
10388 BPF_MOV64_REG(BPF_REG_1, BPF_REG_6), /* 30 */
10389 BPF_MOV64_REG(BPF_REG_2, BPF_REG_8),
10390 BPF_MOV64_REG(BPF_REG_3, BPF_REG_7),
10391 BPF_MOV64_REG(BPF_REG_4, BPF_REG_9),
10392 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 1), /* 34 */
10393 BPF_EXIT_INSN(),
10394
10395 /* subprog 2 */
10396 /* if arg2 == 1 do *arg1 = 0 */
10397 BPF_JMP_IMM(BPF_JNE, BPF_REG_2, 1, 2),
10398 /* fetch map_value_ptr from the stack of this function */
10399 BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, 0),
10400 /* write into map value */
10401 BPF_ST_MEM(BPF_DW, BPF_REG_0, 0, 0),
10402
10403 /* if arg4 == 1 do *arg3 = 0 */
10404 BPF_JMP_IMM(BPF_JNE, BPF_REG_4, 1, 2),
10405 /* fetch map_value_ptr from the stack of this function */
10406 BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_3, 0),
10407 /* write into map value */
10408 BPF_ST_MEM(BPF_DW, BPF_REG_0, 2, 0),
10409 BPF_EXIT_INSN(),
10410 },
10411 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
10412 .fixup_map1 = { 12, 22 },
10413 .result = REJECT,
10414 .errstr = "invalid access to map value, value_size=8 off=2 size=8",
10415 },
10416 {
10417 "calls: two calls that receive map_value via arg=ptr_stack_of_caller. test2",
10418 .insns = {
10419 /* main prog */
10420 /* pass fp-16, fp-8 into a function */
10421 BPF_MOV64_REG(BPF_REG_1, BPF_REG_10),
10422 BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -8),
10423 BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
10424 BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -16),
10425 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 2),
10426 BPF_MOV64_IMM(BPF_REG_0, 0),
10427 BPF_EXIT_INSN(),
10428
10429 /* subprog 1 */
10430 BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
10431 BPF_MOV64_REG(BPF_REG_7, BPF_REG_2),
10432 /* 1st lookup from map */
10433 BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0),
10434 BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
10435 BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
10436 BPF_LD_MAP_FD(BPF_REG_1, 0),
10437 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
10438 BPF_FUNC_map_lookup_elem),
10439 BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 2),
10440 BPF_MOV64_IMM(BPF_REG_8, 0),
10441 BPF_JMP_IMM(BPF_JA, 0, 0, 2),
10442 /* write map_value_ptr into stack frame of main prog at fp-8 */
10443 BPF_STX_MEM(BPF_DW, BPF_REG_6, BPF_REG_0, 0),
10444 BPF_MOV64_IMM(BPF_REG_8, 1),
10445
10446 /* 2nd lookup from map */
10447 BPF_MOV64_REG(BPF_REG_2, BPF_REG_10), /* 20 */
10448 BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
10449 BPF_LD_MAP_FD(BPF_REG_1, 0),
10450 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, /* 24 */
10451 BPF_FUNC_map_lookup_elem),
10452 BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 2),
10453 BPF_MOV64_IMM(BPF_REG_9, 0),
10454 BPF_JMP_IMM(BPF_JA, 0, 0, 2),
10455 /* write map_value_ptr into stack frame of main prog at fp-16 */
10456 BPF_STX_MEM(BPF_DW, BPF_REG_7, BPF_REG_0, 0),
10457 BPF_MOV64_IMM(BPF_REG_9, 1),
10458
10459 /* call 3rd func with fp-8, 0|1, fp-16, 0|1 */
10460 BPF_MOV64_REG(BPF_REG_1, BPF_REG_6), /* 30 */
10461 BPF_MOV64_REG(BPF_REG_2, BPF_REG_8),
10462 BPF_MOV64_REG(BPF_REG_3, BPF_REG_7),
10463 BPF_MOV64_REG(BPF_REG_4, BPF_REG_9),
10464 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 1), /* 34 */
10465 BPF_EXIT_INSN(),
10466
10467 /* subprog 2 */
10468 /* if arg2 == 1 do *arg1 = 0 */
10469 BPF_JMP_IMM(BPF_JNE, BPF_REG_2, 1, 2),
10470 /* fetch map_value_ptr from the stack of this function */
10471 BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, 0),
10472 /* write into map value */
10473 BPF_ST_MEM(BPF_DW, BPF_REG_0, 0, 0),
10474
10475 /* if arg4 == 1 do *arg3 = 0 */
10476 BPF_JMP_IMM(BPF_JNE, BPF_REG_4, 1, 2),
10477 /* fetch map_value_ptr from the stack of this function */
10478 BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_3, 0),
10479 /* write into map value */
10480 BPF_ST_MEM(BPF_DW, BPF_REG_0, 0, 0),
10481 BPF_EXIT_INSN(),
10482 },
10483 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
10484 .fixup_map1 = { 12, 22 },
10485 .result = ACCEPT,
10486 },
10487 {
10488 "calls: two jumps that receive map_value via arg=ptr_stack_of_jumper. test3",
10489 .insns = {
10490 /* main prog */
10491 /* pass fp-16, fp-8 into a function */
10492 BPF_MOV64_REG(BPF_REG_1, BPF_REG_10),
10493 BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -8),
10494 BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
10495 BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -16),
10496 BPF_JMP_IMM(BPF_JNE, BPF_REG_1, 0, 2),
10497 BPF_MOV64_IMM(BPF_REG_0, 0),
10498 BPF_EXIT_INSN(),
10499
10500 /* subprog 1 */
10501 BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
10502 BPF_MOV64_REG(BPF_REG_7, BPF_REG_2),
10503 /* 1st lookup from map */
10504 BPF_ST_MEM(BPF_DW, BPF_REG_10, -24, 0),
10505 BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
10506 BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -24),
10507 BPF_LD_MAP_FD(BPF_REG_1, 0),
10508 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
10509 BPF_FUNC_map_lookup_elem),
10510 BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 2),
10511 BPF_MOV64_IMM(BPF_REG_8, 0),
10512 BPF_JMP_IMM(BPF_JA, 0, 0, 2),
10513 /* write map_value_ptr into stack frame of main prog at fp-8 */
10514 BPF_STX_MEM(BPF_DW, BPF_REG_6, BPF_REG_0, 0),
10515 BPF_MOV64_IMM(BPF_REG_8, 1),
10516
10517 /* 2nd lookup from map */
10518 BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
10519 BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -24),
10520 BPF_LD_MAP_FD(BPF_REG_1, 0),
10521 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
10522 BPF_FUNC_map_lookup_elem),
10523 BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 2),
10524 BPF_MOV64_IMM(BPF_REG_9, 0), // 26
10525 BPF_JMP_IMM(BPF_JA, 0, 0, 2),
10526 /* write map_value_ptr into stack frame of main prog at fp-16 */
10527 BPF_STX_MEM(BPF_DW, BPF_REG_7, BPF_REG_0, 0),
10528 BPF_MOV64_IMM(BPF_REG_9, 1),
10529
10530 /* call 3rd func with fp-8, 0|1, fp-16, 0|1 */
10531 BPF_MOV64_REG(BPF_REG_1, BPF_REG_6), // 30
10532 BPF_MOV64_REG(BPF_REG_2, BPF_REG_8),
10533 BPF_MOV64_REG(BPF_REG_3, BPF_REG_7),
10534 BPF_MOV64_REG(BPF_REG_4, BPF_REG_9),
10535 BPF_JMP_IMM(BPF_JNE, BPF_REG_1, 0, 1), // 34
10536 BPF_JMP_IMM(BPF_JA, 0, 0, -30),
10537
10538 /* subprog 2 */
10539 /* if arg2 == 1 do *arg1 = 0 */
10540 BPF_JMP_IMM(BPF_JNE, BPF_REG_2, 1, 2),
10541 /* fetch map_value_ptr from the stack of this function */
10542 BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, 0),
10543 /* write into map value */
10544 BPF_ST_MEM(BPF_DW, BPF_REG_0, 0, 0),
10545
10546 /* if arg4 == 1 do *arg3 = 0 */
10547 BPF_JMP_IMM(BPF_JNE, BPF_REG_4, 1, 2),
10548 /* fetch map_value_ptr from the stack of this function */
10549 BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_3, 0),
10550 /* write into map value */
10551 BPF_ST_MEM(BPF_DW, BPF_REG_0, 2, 0),
10552 BPF_JMP_IMM(BPF_JA, 0, 0, -8),
10553 },
10554 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
10555 .fixup_map1 = { 12, 22 },
10556 .result = REJECT,
10557 .errstr = "invalid access to map value, value_size=8 off=2 size=8",
10558 },
10559 {
10560 "calls: two calls that receive map_value_ptr_or_null via arg. test1",
10561 .insns = {
10562 /* main prog */
10563 /* pass fp-16, fp-8 into a function */
10564 BPF_MOV64_REG(BPF_REG_1, BPF_REG_10),
10565 BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -8),
10566 BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
10567 BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -16),
10568 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 2),
10569 BPF_MOV64_IMM(BPF_REG_0, 0),
10570 BPF_EXIT_INSN(),
10571
10572 /* subprog 1 */
10573 BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
10574 BPF_MOV64_REG(BPF_REG_7, BPF_REG_2),
10575 /* 1st lookup from map */
10576 BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0),
10577 BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
10578 BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
10579 BPF_LD_MAP_FD(BPF_REG_1, 0),
10580 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
10581 BPF_FUNC_map_lookup_elem),
10582 /* write map_value_ptr_or_null into stack frame of main prog at fp-8 */
10583 BPF_STX_MEM(BPF_DW, BPF_REG_6, BPF_REG_0, 0),
10584 BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 2),
10585 BPF_MOV64_IMM(BPF_REG_8, 0),
10586 BPF_JMP_IMM(BPF_JA, 0, 0, 1),
10587 BPF_MOV64_IMM(BPF_REG_8, 1),
10588
10589 /* 2nd lookup from map */
10590 BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
10591 BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
10592 BPF_LD_MAP_FD(BPF_REG_1, 0),
10593 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
10594 BPF_FUNC_map_lookup_elem),
10595 /* write map_value_ptr_or_null into stack frame of main prog at fp-16 */
10596 BPF_STX_MEM(BPF_DW, BPF_REG_7, BPF_REG_0, 0),
10597 BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 2),
10598 BPF_MOV64_IMM(BPF_REG_9, 0),
10599 BPF_JMP_IMM(BPF_JA, 0, 0, 1),
10600 BPF_MOV64_IMM(BPF_REG_9, 1),
10601
10602 /* call 3rd func with fp-8, 0|1, fp-16, 0|1 */
10603 BPF_MOV64_REG(BPF_REG_1, BPF_REG_6),
10604 BPF_MOV64_REG(BPF_REG_2, BPF_REG_8),
10605 BPF_MOV64_REG(BPF_REG_3, BPF_REG_7),
10606 BPF_MOV64_REG(BPF_REG_4, BPF_REG_9),
10607 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 1),
10608 BPF_EXIT_INSN(),
10609
10610 /* subprog 2 */
10611 /* if arg2 == 1 do *arg1 = 0 */
10612 BPF_JMP_IMM(BPF_JNE, BPF_REG_2, 1, 2),
10613 /* fetch map_value_ptr from the stack of this function */
10614 BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, 0),
10615 /* write into map value */
10616 BPF_ST_MEM(BPF_DW, BPF_REG_0, 0, 0),
10617
10618 /* if arg4 == 1 do *arg3 = 0 */
10619 BPF_JMP_IMM(BPF_JNE, BPF_REG_4, 1, 2),
10620 /* fetch map_value_ptr from the stack of this function */
10621 BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_3, 0),
10622 /* write into map value */
10623 BPF_ST_MEM(BPF_DW, BPF_REG_0, 0, 0),
10624 BPF_EXIT_INSN(),
10625 },
10626 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
10627 .fixup_map1 = { 12, 22 },
10628 .result = ACCEPT,
10629 },
10630 {
10631 "calls: two calls that receive map_value_ptr_or_null via arg. test2",
10632 .insns = {
10633 /* main prog */
10634 /* pass fp-16, fp-8 into a function */
10635 BPF_MOV64_REG(BPF_REG_1, BPF_REG_10),
10636 BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -8),
10637 BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
10638 BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -16),
10639 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 2),
10640 BPF_MOV64_IMM(BPF_REG_0, 0),
10641 BPF_EXIT_INSN(),
10642
10643 /* subprog 1 */
10644 BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
10645 BPF_MOV64_REG(BPF_REG_7, BPF_REG_2),
10646 /* 1st lookup from map */
10647 BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0),
10648 BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
10649 BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
10650 BPF_LD_MAP_FD(BPF_REG_1, 0),
10651 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
10652 BPF_FUNC_map_lookup_elem),
10653 /* write map_value_ptr_or_null into stack frame of main prog at fp-8 */
10654 BPF_STX_MEM(BPF_DW, BPF_REG_6, BPF_REG_0, 0),
10655 BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 2),
10656 BPF_MOV64_IMM(BPF_REG_8, 0),
10657 BPF_JMP_IMM(BPF_JA, 0, 0, 1),
10658 BPF_MOV64_IMM(BPF_REG_8, 1),
10659
10660 /* 2nd lookup from map */
10661 BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
10662 BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
10663 BPF_LD_MAP_FD(BPF_REG_1, 0),
10664 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
10665 BPF_FUNC_map_lookup_elem),
10666 /* write map_value_ptr_or_null into stack frame of main prog at fp-16 */
10667 BPF_STX_MEM(BPF_DW, BPF_REG_7, BPF_REG_0, 0),
10668 BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 2),
10669 BPF_MOV64_IMM(BPF_REG_9, 0),
10670 BPF_JMP_IMM(BPF_JA, 0, 0, 1),
10671 BPF_MOV64_IMM(BPF_REG_9, 1),
10672
10673 /* call 3rd func with fp-8, 0|1, fp-16, 0|1 */
10674 BPF_MOV64_REG(BPF_REG_1, BPF_REG_6),
10675 BPF_MOV64_REG(BPF_REG_2, BPF_REG_8),
10676 BPF_MOV64_REG(BPF_REG_3, BPF_REG_7),
10677 BPF_MOV64_REG(BPF_REG_4, BPF_REG_9),
10678 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 1),
10679 BPF_EXIT_INSN(),
10680
10681 /* subprog 2 */
10682 /* if arg2 == 1 do *arg1 = 0 */
10683 BPF_JMP_IMM(BPF_JNE, BPF_REG_2, 1, 2),
10684 /* fetch map_value_ptr from the stack of this function */
10685 BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, 0),
10686 /* write into map value */
10687 BPF_ST_MEM(BPF_DW, BPF_REG_0, 0, 0),
10688
10689 /* if arg4 == 0 do *arg3 = 0 */
10690 BPF_JMP_IMM(BPF_JNE, BPF_REG_4, 0, 2),
10691 /* fetch map_value_ptr from the stack of this function */
10692 BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_3, 0),
10693 /* write into map value */
10694 BPF_ST_MEM(BPF_DW, BPF_REG_0, 0, 0),
10695 BPF_EXIT_INSN(),
10696 },
10697 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
10698 .fixup_map1 = { 12, 22 },
10699 .result = REJECT,
10700 .errstr = "R0 invalid mem access 'inv'",
10701 },
10702 {
10703 "calls: pkt_ptr spill into caller stack",
10704 .insns = {
10705 BPF_MOV64_REG(BPF_REG_4, BPF_REG_10),
10706 BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, -8),
10707 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 1),
10708 BPF_EXIT_INSN(),
10709
10710 /* subprog 1 */
10711 BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1,
10712 offsetof(struct __sk_buff, data)),
10713 BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1,
10714 offsetof(struct __sk_buff, data_end)),
10715 BPF_MOV64_REG(BPF_REG_0, BPF_REG_2),
10716 BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 8),
10717 /* spill unchecked pkt_ptr into stack of caller */
10718 BPF_STX_MEM(BPF_DW, BPF_REG_4, BPF_REG_2, 0),
10719 BPF_JMP_REG(BPF_JGT, BPF_REG_0, BPF_REG_3, 2),
10720 /* now the pkt range is verified, read pkt_ptr from stack */
10721 BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_4, 0),
10722 /* write 4 bytes into packet */
10723 BPF_ST_MEM(BPF_W, BPF_REG_2, 0, 0),
10724 BPF_EXIT_INSN(),
10725 },
10726 .result = ACCEPT,
10727 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
10728 .retval = POINTER_VALUE,
10729 },
10730 {
10731 "calls: pkt_ptr spill into caller stack 2",
10732 .insns = {
10733 BPF_MOV64_REG(BPF_REG_4, BPF_REG_10),
10734 BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, -8),
10735 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 3),
10736 /* Marking is still kept, but not in all cases safe. */
10737 BPF_LDX_MEM(BPF_DW, BPF_REG_4, BPF_REG_10, -8),
10738 BPF_ST_MEM(BPF_W, BPF_REG_4, 0, 0),
10739 BPF_EXIT_INSN(),
10740
10741 /* subprog 1 */
10742 BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1,
10743 offsetof(struct __sk_buff, data)),
10744 BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1,
10745 offsetof(struct __sk_buff, data_end)),
10746 BPF_MOV64_REG(BPF_REG_0, BPF_REG_2),
10747 BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 8),
10748 /* spill unchecked pkt_ptr into stack of caller */
10749 BPF_STX_MEM(BPF_DW, BPF_REG_4, BPF_REG_2, 0),
10750 BPF_JMP_REG(BPF_JGT, BPF_REG_0, BPF_REG_3, 2),
10751 /* now the pkt range is verified, read pkt_ptr from stack */
10752 BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_4, 0),
10753 /* write 4 bytes into packet */
10754 BPF_ST_MEM(BPF_W, BPF_REG_2, 0, 0),
10755 BPF_EXIT_INSN(),
10756 },
10757 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
10758 .errstr = "invalid access to packet",
10759 .result = REJECT,
10760 },
10761 {
10762 "calls: pkt_ptr spill into caller stack 3",
10763 .insns = {
10764 BPF_MOV64_REG(BPF_REG_4, BPF_REG_10),
10765 BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, -8),
10766 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 4),
10767 BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 2),
10768 /* Marking is still kept and safe here. */
10769 BPF_LDX_MEM(BPF_DW, BPF_REG_4, BPF_REG_10, -8),
10770 BPF_ST_MEM(BPF_W, BPF_REG_4, 0, 0),
10771 BPF_EXIT_INSN(),
10772
10773 /* subprog 1 */
10774 BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1,
10775 offsetof(struct __sk_buff, data)),
10776 BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1,
10777 offsetof(struct __sk_buff, data_end)),
10778 BPF_MOV64_REG(BPF_REG_0, BPF_REG_2),
10779 BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 8),
10780 /* spill unchecked pkt_ptr into stack of caller */
10781 BPF_STX_MEM(BPF_DW, BPF_REG_4, BPF_REG_2, 0),
10782 BPF_MOV64_IMM(BPF_REG_5, 0),
10783 BPF_JMP_REG(BPF_JGT, BPF_REG_0, BPF_REG_3, 3),
10784 BPF_MOV64_IMM(BPF_REG_5, 1),
10785 /* now the pkt range is verified, read pkt_ptr from stack */
10786 BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_4, 0),
10787 /* write 4 bytes into packet */
10788 BPF_ST_MEM(BPF_W, BPF_REG_2, 0, 0),
10789 BPF_MOV64_REG(BPF_REG_0, BPF_REG_5),
10790 BPF_EXIT_INSN(),
10791 },
10792 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
10793 .result = ACCEPT,
10794 .retval = 1,
10795 },
10796 {
10797 "calls: pkt_ptr spill into caller stack 4",
10798 .insns = {
10799 BPF_MOV64_REG(BPF_REG_4, BPF_REG_10),
10800 BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, -8),
10801 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 4),
10802 BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 2),
10803 /* Check marking propagated. */
10804 BPF_LDX_MEM(BPF_DW, BPF_REG_4, BPF_REG_10, -8),
10805 BPF_ST_MEM(BPF_W, BPF_REG_4, 0, 0),
10806 BPF_EXIT_INSN(),
10807
10808 /* subprog 1 */
10809 BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1,
10810 offsetof(struct __sk_buff, data)),
10811 BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1,
10812 offsetof(struct __sk_buff, data_end)),
10813 BPF_MOV64_REG(BPF_REG_0, BPF_REG_2),
10814 BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 8),
10815 /* spill unchecked pkt_ptr into stack of caller */
10816 BPF_STX_MEM(BPF_DW, BPF_REG_4, BPF_REG_2, 0),
10817 BPF_MOV64_IMM(BPF_REG_5, 0),
10818 BPF_JMP_REG(BPF_JGT, BPF_REG_0, BPF_REG_3, 2),
10819 BPF_MOV64_IMM(BPF_REG_5, 1),
10820 /* don't read back pkt_ptr from stack here */
10821 /* write 4 bytes into packet */
10822 BPF_ST_MEM(BPF_W, BPF_REG_2, 0, 0),
10823 BPF_MOV64_REG(BPF_REG_0, BPF_REG_5),
10824 BPF_EXIT_INSN(),
10825 },
10826 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
10827 .result = ACCEPT,
10828 .retval = 1,
10829 },
10830 {
10831 "calls: pkt_ptr spill into caller stack 5",
10832 .insns = {
10833 BPF_MOV64_REG(BPF_REG_4, BPF_REG_10),
10834 BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, -8),
10835 BPF_STX_MEM(BPF_DW, BPF_REG_4, BPF_REG_1, 0),
10836 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 3),
10837 BPF_LDX_MEM(BPF_DW, BPF_REG_4, BPF_REG_10, -8),
10838 BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_4, 0),
10839 BPF_EXIT_INSN(),
10840
10841 /* subprog 1 */
10842 BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1,
10843 offsetof(struct __sk_buff, data)),
10844 BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1,
10845 offsetof(struct __sk_buff, data_end)),
10846 BPF_MOV64_REG(BPF_REG_0, BPF_REG_2),
10847 BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 8),
10848 BPF_MOV64_IMM(BPF_REG_5, 0),
10849 BPF_JMP_REG(BPF_JGT, BPF_REG_0, BPF_REG_3, 3),
10850 /* spill checked pkt_ptr into stack of caller */
10851 BPF_STX_MEM(BPF_DW, BPF_REG_4, BPF_REG_2, 0),
10852 BPF_MOV64_IMM(BPF_REG_5, 1),
10853 /* don't read back pkt_ptr from stack here */
10854 /* write 4 bytes into packet */
10855 BPF_ST_MEM(BPF_W, BPF_REG_2, 0, 0),
10856 BPF_MOV64_REG(BPF_REG_0, BPF_REG_5),
10857 BPF_EXIT_INSN(),
10858 },
10859 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
10860 .errstr = "same insn cannot be used with different",
10861 .result = REJECT,
10862 },
10863 {
10864 "calls: pkt_ptr spill into caller stack 6",
10865 .insns = {
10866 BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1,
10867 offsetof(struct __sk_buff, data_end)),
10868 BPF_MOV64_REG(BPF_REG_4, BPF_REG_10),
10869 BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, -8),
10870 BPF_STX_MEM(BPF_DW, BPF_REG_4, BPF_REG_2, 0),
10871 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 3),
10872 BPF_LDX_MEM(BPF_DW, BPF_REG_4, BPF_REG_10, -8),
10873 BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_4, 0),
10874 BPF_EXIT_INSN(),
10875
10876 /* subprog 1 */
10877 BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1,
10878 offsetof(struct __sk_buff, data)),
10879 BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1,
10880 offsetof(struct __sk_buff, data_end)),
10881 BPF_MOV64_REG(BPF_REG_0, BPF_REG_2),
10882 BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 8),
10883 BPF_MOV64_IMM(BPF_REG_5, 0),
10884 BPF_JMP_REG(BPF_JGT, BPF_REG_0, BPF_REG_3, 3),
10885 /* spill checked pkt_ptr into stack of caller */
10886 BPF_STX_MEM(BPF_DW, BPF_REG_4, BPF_REG_2, 0),
10887 BPF_MOV64_IMM(BPF_REG_5, 1),
10888 /* don't read back pkt_ptr from stack here */
10889 /* write 4 bytes into packet */
10890 BPF_ST_MEM(BPF_W, BPF_REG_2, 0, 0),
10891 BPF_MOV64_REG(BPF_REG_0, BPF_REG_5),
10892 BPF_EXIT_INSN(),
10893 },
10894 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
10895 .errstr = "R4 invalid mem access",
10896 .result = REJECT,
10897 },
10898 {
10899 "calls: pkt_ptr spill into caller stack 7",
10900 .insns = {
10901 BPF_MOV64_IMM(BPF_REG_2, 0),
10902 BPF_MOV64_REG(BPF_REG_4, BPF_REG_10),
10903 BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, -8),
10904 BPF_STX_MEM(BPF_DW, BPF_REG_4, BPF_REG_2, 0),
10905 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 3),
10906 BPF_LDX_MEM(BPF_DW, BPF_REG_4, BPF_REG_10, -8),
10907 BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_4, 0),
10908 BPF_EXIT_INSN(),
10909
10910 /* subprog 1 */
10911 BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1,
10912 offsetof(struct __sk_buff, data)),
10913 BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1,
10914 offsetof(struct __sk_buff, data_end)),
10915 BPF_MOV64_REG(BPF_REG_0, BPF_REG_2),
10916 BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 8),
10917 BPF_MOV64_IMM(BPF_REG_5, 0),
10918 BPF_JMP_REG(BPF_JGT, BPF_REG_0, BPF_REG_3, 3),
10919 /* spill checked pkt_ptr into stack of caller */
10920 BPF_STX_MEM(BPF_DW, BPF_REG_4, BPF_REG_2, 0),
10921 BPF_MOV64_IMM(BPF_REG_5, 1),
10922 /* don't read back pkt_ptr from stack here */
10923 /* write 4 bytes into packet */
10924 BPF_ST_MEM(BPF_W, BPF_REG_2, 0, 0),
10925 BPF_MOV64_REG(BPF_REG_0, BPF_REG_5),
10926 BPF_EXIT_INSN(),
10927 },
10928 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
10929 .errstr = "R4 invalid mem access",
10930 .result = REJECT,
10931 },
10932 {
10933 "calls: pkt_ptr spill into caller stack 8",
10934 .insns = {
10935 BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1,
10936 offsetof(struct __sk_buff, data)),
10937 BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1,
10938 offsetof(struct __sk_buff, data_end)),
10939 BPF_MOV64_REG(BPF_REG_0, BPF_REG_2),
10940 BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 8),
10941 BPF_JMP_REG(BPF_JLE, BPF_REG_0, BPF_REG_3, 1),
10942 BPF_EXIT_INSN(),
10943 BPF_MOV64_REG(BPF_REG_4, BPF_REG_10),
10944 BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, -8),
10945 BPF_STX_MEM(BPF_DW, BPF_REG_4, BPF_REG_2, 0),
10946 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 3),
10947 BPF_LDX_MEM(BPF_DW, BPF_REG_4, BPF_REG_10, -8),
10948 BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_4, 0),
10949 BPF_EXIT_INSN(),
10950
10951 /* subprog 1 */
10952 BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1,
10953 offsetof(struct __sk_buff, data)),
10954 BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1,
10955 offsetof(struct __sk_buff, data_end)),
10956 BPF_MOV64_REG(BPF_REG_0, BPF_REG_2),
10957 BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 8),
10958 BPF_MOV64_IMM(BPF_REG_5, 0),
10959 BPF_JMP_REG(BPF_JGT, BPF_REG_0, BPF_REG_3, 3),
10960 /* spill checked pkt_ptr into stack of caller */
10961 BPF_STX_MEM(BPF_DW, BPF_REG_4, BPF_REG_2, 0),
10962 BPF_MOV64_IMM(BPF_REG_5, 1),
10963 /* don't read back pkt_ptr from stack here */
10964 /* write 4 bytes into packet */
10965 BPF_ST_MEM(BPF_W, BPF_REG_2, 0, 0),
10966 BPF_MOV64_REG(BPF_REG_0, BPF_REG_5),
10967 BPF_EXIT_INSN(),
10968 },
10969 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
10970 .result = ACCEPT,
10971 },
10972 {
10973 "calls: pkt_ptr spill into caller stack 9",
10974 .insns = {
10975 BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1,
10976 offsetof(struct __sk_buff, data)),
10977 BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1,
10978 offsetof(struct __sk_buff, data_end)),
10979 BPF_MOV64_REG(BPF_REG_0, BPF_REG_2),
10980 BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 8),
10981 BPF_JMP_REG(BPF_JLE, BPF_REG_0, BPF_REG_3, 1),
10982 BPF_EXIT_INSN(),
10983 BPF_MOV64_REG(BPF_REG_4, BPF_REG_10),
10984 BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, -8),
10985 BPF_STX_MEM(BPF_DW, BPF_REG_4, BPF_REG_2, 0),
10986 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 3),
10987 BPF_LDX_MEM(BPF_DW, BPF_REG_4, BPF_REG_10, -8),
10988 BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_4, 0),
10989 BPF_EXIT_INSN(),
10990
10991 /* subprog 1 */
10992 BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1,
10993 offsetof(struct __sk_buff, data)),
10994 BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1,
10995 offsetof(struct __sk_buff, data_end)),
10996 BPF_MOV64_REG(BPF_REG_0, BPF_REG_2),
10997 BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 8),
10998 BPF_MOV64_IMM(BPF_REG_5, 0),
10999 /* spill unchecked pkt_ptr into stack of caller */
11000 BPF_STX_MEM(BPF_DW, BPF_REG_4, BPF_REG_2, 0),
11001 BPF_JMP_REG(BPF_JGT, BPF_REG_0, BPF_REG_3, 2),
11002 BPF_MOV64_IMM(BPF_REG_5, 1),
11003 /* don't read back pkt_ptr from stack here */
11004 /* write 4 bytes into packet */
11005 BPF_ST_MEM(BPF_W, BPF_REG_2, 0, 0),
11006 BPF_MOV64_REG(BPF_REG_0, BPF_REG_5),
11007 BPF_EXIT_INSN(),
11008 },
11009 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
11010 .errstr = "invalid access to packet",
11011 .result = REJECT,
11012 },
11013 {
11014 "calls: caller stack init to zero or map_value_or_null",
11015 .insns = {
11016 BPF_MOV64_IMM(BPF_REG_0, 0),
11017 BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, -8),
11018 BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
11019 BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
11020 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 4),
11021 /* fetch map_value_or_null or const_zero from stack */
11022 BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_10, -8),
11023 BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 1),
11024 /* store into map_value */
11025 BPF_ST_MEM(BPF_W, BPF_REG_0, 0, 0),
11026 BPF_EXIT_INSN(),
11027
11028 /* subprog 1 */
11029 /* if (ctx == 0) return; */
11030 BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, 0, 8),
11031 /* else bpf_map_lookup() and *(fp - 8) = r0 */
11032 BPF_MOV64_REG(BPF_REG_6, BPF_REG_2),
11033 BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
11034 BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
11035 BPF_LD_MAP_FD(BPF_REG_1, 0),
11036 BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0),
11037 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
11038 BPF_FUNC_map_lookup_elem),
11039 /* write map_value_ptr_or_null into stack frame of main prog at fp-8 */
11040 BPF_STX_MEM(BPF_DW, BPF_REG_6, BPF_REG_0, 0),
11041 BPF_EXIT_INSN(),
11042 },
11043 .fixup_map1 = { 13 },
11044 .result = ACCEPT,
11045 .prog_type = BPF_PROG_TYPE_XDP,
11046 },
11047 {
11048 "calls: stack init to zero and pruning",
11049 .insns = {
11050 /* first make allocated_stack 16 byte */
11051 BPF_ST_MEM(BPF_DW, BPF_REG_10, -16, 0),
11052 /* now fork the execution such that the false branch
11053 * of JGT insn will be verified second and it skisp zero
11054 * init of fp-8 stack slot. If stack liveness marking
11055 * is missing live_read marks from call map_lookup
11056 * processing then pruning will incorrectly assume
11057 * that fp-8 stack slot was unused in the fall-through
11058 * branch and will accept the program incorrectly
11059 */
11060 BPF_JMP_IMM(BPF_JGT, BPF_REG_1, 2, 2),
11061 BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0),
11062 BPF_JMP_IMM(BPF_JA, 0, 0, 0),
11063 BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
11064 BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
11065 BPF_LD_MAP_FD(BPF_REG_1, 0),
11066 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
11067 BPF_FUNC_map_lookup_elem),
11068 BPF_EXIT_INSN(),
11069 },
11070 .fixup_map2 = { 6 },
11071 .errstr = "invalid indirect read from stack off -8+0 size 8",
11072 .result = REJECT,
11073 .prog_type = BPF_PROG_TYPE_XDP,
11074 },
11075 {
11076 "search pruning: all branches should be verified (nop operation)",
11077 .insns = {
11078 BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
11079 BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
11080 BPF_ST_MEM(BPF_DW, BPF_REG_2, 0, 0),
11081 BPF_LD_MAP_FD(BPF_REG_1, 0),
11082 BPF_EMIT_CALL(BPF_FUNC_map_lookup_elem),
11083 BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 11),
11084 BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0),
11085 BPF_JMP_IMM(BPF_JEQ, BPF_REG_3, 0xbeef, 2),
11086 BPF_MOV64_IMM(BPF_REG_4, 0),
11087 BPF_JMP_A(1),
11088 BPF_MOV64_IMM(BPF_REG_4, 1),
11089 BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_4, -16),
11090 BPF_EMIT_CALL(BPF_FUNC_ktime_get_ns),
11091 BPF_LDX_MEM(BPF_DW, BPF_REG_5, BPF_REG_10, -16),
11092 BPF_JMP_IMM(BPF_JEQ, BPF_REG_5, 0, 2),
11093 BPF_MOV64_IMM(BPF_REG_6, 0),
11094 BPF_ST_MEM(BPF_DW, BPF_REG_6, 0, 0xdead),
11095 BPF_EXIT_INSN(),
11096 },
11097 .fixup_map1 = { 3 },
11098 .errstr = "R6 invalid mem access 'inv'",
11099 .result = REJECT,
11100 .prog_type = BPF_PROG_TYPE_TRACEPOINT,
11101 },
11102 {
11103 "search pruning: all branches should be verified (invalid stack access)",
11104 .insns = {
11105 BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
11106 BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
11107 BPF_ST_MEM(BPF_DW, BPF_REG_2, 0, 0),
11108 BPF_LD_MAP_FD(BPF_REG_1, 0),
11109 BPF_EMIT_CALL(BPF_FUNC_map_lookup_elem),
11110 BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 8),
11111 BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0),
11112 BPF_MOV64_IMM(BPF_REG_4, 0),
11113 BPF_JMP_IMM(BPF_JEQ, BPF_REG_3, 0xbeef, 2),
11114 BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_4, -16),
11115 BPF_JMP_A(1),
11116 BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_4, -24),
11117 BPF_EMIT_CALL(BPF_FUNC_ktime_get_ns),
11118 BPF_LDX_MEM(BPF_DW, BPF_REG_5, BPF_REG_10, -16),
11119 BPF_EXIT_INSN(),
11120 },
11121 .fixup_map1 = { 3 },
11122 .errstr = "invalid read from stack off -16+0 size 8",
11123 .result = REJECT,
11124 .prog_type = BPF_PROG_TYPE_TRACEPOINT,
11125 },
8829}; 11126};
8830 11127
8831static int probe_filter_length(const struct bpf_insn *fp) 11128static int probe_filter_length(const struct bpf_insn *fp)
@@ -8937,10 +11234,12 @@ static void do_test_single(struct bpf_test *test, bool unpriv,
8937 int fd_prog, expected_ret, reject_from_alignment; 11234 int fd_prog, expected_ret, reject_from_alignment;
8938 struct bpf_insn *prog = test->insns; 11235 struct bpf_insn *prog = test->insns;
8939 int prog_len = probe_filter_length(prog); 11236 int prog_len = probe_filter_length(prog);
11237 char data_in[TEST_DATA_LEN] = {};
8940 int prog_type = test->prog_type; 11238 int prog_type = test->prog_type;
8941 int map_fds[MAX_NR_MAPS]; 11239 int map_fds[MAX_NR_MAPS];
8942 const char *expected_err; 11240 const char *expected_err;
8943 int i; 11241 uint32_t retval;
11242 int i, err;
8944 11243
8945 for (i = 0; i < MAX_NR_MAPS; i++) 11244 for (i = 0; i < MAX_NR_MAPS; i++)
8946 map_fds[i] = -1; 11245 map_fds[i] = -1;
@@ -8983,6 +11282,19 @@ static void do_test_single(struct bpf_test *test, bool unpriv,
8983 } 11282 }
8984 } 11283 }
8985 11284
11285 if (fd_prog >= 0) {
11286 err = bpf_prog_test_run(fd_prog, 1, data_in, sizeof(data_in),
11287 NULL, NULL, &retval, NULL);
11288 if (err && errno != 524/*ENOTSUPP*/ && errno != EPERM) {
11289 printf("Unexpected bpf_prog_test_run error\n");
11290 goto fail_log;
11291 }
11292 if (!err && retval != test->retval &&
11293 test->retval != POINTER_VALUE) {
11294 printf("FAIL retval %d != %d\n", retval, test->retval);
11295 goto fail_log;
11296 }
11297 }
8986 (*passes)++; 11298 (*passes)++;
8987 printf("OK%s\n", reject_from_alignment ? 11299 printf("OK%s\n", reject_from_alignment ?
8988 " (NOTE: reject due to unknown alignment)" : ""); 11300 " (NOTE: reject due to unknown alignment)" : "");
diff --git a/tools/testing/selftests/bpf/test_xdp_noinline.c b/tools/testing/selftests/bpf/test_xdp_noinline.c
new file mode 100644
index 000000000000..5e4aac74f9d0
--- /dev/null
+++ b/tools/testing/selftests/bpf/test_xdp_noinline.c
@@ -0,0 +1,833 @@
1// SPDX-License-Identifier: GPL-2.0
2// Copyright (c) 2017 Facebook
3#include <stddef.h>
4#include <stdbool.h>
5#include <string.h>
6#include <linux/pkt_cls.h>
7#include <linux/bpf.h>
8#include <linux/in.h>
9#include <linux/if_ether.h>
10#include <linux/ip.h>
11#include <linux/ipv6.h>
12#include <linux/icmp.h>
13#include <linux/icmpv6.h>
14#include <linux/tcp.h>
15#include <linux/udp.h>
16#include "bpf_helpers.h"
17
18#define bpf_printk(fmt, ...) \
19({ \
20 char ____fmt[] = fmt; \
21 bpf_trace_printk(____fmt, sizeof(____fmt), \
22 ##__VA_ARGS__); \
23})
24
25static __u32 rol32(__u32 word, unsigned int shift)
26{
27 return (word << shift) | (word >> ((-shift) & 31));
28}
29
30/* copy paste of jhash from kernel sources to make sure llvm
31 * can compile it into valid sequence of bpf instructions
32 */
33#define __jhash_mix(a, b, c) \
34{ \
35 a -= c; a ^= rol32(c, 4); c += b; \
36 b -= a; b ^= rol32(a, 6); a += c; \
37 c -= b; c ^= rol32(b, 8); b += a; \
38 a -= c; a ^= rol32(c, 16); c += b; \
39 b -= a; b ^= rol32(a, 19); a += c; \
40 c -= b; c ^= rol32(b, 4); b += a; \
41}
42
43#define __jhash_final(a, b, c) \
44{ \
45 c ^= b; c -= rol32(b, 14); \
46 a ^= c; a -= rol32(c, 11); \
47 b ^= a; b -= rol32(a, 25); \
48 c ^= b; c -= rol32(b, 16); \
49 a ^= c; a -= rol32(c, 4); \
50 b ^= a; b -= rol32(a, 14); \
51 c ^= b; c -= rol32(b, 24); \
52}
53
54#define JHASH_INITVAL 0xdeadbeef
55
56typedef unsigned int u32;
57
58static __attribute__ ((noinline))
59u32 jhash(const void *key, u32 length, u32 initval)
60{
61 u32 a, b, c;
62 const unsigned char *k = key;
63
64 a = b = c = JHASH_INITVAL + length + initval;
65
66 while (length > 12) {
67 a += *(u32 *)(k);
68 b += *(u32 *)(k + 4);
69 c += *(u32 *)(k + 8);
70 __jhash_mix(a, b, c);
71 length -= 12;
72 k += 12;
73 }
74 switch (length) {
75 case 12: c += (u32)k[11]<<24;
76 case 11: c += (u32)k[10]<<16;
77 case 10: c += (u32)k[9]<<8;
78 case 9: c += k[8];
79 case 8: b += (u32)k[7]<<24;
80 case 7: b += (u32)k[6]<<16;
81 case 6: b += (u32)k[5]<<8;
82 case 5: b += k[4];
83 case 4: a += (u32)k[3]<<24;
84 case 3: a += (u32)k[2]<<16;
85 case 2: a += (u32)k[1]<<8;
86 case 1: a += k[0];
87 __jhash_final(a, b, c);
88 case 0: /* Nothing left to add */
89 break;
90 }
91
92 return c;
93}
94
95static __attribute__ ((noinline))
96u32 __jhash_nwords(u32 a, u32 b, u32 c, u32 initval)
97{
98 a += initval;
99 b += initval;
100 c += initval;
101 __jhash_final(a, b, c);
102 return c;
103}
104
105static __attribute__ ((noinline))
106u32 jhash_2words(u32 a, u32 b, u32 initval)
107{
108 return __jhash_nwords(a, b, 0, initval + JHASH_INITVAL + (2 << 2));
109}
110
111struct flow_key {
112 union {
113 __be32 src;
114 __be32 srcv6[4];
115 };
116 union {
117 __be32 dst;
118 __be32 dstv6[4];
119 };
120 union {
121 __u32 ports;
122 __u16 port16[2];
123 };
124 __u8 proto;
125};
126
127struct packet_description {
128 struct flow_key flow;
129 __u8 flags;
130};
131
132struct ctl_value {
133 union {
134 __u64 value;
135 __u32 ifindex;
136 __u8 mac[6];
137 };
138};
139
140struct vip_definition {
141 union {
142 __be32 vip;
143 __be32 vipv6[4];
144 };
145 __u16 port;
146 __u16 family;
147 __u8 proto;
148};
149
150struct vip_meta {
151 __u32 flags;
152 __u32 vip_num;
153};
154
155struct real_pos_lru {
156 __u32 pos;
157 __u64 atime;
158};
159
160struct real_definition {
161 union {
162 __be32 dst;
163 __be32 dstv6[4];
164 };
165 __u8 flags;
166};
167
168struct lb_stats {
169 __u64 v2;
170 __u64 v1;
171};
172
173struct bpf_map_def __attribute__ ((section("maps"), used)) vip_map = {
174 .type = BPF_MAP_TYPE_HASH,
175 .key_size = sizeof(struct vip_definition),
176 .value_size = sizeof(struct vip_meta),
177 .max_entries = 512,
178 .map_flags = 0,
179};
180
181struct bpf_map_def __attribute__ ((section("maps"), used)) lru_cache = {
182 .type = BPF_MAP_TYPE_LRU_HASH,
183 .key_size = sizeof(struct flow_key),
184 .value_size = sizeof(struct real_pos_lru),
185 .max_entries = 300,
186 .map_flags = 1U << 1,
187};
188
189struct bpf_map_def __attribute__ ((section("maps"), used)) ch_rings = {
190 .type = BPF_MAP_TYPE_ARRAY,
191 .key_size = sizeof(__u32),
192 .value_size = sizeof(__u32),
193 .max_entries = 12 * 655,
194 .map_flags = 0,
195};
196
197struct bpf_map_def __attribute__ ((section("maps"), used)) reals = {
198 .type = BPF_MAP_TYPE_ARRAY,
199 .key_size = sizeof(__u32),
200 .value_size = sizeof(struct real_definition),
201 .max_entries = 40,
202 .map_flags = 0,
203};
204
205struct bpf_map_def __attribute__ ((section("maps"), used)) stats = {
206 .type = BPF_MAP_TYPE_PERCPU_ARRAY,
207 .key_size = sizeof(__u32),
208 .value_size = sizeof(struct lb_stats),
209 .max_entries = 515,
210 .map_flags = 0,
211};
212
213struct bpf_map_def __attribute__ ((section("maps"), used)) ctl_array = {
214 .type = BPF_MAP_TYPE_ARRAY,
215 .key_size = sizeof(__u32),
216 .value_size = sizeof(struct ctl_value),
217 .max_entries = 16,
218 .map_flags = 0,
219};
220
221struct eth_hdr {
222 unsigned char eth_dest[6];
223 unsigned char eth_source[6];
224 unsigned short eth_proto;
225};
226
227static inline __u64 calc_offset(bool is_ipv6, bool is_icmp)
228{
229 __u64 off = sizeof(struct eth_hdr);
230 if (is_ipv6) {
231 off += sizeof(struct ipv6hdr);
232 if (is_icmp)
233 off += sizeof(struct icmp6hdr) + sizeof(struct ipv6hdr);
234 } else {
235 off += sizeof(struct iphdr);
236 if (is_icmp)
237 off += sizeof(struct icmphdr) + sizeof(struct iphdr);
238 }
239 return off;
240}
241
242static __attribute__ ((noinline))
243bool parse_udp(void *data, void *data_end,
244 bool is_ipv6, struct packet_description *pckt)
245{
246
247 bool is_icmp = !((pckt->flags & (1 << 0)) == 0);
248 __u64 off = calc_offset(is_ipv6, is_icmp);
249 struct udphdr *udp;
250 udp = data + off;
251
252 if (udp + 1 > data_end)
253 return 0;
254 if (!is_icmp) {
255 pckt->flow.port16[0] = udp->source;
256 pckt->flow.port16[1] = udp->dest;
257 } else {
258 pckt->flow.port16[0] = udp->dest;
259 pckt->flow.port16[1] = udp->source;
260 }
261 return 1;
262}
263
264static __attribute__ ((noinline))
265bool parse_tcp(void *data, void *data_end,
266 bool is_ipv6, struct packet_description *pckt)
267{
268
269 bool is_icmp = !((pckt->flags & (1 << 0)) == 0);
270 __u64 off = calc_offset(is_ipv6, is_icmp);
271 struct tcphdr *tcp;
272
273 tcp = data + off;
274 if (tcp + 1 > data_end)
275 return 0;
276 if (tcp->syn)
277 pckt->flags |= (1 << 1);
278 if (!is_icmp) {
279 pckt->flow.port16[0] = tcp->source;
280 pckt->flow.port16[1] = tcp->dest;
281 } else {
282 pckt->flow.port16[0] = tcp->dest;
283 pckt->flow.port16[1] = tcp->source;
284 }
285 return 1;
286}
287
288static __attribute__ ((noinline))
289bool encap_v6(struct xdp_md *xdp, struct ctl_value *cval,
290 struct packet_description *pckt,
291 struct real_definition *dst, __u32 pkt_bytes)
292{
293 struct eth_hdr *new_eth;
294 struct eth_hdr *old_eth;
295 struct ipv6hdr *ip6h;
296 __u32 ip_suffix;
297 void *data_end;
298 void *data;
299
300 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct ipv6hdr)))
301 return 0;
302 data = (void *)(long)xdp->data;
303 data_end = (void *)(long)xdp->data_end;
304 new_eth = data;
305 ip6h = data + sizeof(struct eth_hdr);
306 old_eth = data + sizeof(struct ipv6hdr);
307 if (new_eth + 1 > data_end ||
308 old_eth + 1 > data_end || ip6h + 1 > data_end)
309 return 0;
310 memcpy(new_eth->eth_dest, cval->mac, 6);
311 memcpy(new_eth->eth_source, old_eth->eth_dest, 6);
312 new_eth->eth_proto = 56710;
313 ip6h->version = 6;
314 ip6h->priority = 0;
315 memset(ip6h->flow_lbl, 0, sizeof(ip6h->flow_lbl));
316
317 ip6h->nexthdr = IPPROTO_IPV6;
318 ip_suffix = pckt->flow.srcv6[3] ^ pckt->flow.port16[0];
319 ip6h->payload_len =
320 __builtin_bswap16(pkt_bytes + sizeof(struct ipv6hdr));
321 ip6h->hop_limit = 4;
322
323 ip6h->saddr.in6_u.u6_addr32[0] = 1;
324 ip6h->saddr.in6_u.u6_addr32[1] = 2;
325 ip6h->saddr.in6_u.u6_addr32[2] = 3;
326 ip6h->saddr.in6_u.u6_addr32[3] = ip_suffix;
327 memcpy(ip6h->daddr.in6_u.u6_addr32, dst->dstv6, 16);
328 return 1;
329}
330
331static __attribute__ ((noinline))
332bool encap_v4(struct xdp_md *xdp, struct ctl_value *cval,
333 struct packet_description *pckt,
334 struct real_definition *dst, __u32 pkt_bytes)
335{
336
337 __u32 ip_suffix = __builtin_bswap16(pckt->flow.port16[0]);
338 struct eth_hdr *new_eth;
339 struct eth_hdr *old_eth;
340 __u16 *next_iph_u16;
341 struct iphdr *iph;
342 __u32 csum = 0;
343 void *data_end;
344 void *data;
345
346 ip_suffix <<= 15;
347 ip_suffix ^= pckt->flow.src;
348 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct iphdr)))
349 return 0;
350 data = (void *)(long)xdp->data;
351 data_end = (void *)(long)xdp->data_end;
352 new_eth = data;
353 iph = data + sizeof(struct eth_hdr);
354 old_eth = data + sizeof(struct iphdr);
355 if (new_eth + 1 > data_end ||
356 old_eth + 1 > data_end || iph + 1 > data_end)
357 return 0;
358 memcpy(new_eth->eth_dest, cval->mac, 6);
359 memcpy(new_eth->eth_source, old_eth->eth_dest, 6);
360 new_eth->eth_proto = 8;
361 iph->version = 4;
362 iph->ihl = 5;
363 iph->frag_off = 0;
364 iph->protocol = IPPROTO_IPIP;
365 iph->check = 0;
366 iph->tos = 1;
367 iph->tot_len = __builtin_bswap16(pkt_bytes + sizeof(struct iphdr));
368 /* don't update iph->daddr, since it will overwrite old eth_proto
369 * and multiple iterations of bpf_prog_run() will fail
370 */
371
372 iph->saddr = ((0xFFFF0000 & ip_suffix) | 4268) ^ dst->dst;
373 iph->ttl = 4;
374
375 next_iph_u16 = (__u16 *) iph;
376#pragma clang loop unroll(full)
377 for (int i = 0; i < sizeof(struct iphdr) >> 1; i++)
378 csum += *next_iph_u16++;
379 iph->check = ~((csum & 0xffff) + (csum >> 16));
380 if (bpf_xdp_adjust_head(xdp, (int)sizeof(struct iphdr)))
381 return 0;
382 return 1;
383}
384
385static __attribute__ ((noinline))
386bool decap_v6(struct xdp_md *xdp, void **data, void **data_end, bool inner_v4)
387{
388 struct eth_hdr *new_eth;
389 struct eth_hdr *old_eth;
390
391 old_eth = *data;
392 new_eth = *data + sizeof(struct ipv6hdr);
393 memcpy(new_eth->eth_source, old_eth->eth_source, 6);
394 memcpy(new_eth->eth_dest, old_eth->eth_dest, 6);
395 if (inner_v4)
396 new_eth->eth_proto = 8;
397 else
398 new_eth->eth_proto = 56710;
399 if (bpf_xdp_adjust_head(xdp, (int)sizeof(struct ipv6hdr)))
400 return 0;
401 *data = (void *)(long)xdp->data;
402 *data_end = (void *)(long)xdp->data_end;
403 return 1;
404}
405
406static __attribute__ ((noinline))
407bool decap_v4(struct xdp_md *xdp, void **data, void **data_end)
408{
409 struct eth_hdr *new_eth;
410 struct eth_hdr *old_eth;
411
412 old_eth = *data;
413 new_eth = *data + sizeof(struct iphdr);
414 memcpy(new_eth->eth_source, old_eth->eth_source, 6);
415 memcpy(new_eth->eth_dest, old_eth->eth_dest, 6);
416 new_eth->eth_proto = 8;
417 if (bpf_xdp_adjust_head(xdp, (int)sizeof(struct iphdr)))
418 return 0;
419 *data = (void *)(long)xdp->data;
420 *data_end = (void *)(long)xdp->data_end;
421 return 1;
422}
423
424static __attribute__ ((noinline))
425int swap_mac_and_send(void *data, void *data_end)
426{
427 unsigned char tmp_mac[6];
428 struct eth_hdr *eth;
429
430 eth = data;
431 memcpy(tmp_mac, eth->eth_source, 6);
432 memcpy(eth->eth_source, eth->eth_dest, 6);
433 memcpy(eth->eth_dest, tmp_mac, 6);
434 return XDP_TX;
435}
436
437static __attribute__ ((noinline))
438int send_icmp_reply(void *data, void *data_end)
439{
440 struct icmphdr *icmp_hdr;
441 __u16 *next_iph_u16;
442 __u32 tmp_addr = 0;
443 struct iphdr *iph;
444 __u32 csum1 = 0;
445 __u32 csum = 0;
446 __u64 off = 0;
447
448 if (data + sizeof(struct eth_hdr)
449 + sizeof(struct iphdr) + sizeof(struct icmphdr) > data_end)
450 return XDP_DROP;
451 off += sizeof(struct eth_hdr);
452 iph = data + off;
453 off += sizeof(struct iphdr);
454 icmp_hdr = data + off;
455 icmp_hdr->type = 0;
456 icmp_hdr->checksum += 0x0007;
457 iph->ttl = 4;
458 tmp_addr = iph->daddr;
459 iph->daddr = iph->saddr;
460 iph->saddr = tmp_addr;
461 iph->check = 0;
462 next_iph_u16 = (__u16 *) iph;
463#pragma clang loop unroll(full)
464 for (int i = 0; i < sizeof(struct iphdr) >> 1; i++)
465 csum += *next_iph_u16++;
466 iph->check = ~((csum & 0xffff) + (csum >> 16));
467 return swap_mac_and_send(data, data_end);
468}
469
470static __attribute__ ((noinline))
471int send_icmp6_reply(void *data, void *data_end)
472{
473 struct icmp6hdr *icmp_hdr;
474 struct ipv6hdr *ip6h;
475 __be32 tmp_addr[4];
476 __u64 off = 0;
477
478 if (data + sizeof(struct eth_hdr)
479 + sizeof(struct ipv6hdr) + sizeof(struct icmp6hdr) > data_end)
480 return XDP_DROP;
481 off += sizeof(struct eth_hdr);
482 ip6h = data + off;
483 off += sizeof(struct ipv6hdr);
484 icmp_hdr = data + off;
485 icmp_hdr->icmp6_type = 129;
486 icmp_hdr->icmp6_cksum -= 0x0001;
487 ip6h->hop_limit = 4;
488 memcpy(tmp_addr, ip6h->saddr.in6_u.u6_addr32, 16);
489 memcpy(ip6h->saddr.in6_u.u6_addr32, ip6h->daddr.in6_u.u6_addr32, 16);
490 memcpy(ip6h->daddr.in6_u.u6_addr32, tmp_addr, 16);
491 return swap_mac_and_send(data, data_end);
492}
493
494static __attribute__ ((noinline))
495int parse_icmpv6(void *data, void *data_end, __u64 off,
496 struct packet_description *pckt)
497{
498 struct icmp6hdr *icmp_hdr;
499 struct ipv6hdr *ip6h;
500
501 icmp_hdr = data + off;
502 if (icmp_hdr + 1 > data_end)
503 return XDP_DROP;
504 if (icmp_hdr->icmp6_type == 128)
505 return send_icmp6_reply(data, data_end);
506 if (icmp_hdr->icmp6_type != 3)
507 return XDP_PASS;
508 off += sizeof(struct icmp6hdr);
509 ip6h = data + off;
510 if (ip6h + 1 > data_end)
511 return XDP_DROP;
512 pckt->flow.proto = ip6h->nexthdr;
513 pckt->flags |= (1 << 0);
514 memcpy(pckt->flow.srcv6, ip6h->daddr.in6_u.u6_addr32, 16);
515 memcpy(pckt->flow.dstv6, ip6h->saddr.in6_u.u6_addr32, 16);
516 return -1;
517}
518
519static __attribute__ ((noinline))
520int parse_icmp(void *data, void *data_end, __u64 off,
521 struct packet_description *pckt)
522{
523 struct icmphdr *icmp_hdr;
524 struct iphdr *iph;
525
526 icmp_hdr = data + off;
527 if (icmp_hdr + 1 > data_end)
528 return XDP_DROP;
529 if (icmp_hdr->type == 8)
530 return send_icmp_reply(data, data_end);
531 if ((icmp_hdr->type != 3) || (icmp_hdr->code != 4))
532 return XDP_PASS;
533 off += sizeof(struct icmphdr);
534 iph = data + off;
535 if (iph + 1 > data_end)
536 return XDP_DROP;
537 if (iph->ihl != 5)
538 return XDP_DROP;
539 pckt->flow.proto = iph->protocol;
540 pckt->flags |= (1 << 0);
541 pckt->flow.src = iph->daddr;
542 pckt->flow.dst = iph->saddr;
543 return -1;
544}
545
546static __attribute__ ((noinline))
547__u32 get_packet_hash(struct packet_description *pckt,
548 bool hash_16bytes)
549{
550 if (hash_16bytes)
551 return jhash_2words(jhash(pckt->flow.srcv6, 16, 12),
552 pckt->flow.ports, 24);
553 else
554 return jhash_2words(pckt->flow.src, pckt->flow.ports,
555 24);
556}
557
558__attribute__ ((noinline))
559static bool get_packet_dst(struct real_definition **real,
560 struct packet_description *pckt,
561 struct vip_meta *vip_info,
562 bool is_ipv6, void *lru_map)
563{
564 struct real_pos_lru new_dst_lru = { };
565 bool hash_16bytes = is_ipv6;
566 __u32 *real_pos, hash, key;
567 __u64 cur_time;
568
569 if (vip_info->flags & (1 << 2))
570 hash_16bytes = 1;
571 if (vip_info->flags & (1 << 3)) {
572 pckt->flow.port16[0] = pckt->flow.port16[1];
573 memset(pckt->flow.srcv6, 0, 16);
574 }
575 hash = get_packet_hash(pckt, hash_16bytes);
576 if (hash != 0x358459b7 /* jhash of ipv4 packet */ &&
577 hash != 0x2f4bc6bb /* jhash of ipv6 packet */)
578 return 0;
579 key = 2 * vip_info->vip_num + hash % 2;
580 real_pos = bpf_map_lookup_elem(&ch_rings, &key);
581 if (!real_pos)
582 return 0;
583 key = *real_pos;
584 *real = bpf_map_lookup_elem(&reals, &key);
585 if (!(*real))
586 return 0;
587 if (!(vip_info->flags & (1 << 1))) {
588 __u32 conn_rate_key = 512 + 2;
589 struct lb_stats *conn_rate_stats =
590 bpf_map_lookup_elem(&stats, &conn_rate_key);
591
592 if (!conn_rate_stats)
593 return 1;
594 cur_time = bpf_ktime_get_ns();
595 if ((cur_time - conn_rate_stats->v2) >> 32 > 0xffFFFF) {
596 conn_rate_stats->v1 = 1;
597 conn_rate_stats->v2 = cur_time;
598 } else {
599 conn_rate_stats->v1 += 1;
600 if (conn_rate_stats->v1 >= 1)
601 return 1;
602 }
603 if (pckt->flow.proto == IPPROTO_UDP)
604 new_dst_lru.atime = cur_time;
605 new_dst_lru.pos = key;
606 bpf_map_update_elem(lru_map, &pckt->flow, &new_dst_lru, 0);
607 }
608 return 1;
609}
610
611__attribute__ ((noinline))
612static void connection_table_lookup(struct real_definition **real,
613 struct packet_description *pckt,
614 void *lru_map)
615{
616
617 struct real_pos_lru *dst_lru;
618 __u64 cur_time;
619 __u32 key;
620
621 dst_lru = bpf_map_lookup_elem(lru_map, &pckt->flow);
622 if (!dst_lru)
623 return;
624 if (pckt->flow.proto == IPPROTO_UDP) {
625 cur_time = bpf_ktime_get_ns();
626 if (cur_time - dst_lru->atime > 300000)
627 return;
628 dst_lru->atime = cur_time;
629 }
630 key = dst_lru->pos;
631 *real = bpf_map_lookup_elem(&reals, &key);
632}
633
634/* don't believe your eyes!
635 * below function has 6 arguments whereas bpf and llvm allow maximum of 5
636 * but since it's _static_ llvm can optimize one argument away
637 */
638__attribute__ ((noinline))
639static int process_l3_headers_v6(struct packet_description *pckt,
640 __u8 *protocol, __u64 off,
641 __u16 *pkt_bytes, void *data,
642 void *data_end)
643{
644 struct ipv6hdr *ip6h;
645 __u64 iph_len;
646 int action;
647
648 ip6h = data + off;
649 if (ip6h + 1 > data_end)
650 return XDP_DROP;
651 iph_len = sizeof(struct ipv6hdr);
652 *protocol = ip6h->nexthdr;
653 pckt->flow.proto = *protocol;
654 *pkt_bytes = __builtin_bswap16(ip6h->payload_len);
655 off += iph_len;
656 if (*protocol == 45) {
657 return XDP_DROP;
658 } else if (*protocol == 59) {
659 action = parse_icmpv6(data, data_end, off, pckt);
660 if (action >= 0)
661 return action;
662 } else {
663 memcpy(pckt->flow.srcv6, ip6h->saddr.in6_u.u6_addr32, 16);
664 memcpy(pckt->flow.dstv6, ip6h->daddr.in6_u.u6_addr32, 16);
665 }
666 return -1;
667}
668
669__attribute__ ((noinline))
670static int process_l3_headers_v4(struct packet_description *pckt,
671 __u8 *protocol, __u64 off,
672 __u16 *pkt_bytes, void *data,
673 void *data_end)
674{
675 struct iphdr *iph;
676 __u64 iph_len;
677 int action;
678
679 iph = data + off;
680 if (iph + 1 > data_end)
681 return XDP_DROP;
682 if (iph->ihl != 5)
683 return XDP_DROP;
684 *protocol = iph->protocol;
685 pckt->flow.proto = *protocol;
686 *pkt_bytes = __builtin_bswap16(iph->tot_len);
687 off += 20;
688 if (iph->frag_off & 65343)
689 return XDP_DROP;
690 if (*protocol == IPPROTO_ICMP) {
691 action = parse_icmp(data, data_end, off, pckt);
692 if (action >= 0)
693 return action;
694 } else {
695 pckt->flow.src = iph->saddr;
696 pckt->flow.dst = iph->daddr;
697 }
698 return -1;
699}
700
701__attribute__ ((noinline))
702static int process_packet(void *data, __u64 off, void *data_end,
703 bool is_ipv6, struct xdp_md *xdp)
704{
705
706 struct real_definition *dst = NULL;
707 struct packet_description pckt = { };
708 struct vip_definition vip = { };
709 struct lb_stats *data_stats;
710 struct eth_hdr *eth = data;
711 void *lru_map = &lru_cache;
712 struct vip_meta *vip_info;
713 __u32 lru_stats_key = 513;
714 __u32 mac_addr_pos = 0;
715 __u32 stats_key = 512;
716 struct ctl_value *cval;
717 __u16 pkt_bytes;
718 __u64 iph_len;
719 __u8 protocol;
720 __u32 vip_num;
721 int action;
722
723 if (is_ipv6)
724 action = process_l3_headers_v6(&pckt, &protocol, off,
725 &pkt_bytes, data, data_end);
726 else
727 action = process_l3_headers_v4(&pckt, &protocol, off,
728 &pkt_bytes, data, data_end);
729 if (action >= 0)
730 return action;
731 protocol = pckt.flow.proto;
732 if (protocol == IPPROTO_TCP) {
733 if (!parse_tcp(data, data_end, is_ipv6, &pckt))
734 return XDP_DROP;
735 } else if (protocol == IPPROTO_UDP) {
736 if (!parse_udp(data, data_end, is_ipv6, &pckt))
737 return XDP_DROP;
738 } else {
739 return XDP_TX;
740 }
741
742 if (is_ipv6)
743 memcpy(vip.vipv6, pckt.flow.dstv6, 16);
744 else
745 vip.vip = pckt.flow.dst;
746 vip.port = pckt.flow.port16[1];
747 vip.proto = pckt.flow.proto;
748 vip_info = bpf_map_lookup_elem(&vip_map, &vip);
749 if (!vip_info) {
750 vip.port = 0;
751 vip_info = bpf_map_lookup_elem(&vip_map, &vip);
752 if (!vip_info)
753 return XDP_PASS;
754 if (!(vip_info->flags & (1 << 4)))
755 pckt.flow.port16[1] = 0;
756 }
757 if (data_end - data > 1400)
758 return XDP_DROP;
759 data_stats = bpf_map_lookup_elem(&stats, &stats_key);
760 if (!data_stats)
761 return XDP_DROP;
762 data_stats->v1 += 1;
763 if (!dst) {
764 if (vip_info->flags & (1 << 0))
765 pckt.flow.port16[0] = 0;
766 if (!(pckt.flags & (1 << 1)) && !(vip_info->flags & (1 << 1)))
767 connection_table_lookup(&dst, &pckt, lru_map);
768 if (dst)
769 goto out;
770 if (pckt.flow.proto == IPPROTO_TCP) {
771 struct lb_stats *lru_stats =
772 bpf_map_lookup_elem(&stats, &lru_stats_key);
773
774 if (!lru_stats)
775 return XDP_DROP;
776 if (pckt.flags & (1 << 1))
777 lru_stats->v1 += 1;
778 else
779 lru_stats->v2 += 1;
780 }
781 if (!get_packet_dst(&dst, &pckt, vip_info, is_ipv6, lru_map))
782 return XDP_DROP;
783 data_stats->v2 += 1;
784 }
785out:
786 cval = bpf_map_lookup_elem(&ctl_array, &mac_addr_pos);
787 if (!cval)
788 return XDP_DROP;
789 if (dst->flags & (1 << 0)) {
790 if (!encap_v6(xdp, cval, &pckt, dst, pkt_bytes))
791 return XDP_DROP;
792 } else {
793 if (!encap_v4(xdp, cval, &pckt, dst, pkt_bytes))
794 return XDP_DROP;
795 }
796 vip_num = vip_info->vip_num;
797 data_stats = bpf_map_lookup_elem(&stats, &vip_num);
798 if (!data_stats)
799 return XDP_DROP;
800 data_stats->v1 += 1;
801 data_stats->v2 += pkt_bytes;
802
803 data = (void *)(long)xdp->data;
804 data_end = (void *)(long)xdp->data_end;
805 if (data + 4 > data_end)
806 return XDP_DROP;
807 *(u32 *)data = dst->dst;
808 return XDP_DROP;
809}
810
811__attribute__ ((section("xdp-test"), used))
812int balancer_ingress(struct xdp_md *ctx)
813{
814 void *data = (void *)(long)ctx->data;
815 void *data_end = (void *)(long)ctx->data_end;
816 struct eth_hdr *eth = data;
817 __u32 eth_proto;
818 __u32 nh_off;
819
820 nh_off = sizeof(struct eth_hdr);
821 if (data + nh_off > data_end)
822 return XDP_DROP;
823 eth_proto = eth->eth_proto;
824 if (eth_proto == 8)
825 return process_packet(data, nh_off, data_end, 0, ctx);
826 else if (eth_proto == 56710)
827 return process_packet(data, nh_off, data_end, 1, ctx);
828 else
829 return XDP_DROP;
830}
831
832char _license[] __attribute__ ((section("license"), used)) = "GPL";
833int _version __attribute__ ((section("version"), used)) = 1;
diff --git a/tools/testing/selftests/net/Makefile b/tools/testing/selftests/net/Makefile
index 500c74db746c..d7c30d366935 100644
--- a/tools/testing/selftests/net/Makefile
+++ b/tools/testing/selftests/net/Makefile
@@ -5,6 +5,7 @@ CFLAGS = -Wall -Wl,--no-as-needed -O2 -g
5CFLAGS += -I../../../../usr/include/ 5CFLAGS += -I../../../../usr/include/
6 6
7TEST_PROGS := run_netsocktests run_afpackettests test_bpf.sh netdevice.sh rtnetlink.sh 7TEST_PROGS := run_netsocktests run_afpackettests test_bpf.sh netdevice.sh rtnetlink.sh
8TEST_PROGS += fib_tests.sh
8TEST_GEN_FILES = socket 9TEST_GEN_FILES = socket
9TEST_GEN_FILES += psock_fanout psock_tpacket msg_zerocopy 10TEST_GEN_FILES += psock_fanout psock_tpacket msg_zerocopy
10TEST_GEN_PROGS = reuseport_bpf reuseport_bpf_cpu reuseport_bpf_numa 11TEST_GEN_PROGS = reuseport_bpf reuseport_bpf_cpu reuseport_bpf_numa
diff --git a/tools/testing/selftests/net/fib_tests.sh b/tools/testing/selftests/net/fib_tests.sh
new file mode 100755
index 000000000000..a9154eefb2e2
--- /dev/null
+++ b/tools/testing/selftests/net/fib_tests.sh
@@ -0,0 +1,429 @@
1#!/bin/bash
2# SPDX-License-Identifier: GPL-2.0
3
4# This test is for checking IPv4 and IPv6 FIB behavior in response to
5# different events.
6
7ret=0
8
9check_err()
10{
11 if [ $ret -eq 0 ]; then
12 ret=$1
13 fi
14}
15
16check_fail()
17{
18 if [ $1 -eq 0 ]; then
19 ret=1
20 fi
21}
22
23netns_create()
24{
25 local testns=$1
26
27 ip netns add $testns
28 ip netns exec $testns ip link set dev lo up
29}
30
31fib_unreg_unicast_test()
32{
33 ret=0
34
35 netns_create "testns"
36
37 ip netns exec testns ip link add dummy0 type dummy
38 ip netns exec testns ip link set dev dummy0 up
39
40 ip netns exec testns ip address add 198.51.100.1/24 dev dummy0
41 ip netns exec testns ip -6 address add 2001:db8:1::1/64 dev dummy0
42
43 ip netns exec testns ip route get fibmatch 198.51.100.2 &> /dev/null
44 check_err $?
45 ip netns exec testns ip -6 route get fibmatch 2001:db8:1::2 &> /dev/null
46 check_err $?
47
48 ip netns exec testns ip link del dev dummy0
49 check_err $?
50
51 ip netns exec testns ip route get fibmatch 198.51.100.2 &> /dev/null
52 check_fail $?
53 ip netns exec testns ip -6 route get fibmatch 2001:db8:1::2 &> /dev/null
54 check_fail $?
55
56 ip netns del testns
57
58 if [ $ret -ne 0 ]; then
59 echo "FAIL: unicast route test"
60 return 1
61 fi
62 echo "PASS: unicast route test"
63}
64
65fib_unreg_multipath_test()
66{
67 ret=0
68
69 netns_create "testns"
70
71 ip netns exec testns ip link add dummy0 type dummy
72 ip netns exec testns ip link set dev dummy0 up
73
74 ip netns exec testns ip link add dummy1 type dummy
75 ip netns exec testns ip link set dev dummy1 up
76
77 ip netns exec testns ip address add 198.51.100.1/24 dev dummy0
78 ip netns exec testns ip -6 address add 2001:db8:1::1/64 dev dummy0
79
80 ip netns exec testns ip address add 192.0.2.1/24 dev dummy1
81 ip netns exec testns ip -6 address add 2001:db8:2::1/64 dev dummy1
82
83 ip netns exec testns ip route add 203.0.113.0/24 \
84 nexthop via 198.51.100.2 dev dummy0 \
85 nexthop via 192.0.2.2 dev dummy1
86 ip netns exec testns ip -6 route add 2001:db8:3::/64 \
87 nexthop via 2001:db8:1::2 dev dummy0 \
88 nexthop via 2001:db8:2::2 dev dummy1
89
90 ip netns exec testns ip route get fibmatch 203.0.113.1 &> /dev/null
91 check_err $?
92 ip netns exec testns ip -6 route get fibmatch 2001:db8:3::1 &> /dev/null
93 check_err $?
94
95 ip netns exec testns ip link del dev dummy0
96 check_err $?
97
98 ip netns exec testns ip route get fibmatch 203.0.113.1 &> /dev/null
99 check_fail $?
100 ip netns exec testns ip -6 route get fibmatch 2001:db8:3::1 &> /dev/null
101 # In IPv6 we do not flush the entire multipath route.
102 check_err $?
103
104 ip netns exec testns ip link del dev dummy1
105
106 ip netns del testns
107
108 if [ $ret -ne 0 ]; then
109 echo "FAIL: multipath route test"
110 return 1
111 fi
112 echo "PASS: multipath route test"
113}
114
115fib_unreg_test()
116{
117 echo "Running netdev unregister tests"
118
119 fib_unreg_unicast_test
120 fib_unreg_multipath_test
121}
122
123fib_down_unicast_test()
124{
125 ret=0
126
127 netns_create "testns"
128
129 ip netns exec testns ip link add dummy0 type dummy
130 ip netns exec testns ip link set dev dummy0 up
131
132 ip netns exec testns ip address add 198.51.100.1/24 dev dummy0
133 ip netns exec testns ip -6 address add 2001:db8:1::1/64 dev dummy0
134
135 ip netns exec testns ip route get fibmatch 198.51.100.2 &> /dev/null
136 check_err $?
137 ip netns exec testns ip -6 route get fibmatch 2001:db8:1::2 &> /dev/null
138 check_err $?
139
140 ip netns exec testns ip link set dev dummy0 down
141 check_err $?
142
143 ip netns exec testns ip route get fibmatch 198.51.100.2 &> /dev/null
144 check_fail $?
145 ip netns exec testns ip -6 route get fibmatch 2001:db8:1::2 &> /dev/null
146 check_fail $?
147
148 ip netns exec testns ip link del dev dummy0
149
150 ip netns del testns
151
152 if [ $ret -ne 0 ]; then
153 echo "FAIL: unicast route test"
154 return 1
155 fi
156 echo "PASS: unicast route test"
157}
158
159fib_down_multipath_test_do()
160{
161 local down_dev=$1
162 local up_dev=$2
163
164 ip netns exec testns ip route get fibmatch 203.0.113.1 \
165 oif $down_dev &> /dev/null
166 check_fail $?
167 ip netns exec testns ip -6 route get fibmatch 2001:db8:3::1 \
168 oif $down_dev &> /dev/null
169 check_fail $?
170
171 ip netns exec testns ip route get fibmatch 203.0.113.1 \
172 oif $up_dev &> /dev/null
173 check_err $?
174 ip netns exec testns ip -6 route get fibmatch 2001:db8:3::1 \
175 oif $up_dev &> /dev/null
176 check_err $?
177
178 ip netns exec testns ip route get fibmatch 203.0.113.1 | \
179 grep $down_dev | grep -q "dead linkdown"
180 check_err $?
181 ip netns exec testns ip -6 route get fibmatch 2001:db8:3::1 | \
182 grep $down_dev | grep -q "dead linkdown"
183 check_err $?
184
185 ip netns exec testns ip route get fibmatch 203.0.113.1 | \
186 grep $up_dev | grep -q "dead linkdown"
187 check_fail $?
188 ip netns exec testns ip -6 route get fibmatch 2001:db8:3::1 | \
189 grep $up_dev | grep -q "dead linkdown"
190 check_fail $?
191}
192
193fib_down_multipath_test()
194{
195 ret=0
196
197 netns_create "testns"
198
199 ip netns exec testns ip link add dummy0 type dummy
200 ip netns exec testns ip link set dev dummy0 up
201
202 ip netns exec testns ip link add dummy1 type dummy
203 ip netns exec testns ip link set dev dummy1 up
204
205 ip netns exec testns ip address add 198.51.100.1/24 dev dummy0
206 ip netns exec testns ip -6 address add 2001:db8:1::1/64 dev dummy0
207
208 ip netns exec testns ip address add 192.0.2.1/24 dev dummy1
209 ip netns exec testns ip -6 address add 2001:db8:2::1/64 dev dummy1
210
211 ip netns exec testns ip route add 203.0.113.0/24 \
212 nexthop via 198.51.100.2 dev dummy0 \
213 nexthop via 192.0.2.2 dev dummy1
214 ip netns exec testns ip -6 route add 2001:db8:3::/64 \
215 nexthop via 2001:db8:1::2 dev dummy0 \
216 nexthop via 2001:db8:2::2 dev dummy1
217
218 ip netns exec testns ip route get fibmatch 203.0.113.1 &> /dev/null
219 check_err $?
220 ip netns exec testns ip -6 route get fibmatch 2001:db8:3::1 &> /dev/null
221 check_err $?
222
223 ip netns exec testns ip link set dev dummy0 down
224 check_err $?
225
226 fib_down_multipath_test_do "dummy0" "dummy1"
227
228 ip netns exec testns ip link set dev dummy0 up
229 check_err $?
230 ip netns exec testns ip link set dev dummy1 down
231 check_err $?
232
233 fib_down_multipath_test_do "dummy1" "dummy0"
234
235 ip netns exec testns ip link set dev dummy0 down
236 check_err $?
237
238 ip netns exec testns ip route get fibmatch 203.0.113.1 &> /dev/null
239 check_fail $?
240 ip netns exec testns ip -6 route get fibmatch 2001:db8:3::1 &> /dev/null
241 check_fail $?
242
243 ip netns exec testns ip link del dev dummy1
244 ip netns exec testns ip link del dev dummy0
245
246 ip netns del testns
247
248 if [ $ret -ne 0 ]; then
249 echo "FAIL: multipath route test"
250 return 1
251 fi
252 echo "PASS: multipath route test"
253}
254
255fib_down_test()
256{
257 echo "Running netdev down tests"
258
259 fib_down_unicast_test
260 fib_down_multipath_test
261}
262
263fib_carrier_local_test()
264{
265 ret=0
266
267 # Local routes should not be affected when carrier changes.
268 netns_create "testns"
269
270 ip netns exec testns ip link add dummy0 type dummy
271 ip netns exec testns ip link set dev dummy0 up
272
273 ip netns exec testns ip link set dev dummy0 carrier on
274
275 ip netns exec testns ip address add 198.51.100.1/24 dev dummy0
276 ip netns exec testns ip -6 address add 2001:db8:1::1/64 dev dummy0
277
278 ip netns exec testns ip route get fibmatch 198.51.100.1 &> /dev/null
279 check_err $?
280 ip netns exec testns ip -6 route get fibmatch 2001:db8:1::1 &> /dev/null
281 check_err $?
282
283 ip netns exec testns ip route get fibmatch 198.51.100.1 | \
284 grep -q "linkdown"
285 check_fail $?
286 ip netns exec testns ip -6 route get fibmatch 2001:db8:1::1 | \
287 grep -q "linkdown"
288 check_fail $?
289
290 ip netns exec testns ip link set dev dummy0 carrier off
291
292 ip netns exec testns ip route get fibmatch 198.51.100.1 &> /dev/null
293 check_err $?
294 ip netns exec testns ip -6 route get fibmatch 2001:db8:1::1 &> /dev/null
295 check_err $?
296
297 ip netns exec testns ip route get fibmatch 198.51.100.1 | \
298 grep -q "linkdown"
299 check_fail $?
300 ip netns exec testns ip -6 route get fibmatch 2001:db8:1::1 | \
301 grep -q "linkdown"
302 check_fail $?
303
304 ip netns exec testns ip address add 192.0.2.1/24 dev dummy0
305 ip netns exec testns ip -6 address add 2001:db8:2::1/64 dev dummy0
306
307 ip netns exec testns ip route get fibmatch 192.0.2.1 &> /dev/null
308 check_err $?
309 ip netns exec testns ip -6 route get fibmatch 2001:db8:2::1 &> /dev/null
310 check_err $?
311
312 ip netns exec testns ip route get fibmatch 192.0.2.1 | \
313 grep -q "linkdown"
314 check_fail $?
315 ip netns exec testns ip -6 route get fibmatch 2001:db8:2::1 | \
316 grep -q "linkdown"
317 check_fail $?
318
319 ip netns exec testns ip link del dev dummy0
320
321 ip netns del testns
322
323 if [ $ret -ne 0 ]; then
324 echo "FAIL: local route carrier test"
325 return 1
326 fi
327 echo "PASS: local route carrier test"
328}
329
330fib_carrier_unicast_test()
331{
332 ret=0
333
334 netns_create "testns"
335
336 ip netns exec testns ip link add dummy0 type dummy
337 ip netns exec testns ip link set dev dummy0 up
338
339 ip netns exec testns ip link set dev dummy0 carrier on
340
341 ip netns exec testns ip address add 198.51.100.1/24 dev dummy0
342 ip netns exec testns ip -6 address add 2001:db8:1::1/64 dev dummy0
343
344 ip netns exec testns ip route get fibmatch 198.51.100.2 &> /dev/null
345 check_err $?
346 ip netns exec testns ip -6 route get fibmatch 2001:db8:1::2 &> /dev/null
347 check_err $?
348
349 ip netns exec testns ip route get fibmatch 198.51.100.2 | \
350 grep -q "linkdown"
351 check_fail $?
352 ip netns exec testns ip -6 route get fibmatch 2001:db8:1::2 | \
353 grep -q "linkdown"
354 check_fail $?
355
356 ip netns exec testns ip link set dev dummy0 carrier off
357
358 ip netns exec testns ip route get fibmatch 198.51.100.2 &> /dev/null
359 check_err $?
360 ip netns exec testns ip -6 route get fibmatch 2001:db8:1::2 &> /dev/null
361 check_err $?
362
363 ip netns exec testns ip route get fibmatch 198.51.100.2 | \
364 grep -q "linkdown"
365 check_err $?
366 ip netns exec testns ip -6 route get fibmatch 2001:db8:1::2 | \
367 grep -q "linkdown"
368 check_err $?
369
370 ip netns exec testns ip address add 192.0.2.1/24 dev dummy0
371 ip netns exec testns ip -6 address add 2001:db8:2::1/64 dev dummy0
372
373 ip netns exec testns ip route get fibmatch 192.0.2.2 &> /dev/null
374 check_err $?
375 ip netns exec testns ip -6 route get fibmatch 2001:db8:2::2 &> /dev/null
376 check_err $?
377
378 ip netns exec testns ip route get fibmatch 192.0.2.2 | \
379 grep -q "linkdown"
380 check_err $?
381 ip netns exec testns ip -6 route get fibmatch 2001:db8:2::2 | \
382 grep -q "linkdown"
383 check_err $?
384
385 ip netns exec testns ip link del dev dummy0
386
387 ip netns del testns
388
389 if [ $ret -ne 0 ]; then
390 echo "FAIL: unicast route carrier test"
391 return 1
392 fi
393 echo "PASS: unicast route carrier test"
394}
395
396fib_carrier_test()
397{
398 echo "Running netdev carrier change tests"
399
400 fib_carrier_local_test
401 fib_carrier_unicast_test
402}
403
404fib_test()
405{
406 fib_unreg_test
407 fib_down_test
408 fib_carrier_test
409}
410
411if [ "$(id -u)" -ne 0 ];then
412 echo "SKIP: Need root privileges"
413 exit 0
414fi
415
416if [ ! -x "$(command -v ip)" ]; then
417 echo "SKIP: Could not run test without ip tool"
418 exit 0
419fi
420
421ip route help 2>&1 | grep -q fibmatch
422if [ $? -ne 0 ]; then
423 echo "SKIP: iproute2 too old, missing fibmatch"
424 exit 0
425fi
426
427fib_test
428
429exit $ret
diff --git a/tools/testing/selftests/net/msg_zerocopy.c b/tools/testing/selftests/net/msg_zerocopy.c
index 3ab6ec403905..e11fe84de0fd 100644
--- a/tools/testing/selftests/net/msg_zerocopy.c
+++ b/tools/testing/selftests/net/msg_zerocopy.c
@@ -259,22 +259,28 @@ static int setup_ip6h(struct ipv6hdr *ip6h, uint16_t payload_len)
259 return sizeof(*ip6h); 259 return sizeof(*ip6h);
260} 260}
261 261
262static void setup_sockaddr(int domain, const char *str_addr, void *sockaddr) 262
263static void setup_sockaddr(int domain, const char *str_addr,
264 struct sockaddr_storage *sockaddr)
263{ 265{
264 struct sockaddr_in6 *addr6 = (void *) sockaddr; 266 struct sockaddr_in6 *addr6 = (void *) sockaddr;
265 struct sockaddr_in *addr4 = (void *) sockaddr; 267 struct sockaddr_in *addr4 = (void *) sockaddr;
266 268
267 switch (domain) { 269 switch (domain) {
268 case PF_INET: 270 case PF_INET:
271 memset(addr4, 0, sizeof(*addr4));
269 addr4->sin_family = AF_INET; 272 addr4->sin_family = AF_INET;
270 addr4->sin_port = htons(cfg_port); 273 addr4->sin_port = htons(cfg_port);
271 if (inet_pton(AF_INET, str_addr, &(addr4->sin_addr)) != 1) 274 if (str_addr &&
275 inet_pton(AF_INET, str_addr, &(addr4->sin_addr)) != 1)
272 error(1, 0, "ipv4 parse error: %s", str_addr); 276 error(1, 0, "ipv4 parse error: %s", str_addr);
273 break; 277 break;
274 case PF_INET6: 278 case PF_INET6:
279 memset(addr6, 0, sizeof(*addr6));
275 addr6->sin6_family = AF_INET6; 280 addr6->sin6_family = AF_INET6;
276 addr6->sin6_port = htons(cfg_port); 281 addr6->sin6_port = htons(cfg_port);
277 if (inet_pton(AF_INET6, str_addr, &(addr6->sin6_addr)) != 1) 282 if (str_addr &&
283 inet_pton(AF_INET6, str_addr, &(addr6->sin6_addr)) != 1)
278 error(1, 0, "ipv6 parse error: %s", str_addr); 284 error(1, 0, "ipv6 parse error: %s", str_addr);
279 break; 285 break;
280 default: 286 default:
@@ -603,6 +609,7 @@ static void parse_opts(int argc, char **argv)
603 sizeof(struct tcphdr) - 609 sizeof(struct tcphdr) -
604 40 /* max tcp options */; 610 40 /* max tcp options */;
605 int c; 611 int c;
612 char *daddr = NULL, *saddr = NULL;
606 613
607 cfg_payload_len = max_payload_len; 614 cfg_payload_len = max_payload_len;
608 615
@@ -627,7 +634,7 @@ static void parse_opts(int argc, char **argv)
627 cfg_cpu = strtol(optarg, NULL, 0); 634 cfg_cpu = strtol(optarg, NULL, 0);
628 break; 635 break;
629 case 'D': 636 case 'D':
630 setup_sockaddr(cfg_family, optarg, &cfg_dst_addr); 637 daddr = optarg;
631 break; 638 break;
632 case 'i': 639 case 'i':
633 cfg_ifindex = if_nametoindex(optarg); 640 cfg_ifindex = if_nametoindex(optarg);
@@ -638,7 +645,7 @@ static void parse_opts(int argc, char **argv)
638 cfg_cork_mixed = true; 645 cfg_cork_mixed = true;
639 break; 646 break;
640 case 'p': 647 case 'p':
641 cfg_port = htons(strtoul(optarg, NULL, 0)); 648 cfg_port = strtoul(optarg, NULL, 0);
642 break; 649 break;
643 case 'r': 650 case 'r':
644 cfg_rx = true; 651 cfg_rx = true;
@@ -647,7 +654,7 @@ static void parse_opts(int argc, char **argv)
647 cfg_payload_len = strtoul(optarg, NULL, 0); 654 cfg_payload_len = strtoul(optarg, NULL, 0);
648 break; 655 break;
649 case 'S': 656 case 'S':
650 setup_sockaddr(cfg_family, optarg, &cfg_src_addr); 657 saddr = optarg;
651 break; 658 break;
652 case 't': 659 case 't':
653 cfg_runtime_ms = 200 + strtoul(optarg, NULL, 10) * 1000; 660 cfg_runtime_ms = 200 + strtoul(optarg, NULL, 10) * 1000;
@@ -660,6 +667,8 @@ static void parse_opts(int argc, char **argv)
660 break; 667 break;
661 } 668 }
662 } 669 }
670 setup_sockaddr(cfg_family, daddr, &cfg_dst_addr);
671 setup_sockaddr(cfg_family, saddr, &cfg_src_addr);
663 672
664 if (cfg_payload_len > max_payload_len) 673 if (cfg_payload_len > max_payload_len)
665 error(1, 0, "-s: payload exceeds max (%d)", max_payload_len); 674 error(1, 0, "-s: payload exceeds max (%d)", max_payload_len);
diff --git a/tools/testing/selftests/net/rtnetlink.sh b/tools/testing/selftests/net/rtnetlink.sh
index 5215493166c9..a622eeecc3a6 100755
--- a/tools/testing/selftests/net/rtnetlink.sh
+++ b/tools/testing/selftests/net/rtnetlink.sh
@@ -502,6 +502,231 @@ kci_test_macsec()
502 echo "PASS: macsec" 502 echo "PASS: macsec"
503} 503}
504 504
505kci_test_gretap()
506{
507 testns="testns"
508 DEV_NS=gretap00
509 ret=0
510
511 ip netns add "$testns"
512 if [ $? -ne 0 ]; then
513 echo "SKIP gretap tests: cannot add net namespace $testns"
514 return 1
515 fi
516
517 ip link help gretap 2>&1 | grep -q "^Usage:"
518 if [ $? -ne 0 ];then
519 echo "SKIP: gretap: iproute2 too old"
520 return 1
521 fi
522
523 # test native tunnel
524 ip netns exec "$testns" ip link add dev "$DEV_NS" type gretap seq \
525 key 102 local 172.16.1.100 remote 172.16.1.200
526 check_err $?
527
528 ip netns exec "$testns" ip addr add dev "$DEV_NS" 10.1.1.100/24
529 check_err $?
530
531 ip netns exec "$testns" ip link set dev $DEV_NS up
532 check_err $?
533
534 ip netns exec "$testns" ip link del "$DEV_NS"
535 check_err $?
536
537 # test external mode
538 ip netns exec "$testns" ip link add dev "$DEV_NS" type gretap external
539 check_err $?
540
541 ip netns exec "$testns" ip link del "$DEV_NS"
542 check_err $?
543
544 if [ $ret -ne 0 ]; then
545 echo "FAIL: gretap"
546 return 1
547 fi
548 echo "PASS: gretap"
549
550 ip netns del "$testns"
551}
552
553kci_test_ip6gretap()
554{
555 testns="testns"
556 DEV_NS=ip6gretap00
557 ret=0
558
559 ip netns add "$testns"
560 if [ $? -ne 0 ]; then
561 echo "SKIP ip6gretap tests: cannot add net namespace $testns"
562 return 1
563 fi
564
565 ip link help ip6gretap 2>&1 | grep -q "^Usage:"
566 if [ $? -ne 0 ];then
567 echo "SKIP: ip6gretap: iproute2 too old"
568 return 1
569 fi
570
571 # test native tunnel
572 ip netns exec "$testns" ip link add dev "$DEV_NS" type ip6gretap seq \
573 key 102 local fc00:100::1 remote fc00:100::2
574 check_err $?
575
576 ip netns exec "$testns" ip addr add dev "$DEV_NS" fc00:200::1/96
577 check_err $?
578
579 ip netns exec "$testns" ip link set dev $DEV_NS up
580 check_err $?
581
582 ip netns exec "$testns" ip link del "$DEV_NS"
583 check_err $?
584
585 # test external mode
586 ip netns exec "$testns" ip link add dev "$DEV_NS" type ip6gretap external
587 check_err $?
588
589 ip netns exec "$testns" ip link del "$DEV_NS"
590 check_err $?
591
592 if [ $ret -ne 0 ]; then
593 echo "FAIL: ip6gretap"
594 return 1
595 fi
596 echo "PASS: ip6gretap"
597
598 ip netns del "$testns"
599}
600
601kci_test_erspan()
602{
603 testns="testns"
604 DEV_NS=erspan00
605 ret=0
606
607 ip link help erspan 2>&1 | grep -q "^Usage:"
608 if [ $? -ne 0 ];then
609 echo "SKIP: erspan: iproute2 too old"
610 return 1
611 fi
612
613 ip netns add "$testns"
614 if [ $? -ne 0 ]; then
615 echo "SKIP erspan tests: cannot add net namespace $testns"
616 return 1
617 fi
618
619 # test native tunnel erspan v1
620 ip netns exec "$testns" ip link add dev "$DEV_NS" type erspan seq \
621 key 102 local 172.16.1.100 remote 172.16.1.200 \
622 erspan_ver 1 erspan 488
623 check_err $?
624
625 ip netns exec "$testns" ip addr add dev "$DEV_NS" 10.1.1.100/24
626 check_err $?
627
628 ip netns exec "$testns" ip link set dev $DEV_NS up
629 check_err $?
630
631 ip netns exec "$testns" ip link del "$DEV_NS"
632 check_err $?
633
634 # test native tunnel erspan v2
635 ip netns exec "$testns" ip link add dev "$DEV_NS" type erspan seq \
636 key 102 local 172.16.1.100 remote 172.16.1.200 \
637 erspan_ver 2 erspan_dir ingress erspan_hwid 7
638 check_err $?
639
640 ip netns exec "$testns" ip addr add dev "$DEV_NS" 10.1.1.100/24
641 check_err $?
642
643 ip netns exec "$testns" ip link set dev $DEV_NS up
644 check_err $?
645
646 ip netns exec "$testns" ip link del "$DEV_NS"
647 check_err $?
648
649 # test external mode
650 ip netns exec "$testns" ip link add dev "$DEV_NS" type erspan external
651 check_err $?
652
653 ip netns exec "$testns" ip link del "$DEV_NS"
654 check_err $?
655
656 if [ $ret -ne 0 ]; then
657 echo "FAIL: erspan"
658 return 1
659 fi
660 echo "PASS: erspan"
661
662 ip netns del "$testns"
663}
664
665kci_test_ip6erspan()
666{
667 testns="testns"
668 DEV_NS=ip6erspan00
669 ret=0
670
671 ip link help ip6erspan 2>&1 | grep -q "^Usage:"
672 if [ $? -ne 0 ];then
673 echo "SKIP: ip6erspan: iproute2 too old"
674 return 1
675 fi
676
677 ip netns add "$testns"
678 if [ $? -ne 0 ]; then
679 echo "SKIP ip6erspan tests: cannot add net namespace $testns"
680 return 1
681 fi
682
683 # test native tunnel ip6erspan v1
684 ip netns exec "$testns" ip link add dev "$DEV_NS" type ip6erspan seq \
685 key 102 local fc00:100::1 remote fc00:100::2 \
686 erspan_ver 1 erspan 488
687 check_err $?
688
689 ip netns exec "$testns" ip addr add dev "$DEV_NS" 10.1.1.100/24
690 check_err $?
691
692 ip netns exec "$testns" ip link set dev $DEV_NS up
693 check_err $?
694
695 ip netns exec "$testns" ip link del "$DEV_NS"
696 check_err $?
697
698 # test native tunnel ip6erspan v2
699 ip netns exec "$testns" ip link add dev "$DEV_NS" type ip6erspan seq \
700 key 102 local fc00:100::1 remote fc00:100::2 \
701 erspan_ver 2 erspan_dir ingress erspan_hwid 7
702 check_err $?
703
704 ip netns exec "$testns" ip addr add dev "$DEV_NS" 10.1.1.100/24
705 check_err $?
706
707 ip netns exec "$testns" ip link set dev $DEV_NS up
708 check_err $?
709
710 ip netns exec "$testns" ip link del "$DEV_NS"
711 check_err $?
712
713 # test external mode
714 ip netns exec "$testns" ip link add dev "$DEV_NS" \
715 type ip6erspan external
716 check_err $?
717
718 ip netns exec "$testns" ip link del "$DEV_NS"
719 check_err $?
720
721 if [ $ret -ne 0 ]; then
722 echo "FAIL: ip6erspan"
723 return 1
724 fi
725 echo "PASS: ip6erspan"
726
727 ip netns del "$testns"
728}
729
505kci_test_rtnl() 730kci_test_rtnl()
506{ 731{
507 kci_add_dummy 732 kci_add_dummy
@@ -514,6 +739,10 @@ kci_test_rtnl()
514 kci_test_route_get 739 kci_test_route_get
515 kci_test_tc 740 kci_test_tc
516 kci_test_gre 741 kci_test_gre
742 kci_test_gretap
743 kci_test_ip6gretap
744 kci_test_erspan
745 kci_test_ip6erspan
517 kci_test_bridge 746 kci_test_bridge
518 kci_test_addrlabel 747 kci_test_addrlabel
519 kci_test_ifalias 748 kci_test_ifalias
diff --git a/tools/virtio/linux/kernel.h b/tools/virtio/linux/kernel.h
index 395521a7a8d8..fca8381bbe04 100644
--- a/tools/virtio/linux/kernel.h
+++ b/tools/virtio/linux/kernel.h
@@ -118,7 +118,7 @@ static inline void free_page(unsigned long addr)
118#define dev_err(dev, format, ...) fprintf (stderr, format, ## __VA_ARGS__) 118#define dev_err(dev, format, ...) fprintf (stderr, format, ## __VA_ARGS__)
119#define dev_warn(dev, format, ...) fprintf (stderr, format, ## __VA_ARGS__) 119#define dev_warn(dev, format, ...) fprintf (stderr, format, ## __VA_ARGS__)
120 120
121#define WARN_ON_ONCE(cond) ((cond) && fprintf (stderr, "WARNING\n")) 121#define WARN_ON_ONCE(cond) ((cond) ? fprintf (stderr, "WARNING\n") : 0)
122 122
123#define min(x, y) ({ \ 123#define min(x, y) ({ \
124 typeof(x) _min1 = (x); \ 124 typeof(x) _min1 = (x); \
diff --git a/tools/virtio/linux/thread_info.h b/tools/virtio/linux/thread_info.h
new file mode 100644
index 000000000000..e0f610d08006
--- /dev/null
+++ b/tools/virtio/linux/thread_info.h
@@ -0,0 +1 @@
#define check_copy_size(A, B, C) (1)
diff --git a/tools/virtio/ringtest/main.h b/tools/virtio/ringtest/main.h
index 5706e075adf2..301d59bfcd0a 100644
--- a/tools/virtio/ringtest/main.h
+++ b/tools/virtio/ringtest/main.h
@@ -111,7 +111,7 @@ static inline void busy_wait(void)
111} 111}
112 112
113#if defined(__x86_64__) || defined(__i386__) 113#if defined(__x86_64__) || defined(__i386__)
114#define smp_mb() asm volatile("lock; addl $0,-128(%%rsp)" ::: "memory", "cc") 114#define smp_mb() asm volatile("lock; addl $0,-132(%%rsp)" ::: "memory", "cc")
115#else 115#else
116/* 116/*
117 * Not using __ATOMIC_SEQ_CST since gcc docs say they are only synchronized 117 * Not using __ATOMIC_SEQ_CST since gcc docs say they are only synchronized
@@ -134,4 +134,61 @@ static inline void busy_wait(void)
134 barrier(); \ 134 barrier(); \
135} while (0) 135} while (0)
136 136
137#if defined(__i386__) || defined(__x86_64__) || defined(__s390x__)
138#define smp_wmb() barrier()
139#else
140#define smp_wmb() smp_release()
141#endif
142
143#ifdef __alpha__
144#define smp_read_barrier_depends() smp_acquire()
145#else
146#define smp_read_barrier_depends() do {} while(0)
147#endif
148
149static __always_inline
150void __read_once_size(const volatile void *p, void *res, int size)
151{
152 switch (size) { \
153 case 1: *(unsigned char *)res = *(volatile unsigned char *)p; break; \
154 case 2: *(unsigned short *)res = *(volatile unsigned short *)p; break; \
155 case 4: *(unsigned int *)res = *(volatile unsigned int *)p; break; \
156 case 8: *(unsigned long long *)res = *(volatile unsigned long long *)p; break; \
157 default: \
158 barrier(); \
159 __builtin_memcpy((void *)res, (const void *)p, size); \
160 barrier(); \
161 } \
162}
163
164static __always_inline void __write_once_size(volatile void *p, void *res, int size)
165{
166 switch (size) {
167 case 1: *(volatile unsigned char *)p = *(unsigned char *)res; break;
168 case 2: *(volatile unsigned short *)p = *(unsigned short *)res; break;
169 case 4: *(volatile unsigned int *)p = *(unsigned int *)res; break;
170 case 8: *(volatile unsigned long long *)p = *(unsigned long long *)res; break;
171 default:
172 barrier();
173 __builtin_memcpy((void *)p, (const void *)res, size);
174 barrier();
175 }
176}
177
178#define READ_ONCE(x) \
179({ \
180 union { typeof(x) __val; char __c[1]; } __u; \
181 __read_once_size(&(x), __u.__c, sizeof(x)); \
182 smp_read_barrier_depends(); /* Enforce dependency ordering from x */ \
183 __u.__val; \
184})
185
186#define WRITE_ONCE(x, val) \
187({ \
188 union { typeof(x) __val; char __c[1]; } __u = \
189 { .__val = (typeof(x)) (val) }; \
190 __write_once_size(&(x), __u.__c, sizeof(x)); \
191 __u.__val; \
192})
193
137#endif 194#endif
diff --git a/tools/virtio/ringtest/ptr_ring.c b/tools/virtio/ringtest/ptr_ring.c
index e6e81305ef46..477899c12c51 100644
--- a/tools/virtio/ringtest/ptr_ring.c
+++ b/tools/virtio/ringtest/ptr_ring.c
@@ -187,7 +187,7 @@ bool enable_kick()
187 187
188bool avail_empty() 188bool avail_empty()
189{ 189{
190 return !__ptr_ring_peek(&array); 190 return __ptr_ring_empty(&array);
191} 191}
192 192
193bool use_buf(unsigned *lenp, void **bufp) 193bool use_buf(unsigned *lenp, void **bufp)
generated by cgit v1.2.2 (git 2.25.0) at 2025-08-23 17:41:13 -0400