aboutsummaryrefslogtreecommitdiffstats
path: root/tools
diff options
context:
space:
mode:
authorDavid S. Miller <davem@davemloft.net>2018-05-07 23:35:08 -0400
committerDavid S. Miller <davem@davemloft.net>2018-05-07 23:35:08 -0400
commit01adc4851a8090b46c7a5ed9cfc4b97e65abfbf4 (patch)
tree2ae02593d7139962648dff203f3f9701e34ccbc3 /tools
parent18b338f5f9539512e76fd9ebd4c6ca1a0e159e2b (diff)
parente94fa1d93117e7f1eb783dc9cae6c70650944449 (diff)
Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next
Minor conflict, a CHECK was placed into an if() statement in net-next, whilst a newline was added to that CHECK call in 'net'. Thanks to Daniel for the merge resolution. Signed-off-by: David S. Miller <davem@davemloft.net>
Diffstat (limited to 'tools')
-rw-r--r--tools/bpf/bpftool/Documentation/bpftool-map.rst40
-rw-r--r--tools/bpf/bpftool/Documentation/bpftool.rst2
-rw-r--r--tools/bpf/bpftool/Makefile7
-rw-r--r--tools/bpf/bpftool/bash-completion/bpftool36
-rw-r--r--tools/bpf/bpftool/common.c77
-rw-r--r--tools/bpf/bpftool/main.h7
-rw-r--r--tools/bpf/bpftool/map.c80
-rw-r--r--tools/bpf/bpftool/map_perf_ring.c347
-rw-r--r--tools/bpf/bpftool/prog.c8
-rw-r--r--tools/include/uapi/linux/bpf.h93
-rw-r--r--tools/include/uapi/linux/erspan.h52
-rw-r--r--tools/testing/selftests/bpf/Makefile4
-rw-r--r--tools/testing/selftests/bpf/bpf_helpers.h2
-rw-r--r--tools/testing/selftests/bpf/test_get_stack_rawtp.c102
-rw-r--r--tools/testing/selftests/bpf/test_progs.c242
-rw-r--r--tools/testing/selftests/bpf/test_stacktrace_build_id.c20
-rw-r--r--tools/testing/selftests/bpf/test_stacktrace_map.c19
-rw-r--r--tools/testing/selftests/bpf/test_verifier.c311
-rw-r--r--tools/testing/selftests/bpf/trace_helpers.c180
-rw-r--r--tools/testing/selftests/bpf/trace_helpers.h23
20 files changed, 1524 insertions, 128 deletions
diff --git a/tools/bpf/bpftool/Documentation/bpftool-map.rst b/tools/bpf/bpftool/Documentation/bpftool-map.rst
index 5f512b14bff9..a6258bc8ec4f 100644
--- a/tools/bpf/bpftool/Documentation/bpftool-map.rst
+++ b/tools/bpf/bpftool/Documentation/bpftool-map.rst
@@ -22,17 +22,19 @@ MAP COMMANDS
22============= 22=============
23 23
24| **bpftool** **map { show | list }** [*MAP*] 24| **bpftool** **map { show | list }** [*MAP*]
25| **bpftool** **map dump** *MAP* 25| **bpftool** **map dump** *MAP*
26| **bpftool** **map update** *MAP* **key** [**hex**] *BYTES* **value** [**hex**] *VALUE* [*UPDATE_FLAGS*] 26| **bpftool** **map update** *MAP* **key** *DATA* **value** *VALUE* [*UPDATE_FLAGS*]
27| **bpftool** **map lookup** *MAP* **key** [**hex**] *BYTES* 27| **bpftool** **map lookup** *MAP* **key** *DATA*
28| **bpftool** **map getnext** *MAP* [**key** [**hex**] *BYTES*] 28| **bpftool** **map getnext** *MAP* [**key** *DATA*]
29| **bpftool** **map delete** *MAP* **key** [**hex**] *BYTES* 29| **bpftool** **map delete** *MAP* **key** *DATA*
30| **bpftool** **map pin** *MAP* *FILE* 30| **bpftool** **map pin** *MAP* *FILE*
31| **bpftool** **map event_pipe** *MAP* [**cpu** *N* **index** *M*]
31| **bpftool** **map help** 32| **bpftool** **map help**
32| 33|
33| *MAP* := { **id** *MAP_ID* | **pinned** *FILE* } 34| *MAP* := { **id** *MAP_ID* | **pinned** *FILE* }
35| *DATA* := { [**hex**] *BYTES* }
34| *PROG* := { **id** *PROG_ID* | **pinned** *FILE* | **tag** *PROG_TAG* } 36| *PROG* := { **id** *PROG_ID* | **pinned** *FILE* | **tag** *PROG_TAG* }
35| *VALUE* := { *BYTES* | *MAP* | *PROG* } 37| *VALUE* := { *DATA* | *MAP* | *PROG* }
36| *UPDATE_FLAGS* := { **any** | **exist** | **noexist** } 38| *UPDATE_FLAGS* := { **any** | **exist** | **noexist** }
37 39
38DESCRIPTION 40DESCRIPTION
@@ -48,7 +50,7 @@ DESCRIPTION
48 **bpftool map dump** *MAP* 50 **bpftool map dump** *MAP*
49 Dump all entries in a given *MAP*. 51 Dump all entries in a given *MAP*.
50 52
51 **bpftool map update** *MAP* **key** [**hex**] *BYTES* **value** [**hex**] *VALUE* [*UPDATE_FLAGS*] 53 **bpftool map update** *MAP* **key** *DATA* **value** *VALUE* [*UPDATE_FLAGS*]
52 Update map entry for a given *KEY*. 54 Update map entry for a given *KEY*.
53 55
54 *UPDATE_FLAGS* can be one of: **any** update existing entry 56 *UPDATE_FLAGS* can be one of: **any** update existing entry
@@ -61,13 +63,13 @@ DESCRIPTION
61 the bytes are parsed as decimal values, unless a "0x" prefix 63 the bytes are parsed as decimal values, unless a "0x" prefix
62 (for hexadecimal) or a "0" prefix (for octal) is provided. 64 (for hexadecimal) or a "0" prefix (for octal) is provided.
63 65
64 **bpftool map lookup** *MAP* **key** [**hex**] *BYTES* 66 **bpftool map lookup** *MAP* **key** *DATA*
65 Lookup **key** in the map. 67 Lookup **key** in the map.
66 68
67 **bpftool map getnext** *MAP* [**key** [**hex**] *BYTES*] 69 **bpftool map getnext** *MAP* [**key** *DATA*]
68 Get next key. If *key* is not specified, get first key. 70 Get next key. If *key* is not specified, get first key.
69 71
70 **bpftool map delete** *MAP* **key** [**hex**] *BYTES* 72 **bpftool map delete** *MAP* **key** *DATA*
71 Remove entry from the map. 73 Remove entry from the map.
72 74
73 **bpftool map pin** *MAP* *FILE* 75 **bpftool map pin** *MAP* *FILE*
@@ -75,6 +77,22 @@ DESCRIPTION
75 77
76 Note: *FILE* must be located in *bpffs* mount. 78 Note: *FILE* must be located in *bpffs* mount.
77 79
80 **bpftool** **map event_pipe** *MAP* [**cpu** *N* **index** *M*]
81 Read events from a BPF_MAP_TYPE_PERF_EVENT_ARRAY map.
82
83 Install perf rings into a perf event array map and dump
84 output of any bpf_perf_event_output() call in the kernel.
85 By default read the number of CPUs on the system and
86 install perf ring for each CPU in the corresponding index
87 in the array.
88
89 If **cpu** and **index** are specified, install perf ring
90 for given **cpu** at **index** in the array (single ring).
91
92 Note that installing a perf ring into an array will silently
93 replace any existing ring. Any other application will stop
94 receiving events if it installed its rings earlier.
95
78 **bpftool map help** 96 **bpftool map help**
79 Print short help message. 97 Print short help message.
80 98
diff --git a/tools/bpf/bpftool/Documentation/bpftool.rst b/tools/bpf/bpftool/Documentation/bpftool.rst
index 20689a321ffe..564cb0d9692b 100644
--- a/tools/bpf/bpftool/Documentation/bpftool.rst
+++ b/tools/bpf/bpftool/Documentation/bpftool.rst
@@ -23,7 +23,7 @@ SYNOPSIS
23 23
24 *MAP-COMMANDS* := 24 *MAP-COMMANDS* :=
25 { **show** | **list** | **dump** | **update** | **lookup** | **getnext** | **delete** 25 { **show** | **list** | **dump** | **update** | **lookup** | **getnext** | **delete**
26 | **pin** | **help** } 26 | **pin** | **event_pipe** | **help** }
27 27
28 *PROG-COMMANDS* := { **show** | **list** | **dump jited** | **dump xlated** | **pin** 28 *PROG-COMMANDS* := { **show** | **list** | **dump jited** | **dump xlated** | **pin**
29 | **load** | **help** } 29 | **load** | **help** }
diff --git a/tools/bpf/bpftool/Makefile b/tools/bpf/bpftool/Makefile
index 4e69782c4a79..892dbf095bff 100644
--- a/tools/bpf/bpftool/Makefile
+++ b/tools/bpf/bpftool/Makefile
@@ -39,7 +39,12 @@ CC = gcc
39 39
40CFLAGS += -O2 40CFLAGS += -O2
41CFLAGS += -W -Wall -Wextra -Wno-unused-parameter -Wshadow -Wno-missing-field-initializers 41CFLAGS += -W -Wall -Wextra -Wno-unused-parameter -Wshadow -Wno-missing-field-initializers
42CFLAGS += -DPACKAGE='"bpftool"' -D__EXPORTED_HEADERS__ -I$(srctree)/tools/include/uapi -I$(srctree)/tools/include -I$(srctree)/tools/lib/bpf -I$(srctree)/kernel/bpf/ 42CFLAGS += -DPACKAGE='"bpftool"' -D__EXPORTED_HEADERS__ \
43 -I$(srctree)/kernel/bpf/ \
44 -I$(srctree)/tools/include \
45 -I$(srctree)/tools/include/uapi \
46 -I$(srctree)/tools/lib/bpf \
47 -I$(srctree)/tools/perf
43CFLAGS += -DBPFTOOL_VERSION='"$(BPFTOOL_VERSION)"' 48CFLAGS += -DBPFTOOL_VERSION='"$(BPFTOOL_VERSION)"'
44LIBS = -lelf -lbfd -lopcodes $(LIBBPF) 49LIBS = -lelf -lbfd -lopcodes $(LIBBPF)
45 50
diff --git a/tools/bpf/bpftool/bash-completion/bpftool b/tools/bpf/bpftool/bash-completion/bpftool
index 852d84a98acd..b301c9b315f1 100644
--- a/tools/bpf/bpftool/bash-completion/bpftool
+++ b/tools/bpf/bpftool/bash-completion/bpftool
@@ -1,6 +1,6 @@
1# bpftool(8) bash completion -*- shell-script -*- 1# bpftool(8) bash completion -*- shell-script -*-
2# 2#
3# Copyright (C) 2017 Netronome Systems, Inc. 3# Copyright (C) 2017-2018 Netronome Systems, Inc.
4# 4#
5# This software is dual licensed under the GNU General License 5# This software is dual licensed under the GNU General License
6# Version 2, June 1991 as shown in the file COPYING in the top-level 6# Version 2, June 1991 as shown in the file COPYING in the top-level
@@ -79,6 +79,14 @@ _bpftool_get_map_ids()
79 command sed -n 's/.*"id": \(.*\),$/\1/p' )" -- "$cur" ) ) 79 command sed -n 's/.*"id": \(.*\),$/\1/p' )" -- "$cur" ) )
80} 80}
81 81
82_bpftool_get_perf_map_ids()
83{
84 COMPREPLY+=( $( compgen -W "$( bpftool -jp map 2>&1 | \
85 command grep -C2 perf_event_array | \
86 command sed -n 's/.*"id": \(.*\),$/\1/p' )" -- "$cur" ) )
87}
88
89
82_bpftool_get_prog_ids() 90_bpftool_get_prog_ids()
83{ 91{
84 COMPREPLY+=( $( compgen -W "$( bpftool -jp prog 2>&1 | \ 92 COMPREPLY+=( $( compgen -W "$( bpftool -jp prog 2>&1 | \
@@ -359,10 +367,34 @@ _bpftool()
359 fi 367 fi
360 return 0 368 return 0
361 ;; 369 ;;
370 event_pipe)
371 case $prev in
372 $command)
373 COMPREPLY=( $( compgen -W "$MAP_TYPE" -- "$cur" ) )
374 return 0
375 ;;
376 id)
377 _bpftool_get_perf_map_ids
378 return 0
379 ;;
380 cpu)
381 return 0
382 ;;
383 index)
384 return 0
385 ;;
386 *)
387 _bpftool_once_attr 'cpu'
388 _bpftool_once_attr 'index'
389 return 0
390 ;;
391 esac
392 ;;
362 *) 393 *)
363 [[ $prev == $object ]] && \ 394 [[ $prev == $object ]] && \
364 COMPREPLY=( $( compgen -W 'delete dump getnext help \ 395 COMPREPLY=( $( compgen -W 'delete dump getnext help \
365 lookup pin show list update' -- "$cur" ) ) 396 lookup pin event_pipe show list update' -- \
397 "$cur" ) )
366 ;; 398 ;;
367 esac 399 esac
368 ;; 400 ;;
diff --git a/tools/bpf/bpftool/common.c b/tools/bpf/bpftool/common.c
index 465995281dcd..32f9e397a6c0 100644
--- a/tools/bpf/bpftool/common.c
+++ b/tools/bpf/bpftool/common.c
@@ -1,5 +1,5 @@
1/* 1/*
2 * Copyright (C) 2017 Netronome Systems, Inc. 2 * Copyright (C) 2017-2018 Netronome Systems, Inc.
3 * 3 *
4 * This software is dual licensed under the GNU General License Version 2, 4 * This software is dual licensed under the GNU General License Version 2,
5 * June 1991 as shown in the file COPYING in the top-level directory of this 5 * June 1991 as shown in the file COPYING in the top-level directory of this
@@ -33,6 +33,7 @@
33 33
34/* Author: Jakub Kicinski <kubakici@wp.pl> */ 34/* Author: Jakub Kicinski <kubakici@wp.pl> */
35 35
36#include <ctype.h>
36#include <errno.h> 37#include <errno.h>
37#include <fcntl.h> 38#include <fcntl.h>
38#include <fts.h> 39#include <fts.h>
@@ -330,6 +331,16 @@ char *get_fdinfo(int fd, const char *key)
330 return NULL; 331 return NULL;
331} 332}
332 333
334void print_data_json(uint8_t *data, size_t len)
335{
336 unsigned int i;
337
338 jsonw_start_array(json_wtr);
339 for (i = 0; i < len; i++)
340 jsonw_printf(json_wtr, "%d", data[i]);
341 jsonw_end_array(json_wtr);
342}
343
333void print_hex_data_json(uint8_t *data, size_t len) 344void print_hex_data_json(uint8_t *data, size_t len)
334{ 345{
335 unsigned int i; 346 unsigned int i;
@@ -420,6 +431,70 @@ void delete_pinned_obj_table(struct pinned_obj_table *tab)
420 } 431 }
421} 432}
422 433
434unsigned int get_page_size(void)
435{
436 static int result;
437
438 if (!result)
439 result = getpagesize();
440 return result;
441}
442
443unsigned int get_possible_cpus(void)
444{
445 static unsigned int result;
446 char buf[128];
447 long int n;
448 char *ptr;
449 int fd;
450
451 if (result)
452 return result;
453
454 fd = open("/sys/devices/system/cpu/possible", O_RDONLY);
455 if (fd < 0) {
456 p_err("can't open sysfs possible cpus");
457 exit(-1);
458 }
459
460 n = read(fd, buf, sizeof(buf));
461 if (n < 2) {
462 p_err("can't read sysfs possible cpus");
463 exit(-1);
464 }
465 close(fd);
466
467 if (n == sizeof(buf)) {
468 p_err("read sysfs possible cpus overflow");
469 exit(-1);
470 }
471
472 ptr = buf;
473 n = 0;
474 while (*ptr && *ptr != '\n') {
475 unsigned int a, b;
476
477 if (sscanf(ptr, "%u-%u", &a, &b) == 2) {
478 n += b - a + 1;
479
480 ptr = strchr(ptr, '-') + 1;
481 } else if (sscanf(ptr, "%u", &a) == 1) {
482 n++;
483 } else {
484 assert(0);
485 }
486
487 while (isdigit(*ptr))
488 ptr++;
489 if (*ptr == ',')
490 ptr++;
491 }
492
493 result = n;
494
495 return result;
496}
497
423static char * 498static char *
424ifindex_to_name_ns(__u32 ifindex, __u32 ns_dev, __u32 ns_ino, char *buf) 499ifindex_to_name_ns(__u32 ifindex, __u32 ns_dev, __u32 ns_ino, char *buf)
425{ 500{
diff --git a/tools/bpf/bpftool/main.h b/tools/bpf/bpftool/main.h
index b8e9584d6246..6173cd997e7a 100644
--- a/tools/bpf/bpftool/main.h
+++ b/tools/bpf/bpftool/main.h
@@ -1,5 +1,5 @@
1/* 1/*
2 * Copyright (C) 2017 Netronome Systems, Inc. 2 * Copyright (C) 2017-2018 Netronome Systems, Inc.
3 * 3 *
4 * This software is dual licensed under the GNU General License Version 2, 4 * This software is dual licensed under the GNU General License Version 2,
5 * June 1991 as shown in the file COPYING in the top-level directory of this 5 * June 1991 as shown in the file COPYING in the top-level directory of this
@@ -117,14 +117,19 @@ int do_pin_fd(int fd, const char *name);
117 117
118int do_prog(int argc, char **arg); 118int do_prog(int argc, char **arg);
119int do_map(int argc, char **arg); 119int do_map(int argc, char **arg);
120int do_event_pipe(int argc, char **argv);
120int do_cgroup(int argc, char **arg); 121int do_cgroup(int argc, char **arg);
121 122
122int prog_parse_fd(int *argc, char ***argv); 123int prog_parse_fd(int *argc, char ***argv);
124int map_parse_fd_and_info(int *argc, char ***argv, void *info, __u32 *info_len);
123 125
124void disasm_print_insn(unsigned char *image, ssize_t len, int opcodes, 126void disasm_print_insn(unsigned char *image, ssize_t len, int opcodes,
125 const char *arch); 127 const char *arch);
128void print_data_json(uint8_t *data, size_t len);
126void print_hex_data_json(uint8_t *data, size_t len); 129void print_hex_data_json(uint8_t *data, size_t len);
127 130
131unsigned int get_page_size(void);
132unsigned int get_possible_cpus(void);
128const char *ifindex_to_bfd_name_ns(__u32 ifindex, __u64 ns_dev, __u64 ns_ino); 133const char *ifindex_to_bfd_name_ns(__u32 ifindex, __u64 ns_dev, __u64 ns_ino);
129 134
130#endif 135#endif
diff --git a/tools/bpf/bpftool/map.c b/tools/bpf/bpftool/map.c
index a6cdb640a0d7..af6766e956ba 100644
--- a/tools/bpf/bpftool/map.c
+++ b/tools/bpf/bpftool/map.c
@@ -1,5 +1,5 @@
1/* 1/*
2 * Copyright (C) 2017 Netronome Systems, Inc. 2 * Copyright (C) 2017-2018 Netronome Systems, Inc.
3 * 3 *
4 * This software is dual licensed under the GNU General License Version 2, 4 * This software is dual licensed under the GNU General License Version 2,
5 * June 1991 as shown in the file COPYING in the top-level directory of this 5 * June 1991 as shown in the file COPYING in the top-level directory of this
@@ -34,7 +34,6 @@
34/* Author: Jakub Kicinski <kubakici@wp.pl> */ 34/* Author: Jakub Kicinski <kubakici@wp.pl> */
35 35
36#include <assert.h> 36#include <assert.h>
37#include <ctype.h>
38#include <errno.h> 37#include <errno.h>
39#include <fcntl.h> 38#include <fcntl.h>
40#include <stdbool.h> 39#include <stdbool.h>
@@ -69,61 +68,6 @@ static const char * const map_type_name[] = {
69 [BPF_MAP_TYPE_CPUMAP] = "cpumap", 68 [BPF_MAP_TYPE_CPUMAP] = "cpumap",
70}; 69};
71 70
72static unsigned int get_possible_cpus(void)
73{
74 static unsigned int result;
75 char buf[128];
76 long int n;
77 char *ptr;
78 int fd;
79
80 if (result)
81 return result;
82
83 fd = open("/sys/devices/system/cpu/possible", O_RDONLY);
84 if (fd < 0) {
85 p_err("can't open sysfs possible cpus");
86 exit(-1);
87 }
88
89 n = read(fd, buf, sizeof(buf));
90 if (n < 2) {
91 p_err("can't read sysfs possible cpus");
92 exit(-1);
93 }
94 close(fd);
95
96 if (n == sizeof(buf)) {
97 p_err("read sysfs possible cpus overflow");
98 exit(-1);
99 }
100
101 ptr = buf;
102 n = 0;
103 while (*ptr && *ptr != '\n') {
104 unsigned int a, b;
105
106 if (sscanf(ptr, "%u-%u", &a, &b) == 2) {
107 n += b - a + 1;
108
109 ptr = strchr(ptr, '-') + 1;
110 } else if (sscanf(ptr, "%u", &a) == 1) {
111 n++;
112 } else {
113 assert(0);
114 }
115
116 while (isdigit(*ptr))
117 ptr++;
118 if (*ptr == ',')
119 ptr++;
120 }
121
122 result = n;
123
124 return result;
125}
126
127static bool map_is_per_cpu(__u32 type) 71static bool map_is_per_cpu(__u32 type)
128{ 72{
129 return type == BPF_MAP_TYPE_PERCPU_HASH || 73 return type == BPF_MAP_TYPE_PERCPU_HASH ||
@@ -186,8 +130,7 @@ static int map_parse_fd(int *argc, char ***argv)
186 return -1; 130 return -1;
187} 131}
188 132
189static int 133int map_parse_fd_and_info(int *argc, char ***argv, void *info, __u32 *info_len)
190map_parse_fd_and_info(int *argc, char ***argv, void *info, __u32 *info_len)
191{ 134{
192 int err; 135 int err;
193 int fd; 136 int fd;
@@ -873,23 +816,25 @@ static int do_help(int argc, char **argv)
873 816
874 fprintf(stderr, 817 fprintf(stderr,
875 "Usage: %s %s { show | list } [MAP]\n" 818 "Usage: %s %s { show | list } [MAP]\n"
876 " %s %s dump MAP\n" 819 " %s %s dump MAP\n"
877 " %s %s update MAP key [hex] BYTES value [hex] VALUE [UPDATE_FLAGS]\n" 820 " %s %s update MAP key DATA value VALUE [UPDATE_FLAGS]\n"
878 " %s %s lookup MAP key [hex] BYTES\n" 821 " %s %s lookup MAP key DATA\n"
879 " %s %s getnext MAP [key [hex] BYTES]\n" 822 " %s %s getnext MAP [key DATA]\n"
880 " %s %s delete MAP key [hex] BYTES\n" 823 " %s %s delete MAP key DATA\n"
881 " %s %s pin MAP FILE\n" 824 " %s %s pin MAP FILE\n"
825 " %s %s event_pipe MAP [cpu N index M]\n"
882 " %s %s help\n" 826 " %s %s help\n"
883 "\n" 827 "\n"
884 " MAP := { id MAP_ID | pinned FILE }\n" 828 " MAP := { id MAP_ID | pinned FILE }\n"
829 " DATA := { [hex] BYTES }\n"
885 " " HELP_SPEC_PROGRAM "\n" 830 " " HELP_SPEC_PROGRAM "\n"
886 " VALUE := { BYTES | MAP | PROG }\n" 831 " VALUE := { DATA | MAP | PROG }\n"
887 " UPDATE_FLAGS := { any | exist | noexist }\n" 832 " UPDATE_FLAGS := { any | exist | noexist }\n"
888 " " HELP_SPEC_OPTIONS "\n" 833 " " HELP_SPEC_OPTIONS "\n"
889 "", 834 "",
890 bin_name, argv[-2], bin_name, argv[-2], bin_name, argv[-2], 835 bin_name, argv[-2], bin_name, argv[-2], bin_name, argv[-2],
891 bin_name, argv[-2], bin_name, argv[-2], bin_name, argv[-2], 836 bin_name, argv[-2], bin_name, argv[-2], bin_name, argv[-2],
892 bin_name, argv[-2], bin_name, argv[-2]); 837 bin_name, argv[-2], bin_name, argv[-2], bin_name, argv[-2]);
893 838
894 return 0; 839 return 0;
895} 840}
@@ -904,6 +849,7 @@ static const struct cmd cmds[] = {
904 { "getnext", do_getnext }, 849 { "getnext", do_getnext },
905 { "delete", do_delete }, 850 { "delete", do_delete },
906 { "pin", do_pin }, 851 { "pin", do_pin },
852 { "event_pipe", do_event_pipe },
907 { 0 } 853 { 0 }
908}; 854};
909 855
diff --git a/tools/bpf/bpftool/map_perf_ring.c b/tools/bpf/bpftool/map_perf_ring.c
new file mode 100644
index 000000000000..c5a2ced8552d
--- /dev/null
+++ b/tools/bpf/bpftool/map_perf_ring.c
@@ -0,0 +1,347 @@
1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright (C) 2018 Netronome Systems, Inc. */
3/* This program is free software; you can redistribute it and/or
4 * modify it under the terms of version 2 of the GNU General Public
5 * License as published by the Free Software Foundation.
6 */
7#include <errno.h>
8#include <fcntl.h>
9#include <libbpf.h>
10#include <poll.h>
11#include <signal.h>
12#include <stdbool.h>
13#include <stdio.h>
14#include <stdlib.h>
15#include <string.h>
16#include <time.h>
17#include <unistd.h>
18#include <linux/bpf.h>
19#include <linux/perf_event.h>
20#include <sys/ioctl.h>
21#include <sys/mman.h>
22#include <sys/syscall.h>
23
24#include <bpf.h>
25#include <perf-sys.h>
26
27#include "main.h"
28
29#define MMAP_PAGE_CNT 16
30
31static bool stop;
32
33struct event_ring_info {
34 int fd;
35 int key;
36 unsigned int cpu;
37 void *mem;
38};
39
40struct perf_event_sample {
41 struct perf_event_header header;
42 __u32 size;
43 unsigned char data[];
44};
45
46static void int_exit(int signo)
47{
48 fprintf(stderr, "Stopping...\n");
49 stop = true;
50}
51
52static void
53print_bpf_output(struct event_ring_info *ring, struct perf_event_sample *e)
54{
55 struct {
56 struct perf_event_header header;
57 __u64 id;
58 __u64 lost;
59 } *lost = (void *)e;
60 struct timespec ts;
61
62 if (clock_gettime(CLOCK_MONOTONIC, &ts)) {
63 perror("Can't read clock for timestamp");
64 return;
65 }
66
67 if (json_output) {
68 jsonw_start_object(json_wtr);
69 jsonw_name(json_wtr, "timestamp");
70 jsonw_uint(json_wtr, ts.tv_sec * 1000000000ull + ts.tv_nsec);
71 jsonw_name(json_wtr, "type");
72 jsonw_uint(json_wtr, e->header.type);
73 jsonw_name(json_wtr, "cpu");
74 jsonw_uint(json_wtr, ring->cpu);
75 jsonw_name(json_wtr, "index");
76 jsonw_uint(json_wtr, ring->key);
77 if (e->header.type == PERF_RECORD_SAMPLE) {
78 jsonw_name(json_wtr, "data");
79 print_data_json(e->data, e->size);
80 } else if (e->header.type == PERF_RECORD_LOST) {
81 jsonw_name(json_wtr, "lost");
82 jsonw_start_object(json_wtr);
83 jsonw_name(json_wtr, "id");
84 jsonw_uint(json_wtr, lost->id);
85 jsonw_name(json_wtr, "count");
86 jsonw_uint(json_wtr, lost->lost);
87 jsonw_end_object(json_wtr);
88 }
89 jsonw_end_object(json_wtr);
90 } else {
91 if (e->header.type == PERF_RECORD_SAMPLE) {
92 printf("== @%ld.%ld CPU: %d index: %d =====\n",
93 (long)ts.tv_sec, ts.tv_nsec,
94 ring->cpu, ring->key);
95 fprint_hex(stdout, e->data, e->size, " ");
96 printf("\n");
97 } else if (e->header.type == PERF_RECORD_LOST) {
98 printf("lost %lld events\n", lost->lost);
99 } else {
100 printf("unknown event type=%d size=%d\n",
101 e->header.type, e->header.size);
102 }
103 }
104}
105
106static void
107perf_event_read(struct event_ring_info *ring, void **buf, size_t *buf_len)
108{
109 volatile struct perf_event_mmap_page *header = ring->mem;
110 __u64 buffer_size = MMAP_PAGE_CNT * get_page_size();
111 __u64 data_tail = header->data_tail;
112 __u64 data_head = header->data_head;
113 void *base, *begin, *end;
114
115 asm volatile("" ::: "memory"); /* in real code it should be smp_rmb() */
116 if (data_head == data_tail)
117 return;
118
119 base = ((char *)header) + get_page_size();
120
121 begin = base + data_tail % buffer_size;
122 end = base + data_head % buffer_size;
123
124 while (begin != end) {
125 struct perf_event_sample *e;
126
127 e = begin;
128 if (begin + e->header.size > base + buffer_size) {
129 long len = base + buffer_size - begin;
130
131 if (*buf_len < e->header.size) {
132 free(*buf);
133 *buf = malloc(e->header.size);
134 if (!*buf) {
135 fprintf(stderr,
136 "can't allocate memory");
137 stop = true;
138 return;
139 }
140 *buf_len = e->header.size;
141 }
142
143 memcpy(*buf, begin, len);
144 memcpy(*buf + len, base, e->header.size - len);
145 e = (void *)*buf;
146 begin = base + e->header.size - len;
147 } else if (begin + e->header.size == base + buffer_size) {
148 begin = base;
149 } else {
150 begin += e->header.size;
151 }
152
153 print_bpf_output(ring, e);
154 }
155
156 __sync_synchronize(); /* smp_mb() */
157 header->data_tail = data_head;
158}
159
160static int perf_mmap_size(void)
161{
162 return get_page_size() * (MMAP_PAGE_CNT + 1);
163}
164
165static void *perf_event_mmap(int fd)
166{
167 int mmap_size = perf_mmap_size();
168 void *base;
169
170 base = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
171 if (base == MAP_FAILED) {
172 p_err("event mmap failed: %s\n", strerror(errno));
173 return NULL;
174 }
175
176 return base;
177}
178
179static void perf_event_unmap(void *mem)
180{
181 if (munmap(mem, perf_mmap_size()))
182 fprintf(stderr, "Can't unmap ring memory!\n");
183}
184
185static int bpf_perf_event_open(int map_fd, int key, int cpu)
186{
187 struct perf_event_attr attr = {
188 .sample_type = PERF_SAMPLE_RAW,
189 .type = PERF_TYPE_SOFTWARE,
190 .config = PERF_COUNT_SW_BPF_OUTPUT,
191 };
192 int pmu_fd;
193
194 pmu_fd = sys_perf_event_open(&attr, -1, cpu, -1, 0);
195 if (pmu_fd < 0) {
196 p_err("failed to open perf event %d for CPU %d", key, cpu);
197 return -1;
198 }
199
200 if (bpf_map_update_elem(map_fd, &key, &pmu_fd, BPF_ANY)) {
201 p_err("failed to update map for event %d for CPU %d", key, cpu);
202 goto err_close;
203 }
204 if (ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0)) {
205 p_err("failed to enable event %d for CPU %d", key, cpu);
206 goto err_close;
207 }
208
209 return pmu_fd;
210
211err_close:
212 close(pmu_fd);
213 return -1;
214}
215
216int do_event_pipe(int argc, char **argv)
217{
218 int i, nfds, map_fd, index = -1, cpu = -1;
219 struct bpf_map_info map_info = {};
220 struct event_ring_info *rings;
221 size_t tmp_buf_sz = 0;
222 void *tmp_buf = NULL;
223 struct pollfd *pfds;
224 __u32 map_info_len;
225 bool do_all = true;
226
227 map_info_len = sizeof(map_info);
228 map_fd = map_parse_fd_and_info(&argc, &argv, &map_info, &map_info_len);
229 if (map_fd < 0)
230 return -1;
231
232 if (map_info.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) {
233 p_err("map is not a perf event array");
234 goto err_close_map;
235 }
236
237 while (argc) {
238 if (argc < 2)
239 BAD_ARG();
240
241 if (is_prefix(*argv, "cpu")) {
242 char *endptr;
243
244 NEXT_ARG();
245 cpu = strtoul(*argv, &endptr, 0);
246 if (*endptr) {
247 p_err("can't parse %s as CPU ID", **argv);
248 goto err_close_map;
249 }
250
251 NEXT_ARG();
252 } else if (is_prefix(*argv, "index")) {
253 char *endptr;
254
255 NEXT_ARG();
256 index = strtoul(*argv, &endptr, 0);
257 if (*endptr) {
258 p_err("can't parse %s as index", **argv);
259 goto err_close_map;
260 }
261
262 NEXT_ARG();
263 } else {
264 BAD_ARG();
265 }
266
267 do_all = false;
268 }
269
270 if (!do_all) {
271 if (index == -1 || cpu == -1) {
272 p_err("cpu and index must be specified together");
273 goto err_close_map;
274 }
275
276 nfds = 1;
277 } else {
278 nfds = min(get_possible_cpus(), map_info.max_entries);
279 cpu = 0;
280 index = 0;
281 }
282
283 rings = calloc(nfds, sizeof(rings[0]));
284 if (!rings)
285 goto err_close_map;
286
287 pfds = calloc(nfds, sizeof(pfds[0]));
288 if (!pfds)
289 goto err_free_rings;
290
291 for (i = 0; i < nfds; i++) {
292 rings[i].cpu = cpu + i;
293 rings[i].key = index + i;
294
295 rings[i].fd = bpf_perf_event_open(map_fd, rings[i].key,
296 rings[i].cpu);
297 if (rings[i].fd < 0)
298 goto err_close_fds_prev;
299
300 rings[i].mem = perf_event_mmap(rings[i].fd);
301 if (!rings[i].mem)
302 goto err_close_fds_current;
303
304 pfds[i].fd = rings[i].fd;
305 pfds[i].events = POLLIN;
306 }
307
308 signal(SIGINT, int_exit);
309 signal(SIGHUP, int_exit);
310 signal(SIGTERM, int_exit);
311
312 if (json_output)
313 jsonw_start_array(json_wtr);
314
315 while (!stop) {
316 poll(pfds, nfds, 200);
317 for (i = 0; i < nfds; i++)
318 perf_event_read(&rings[i], &tmp_buf, &tmp_buf_sz);
319 }
320 free(tmp_buf);
321
322 if (json_output)
323 jsonw_end_array(json_wtr);
324
325 for (i = 0; i < nfds; i++) {
326 perf_event_unmap(rings[i].mem);
327 close(rings[i].fd);
328 }
329 free(pfds);
330 free(rings);
331 close(map_fd);
332
333 return 0;
334
335err_close_fds_prev:
336 while (i--) {
337 perf_event_unmap(rings[i].mem);
338err_close_fds_current:
339 close(rings[i].fd);
340 }
341 free(pfds);
342err_free_rings:
343 free(rings);
344err_close_map:
345 close(map_fd);
346 return -1;
347}
diff --git a/tools/bpf/bpftool/prog.c b/tools/bpf/bpftool/prog.c
index e71a0a11afde..9bdfdf2d3fbe 100644
--- a/tools/bpf/bpftool/prog.c
+++ b/tools/bpf/bpftool/prog.c
@@ -96,7 +96,10 @@ static void print_boot_time(__u64 nsecs, char *buf, unsigned int size)
96 return; 96 return;
97 } 97 }
98 98
99 strftime(buf, size, "%b %d/%H:%M", &load_tm); 99 if (json_output)
100 strftime(buf, size, "%s", &load_tm);
101 else
102 strftime(buf, size, "%FT%T%z", &load_tm);
100} 103}
101 104
102static int prog_fd_by_tag(unsigned char *tag) 105static int prog_fd_by_tag(unsigned char *tag)
@@ -245,7 +248,8 @@ static void print_prog_json(struct bpf_prog_info *info, int fd)
245 print_boot_time(info->load_time, buf, sizeof(buf)); 248 print_boot_time(info->load_time, buf, sizeof(buf));
246 249
247 /* Piggy back on load_time, since 0 uid is a valid one */ 250 /* Piggy back on load_time, since 0 uid is a valid one */
248 jsonw_string_field(json_wtr, "loaded_at", buf); 251 jsonw_name(json_wtr, "loaded_at");
252 jsonw_printf(json_wtr, "%s", buf);
249 jsonw_uint_field(json_wtr, "uid", info->created_by_uid); 253 jsonw_uint_field(json_wtr, "uid", info->created_by_uid);
250 } 254 }
251 255
diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
index da77a9388947..83a95ae388dd 100644
--- a/tools/include/uapi/linux/bpf.h
+++ b/tools/include/uapi/linux/bpf.h
@@ -828,12 +828,12 @@ union bpf_attr {
828 * 828 *
829 * Also, be aware that the newer helper 829 * Also, be aware that the newer helper
830 * **bpf_perf_event_read_value**\ () is recommended over 830 * **bpf_perf_event_read_value**\ () is recommended over
831 * **bpf_perf_event_read*\ () in general. The latter has some ABI 831 * **bpf_perf_event_read**\ () in general. The latter has some ABI
832 * quirks where error and counter value are used as a return code 832 * quirks where error and counter value are used as a return code
833 * (which is wrong to do since ranges may overlap). This issue is 833 * (which is wrong to do since ranges may overlap). This issue is
834 * fixed with bpf_perf_event_read_value(), which at the same time 834 * fixed with **bpf_perf_event_read_value**\ (), which at the same
835 * provides more features over the **bpf_perf_event_read**\ () 835 * time provides more features over the **bpf_perf_event_read**\
836 * interface. Please refer to the description of 836 * () interface. Please refer to the description of
837 * **bpf_perf_event_read_value**\ () for details. 837 * **bpf_perf_event_read_value**\ () for details.
838 * Return 838 * Return
839 * The value of the perf event counter read from the map, or a 839 * The value of the perf event counter read from the map, or a
@@ -1361,7 +1361,7 @@ union bpf_attr {
1361 * Return 1361 * Return
1362 * 0 1362 * 0
1363 * 1363 *
1364 * int bpf_setsockopt(struct bpf_sock_ops_kern *bpf_socket, int level, int optname, char *optval, int optlen) 1364 * int bpf_setsockopt(struct bpf_sock_ops *bpf_socket, int level, int optname, char *optval, int optlen)
1365 * Description 1365 * Description
1366 * Emulate a call to **setsockopt()** on the socket associated to 1366 * Emulate a call to **setsockopt()** on the socket associated to
1367 * *bpf_socket*, which must be a full socket. The *level* at 1367 * *bpf_socket*, which must be a full socket. The *level* at
@@ -1435,7 +1435,7 @@ union bpf_attr {
1435 * Return 1435 * Return
1436 * **SK_PASS** on success, or **SK_DROP** on error. 1436 * **SK_PASS** on success, or **SK_DROP** on error.
1437 * 1437 *
1438 * int bpf_sock_map_update(struct bpf_sock_ops_kern *skops, struct bpf_map *map, void *key, u64 flags) 1438 * int bpf_sock_map_update(struct bpf_sock_ops *skops, struct bpf_map *map, void *key, u64 flags)
1439 * Description 1439 * Description
1440 * Add an entry to, or update a *map* referencing sockets. The 1440 * Add an entry to, or update a *map* referencing sockets. The
1441 * *skops* is used as a new value for the entry associated to 1441 * *skops* is used as a new value for the entry associated to
@@ -1533,7 +1533,7 @@ union bpf_attr {
1533 * Return 1533 * Return
1534 * 0 on success, or a negative error in case of failure. 1534 * 0 on success, or a negative error in case of failure.
1535 * 1535 *
1536 * int bpf_perf_prog_read_value(struct bpf_perf_event_data_kern *ctx, struct bpf_perf_event_value *buf, u32 buf_size) 1536 * int bpf_perf_prog_read_value(struct bpf_perf_event_data *ctx, struct bpf_perf_event_value *buf, u32 buf_size)
1537 * Description 1537 * Description
1538 * For en eBPF program attached to a perf event, retrieve the 1538 * For en eBPF program attached to a perf event, retrieve the
1539 * value of the event counter associated to *ctx* and store it in 1539 * value of the event counter associated to *ctx* and store it in
@@ -1544,7 +1544,7 @@ union bpf_attr {
1544 * Return 1544 * Return
1545 * 0 on success, or a negative error in case of failure. 1545 * 0 on success, or a negative error in case of failure.
1546 * 1546 *
1547 * int bpf_getsockopt(struct bpf_sock_ops_kern *bpf_socket, int level, int optname, char *optval, int optlen) 1547 * int bpf_getsockopt(struct bpf_sock_ops *bpf_socket, int level, int optname, char *optval, int optlen)
1548 * Description 1548 * Description
1549 * Emulate a call to **getsockopt()** on the socket associated to 1549 * Emulate a call to **getsockopt()** on the socket associated to
1550 * *bpf_socket*, which must be a full socket. The *level* at 1550 * *bpf_socket*, which must be a full socket. The *level* at
@@ -1588,7 +1588,7 @@ union bpf_attr {
1588 * Return 1588 * Return
1589 * 0 1589 * 0
1590 * 1590 *
1591 * int bpf_sock_ops_cb_flags_set(struct bpf_sock_ops_kern *bpf_sock, int argval) 1591 * int bpf_sock_ops_cb_flags_set(struct bpf_sock_ops *bpf_sock, int argval)
1592 * Description 1592 * Description
1593 * Attempt to set the value of the **bpf_sock_ops_cb_flags** field 1593 * Attempt to set the value of the **bpf_sock_ops_cb_flags** field
1594 * for the full TCP socket associated to *bpf_sock_ops* to 1594 * for the full TCP socket associated to *bpf_sock_ops* to
@@ -1721,7 +1721,7 @@ union bpf_attr {
1721 * Return 1721 * Return
1722 * 0 on success, or a negative error in case of failure. 1722 * 0 on success, or a negative error in case of failure.
1723 * 1723 *
1724 * int bpf_bind(struct bpf_sock_addr_kern *ctx, struct sockaddr *addr, int addr_len) 1724 * int bpf_bind(struct bpf_sock_addr *ctx, struct sockaddr *addr, int addr_len)
1725 * Description 1725 * Description
1726 * Bind the socket associated to *ctx* to the address pointed by 1726 * Bind the socket associated to *ctx* to the address pointed by
1727 * *addr*, of length *addr_len*. This allows for making outgoing 1727 * *addr*, of length *addr_len*. This allows for making outgoing
@@ -1767,6 +1767,64 @@ union bpf_attr {
1767 * **CONFIG_XFRM** configuration option. 1767 * **CONFIG_XFRM** configuration option.
1768 * Return 1768 * Return
1769 * 0 on success, or a negative error in case of failure. 1769 * 0 on success, or a negative error in case of failure.
1770 *
1771 * int bpf_get_stack(struct pt_regs *regs, void *buf, u32 size, u64 flags)
1772 * Description
1773 * Return a user or a kernel stack in bpf program provided buffer.
1774 * To achieve this, the helper needs *ctx*, which is a pointer
1775 * to the context on which the tracing program is executed.
1776 * To store the stacktrace, the bpf program provides *buf* with
1777 * a nonnegative *size*.
1778 *
1779 * The last argument, *flags*, holds the number of stack frames to
1780 * skip (from 0 to 255), masked with
1781 * **BPF_F_SKIP_FIELD_MASK**. The next bits can be used to set
1782 * the following flags:
1783 *
1784 * **BPF_F_USER_STACK**
1785 * Collect a user space stack instead of a kernel stack.
1786 * **BPF_F_USER_BUILD_ID**
1787 * Collect buildid+offset instead of ips for user stack,
1788 * only valid if **BPF_F_USER_STACK** is also specified.
1789 *
1790 * **bpf_get_stack**\ () can collect up to
1791 * **PERF_MAX_STACK_DEPTH** both kernel and user frames, subject
1792 * to sufficient large buffer size. Note that
1793 * this limit can be controlled with the **sysctl** program, and
1794 * that it should be manually increased in order to profile long
1795 * user stacks (such as stacks for Java programs). To do so, use:
1796 *
1797 * ::
1798 *
1799 * # sysctl kernel.perf_event_max_stack=<new value>
1800 *
1801 * Return
1802 * a non-negative value equal to or less than size on success, or
1803 * a negative error in case of failure.
1804 *
1805 * int skb_load_bytes_relative(const struct sk_buff *skb, u32 offset, void *to, u32 len, u32 start_header)
1806 * Description
1807 * This helper is similar to **bpf_skb_load_bytes**\ () in that
1808 * it provides an easy way to load *len* bytes from *offset*
1809 * from the packet associated to *skb*, into the buffer pointed
1810 * by *to*. The difference to **bpf_skb_load_bytes**\ () is that
1811 * a fifth argument *start_header* exists in order to select a
1812 * base offset to start from. *start_header* can be one of:
1813 *
1814 * **BPF_HDR_START_MAC**
1815 * Base offset to load data from is *skb*'s mac header.
1816 * **BPF_HDR_START_NET**
1817 * Base offset to load data from is *skb*'s network header.
1818 *
1819 * In general, "direct packet access" is the preferred method to
1820 * access packet data, however, this helper is in particular useful
1821 * in socket filters where *skb*\ **->data** does not always point
1822 * to the start of the mac header and where "direct packet access"
1823 * is not available.
1824 *
1825 * Return
1826 * 0 on success, or a negative error in case of failure.
1827 *
1770 */ 1828 */
1771#define __BPF_FUNC_MAPPER(FN) \ 1829#define __BPF_FUNC_MAPPER(FN) \
1772 FN(unspec), \ 1830 FN(unspec), \
@@ -1835,7 +1893,9 @@ union bpf_attr {
1835 FN(msg_pull_data), \ 1893 FN(msg_pull_data), \
1836 FN(bind), \ 1894 FN(bind), \
1837 FN(xdp_adjust_tail), \ 1895 FN(xdp_adjust_tail), \
1838 FN(skb_get_xfrm_state), 1896 FN(skb_get_xfrm_state), \
1897 FN(get_stack), \
1898 FN(skb_load_bytes_relative),
1839 1899
1840/* integer value in 'imm' field of BPF_CALL instruction selects which helper 1900/* integer value in 'imm' field of BPF_CALL instruction selects which helper
1841 * function eBPF program intends to call 1901 * function eBPF program intends to call
@@ -1869,11 +1929,14 @@ enum bpf_func_id {
1869/* BPF_FUNC_skb_set_tunnel_key and BPF_FUNC_skb_get_tunnel_key flags. */ 1929/* BPF_FUNC_skb_set_tunnel_key and BPF_FUNC_skb_get_tunnel_key flags. */
1870#define BPF_F_TUNINFO_IPV6 (1ULL << 0) 1930#define BPF_F_TUNINFO_IPV6 (1ULL << 0)
1871 1931
1872/* BPF_FUNC_get_stackid flags. */ 1932/* flags for both BPF_FUNC_get_stackid and BPF_FUNC_get_stack. */
1873#define BPF_F_SKIP_FIELD_MASK 0xffULL 1933#define BPF_F_SKIP_FIELD_MASK 0xffULL
1874#define BPF_F_USER_STACK (1ULL << 8) 1934#define BPF_F_USER_STACK (1ULL << 8)
1935/* flags used by BPF_FUNC_get_stackid only. */
1875#define BPF_F_FAST_STACK_CMP (1ULL << 9) 1936#define BPF_F_FAST_STACK_CMP (1ULL << 9)
1876#define BPF_F_REUSE_STACKID (1ULL << 10) 1937#define BPF_F_REUSE_STACKID (1ULL << 10)
1938/* flags used by BPF_FUNC_get_stack only. */
1939#define BPF_F_USER_BUILD_ID (1ULL << 11)
1877 1940
1878/* BPF_FUNC_skb_set_tunnel_key flags. */ 1941/* BPF_FUNC_skb_set_tunnel_key flags. */
1879#define BPF_F_ZERO_CSUM_TX (1ULL << 1) 1942#define BPF_F_ZERO_CSUM_TX (1ULL << 1)
@@ -1893,6 +1956,12 @@ enum bpf_adj_room_mode {
1893 BPF_ADJ_ROOM_NET, 1956 BPF_ADJ_ROOM_NET,
1894}; 1957};
1895 1958
1959/* Mode for BPF_FUNC_skb_load_bytes_relative helper. */
1960enum bpf_hdr_start_off {
1961 BPF_HDR_START_MAC,
1962 BPF_HDR_START_NET,
1963};
1964
1896/* user accessible mirror of in-kernel sk_buff. 1965/* user accessible mirror of in-kernel sk_buff.
1897 * new fields can only be added to the end of this structure 1966 * new fields can only be added to the end of this structure
1898 */ 1967 */
diff --git a/tools/include/uapi/linux/erspan.h b/tools/include/uapi/linux/erspan.h
new file mode 100644
index 000000000000..841573019ae1
--- /dev/null
+++ b/tools/include/uapi/linux/erspan.h
@@ -0,0 +1,52 @@
1/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
2/*
3 * ERSPAN Tunnel Metadata
4 *
5 * Copyright (c) 2018 VMware
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2
9 * as published by the Free Software Foundation.
10 *
11 * Userspace API for metadata mode ERSPAN tunnel
12 */
13#ifndef _UAPI_ERSPAN_H
14#define _UAPI_ERSPAN_H
15
16#include <linux/types.h> /* For __beXX in userspace */
17#include <asm/byteorder.h>
18
19/* ERSPAN version 2 metadata header */
20struct erspan_md2 {
21 __be32 timestamp;
22 __be16 sgt; /* security group tag */
23#if defined(__LITTLE_ENDIAN_BITFIELD)
24 __u8 hwid_upper:2,
25 ft:5,
26 p:1;
27 __u8 o:1,
28 gra:2,
29 dir:1,
30 hwid:4;
31#elif defined(__BIG_ENDIAN_BITFIELD)
32 __u8 p:1,
33 ft:5,
34 hwid_upper:2;
35 __u8 hwid:4,
36 dir:1,
37 gra:2,
38 o:1;
39#else
40#error "Please fix <asm/byteorder.h>"
41#endif
42};
43
44struct erspan_metadata {
45 int version;
46 union {
47 __be32 index; /* Version 1 (type II)*/
48 struct erspan_md2 md2; /* Version 2 (type III) */
49 } u;
50};
51
52#endif /* _UAPI_ERSPAN_H */
diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile
index b64a7a39cbc8..9d762184b805 100644
--- a/tools/testing/selftests/bpf/Makefile
+++ b/tools/testing/selftests/bpf/Makefile
@@ -32,7 +32,8 @@ TEST_GEN_FILES = test_pkt_access.o test_xdp.o test_l4lb.o test_tcp_estats.o test
32 test_l4lb_noinline.o test_xdp_noinline.o test_stacktrace_map.o \ 32 test_l4lb_noinline.o test_xdp_noinline.o test_stacktrace_map.o \
33 sample_map_ret0.o test_tcpbpf_kern.o test_stacktrace_build_id.o \ 33 sample_map_ret0.o test_tcpbpf_kern.o test_stacktrace_build_id.o \
34 sockmap_tcp_msg_prog.o connect4_prog.o connect6_prog.o test_adjust_tail.o \ 34 sockmap_tcp_msg_prog.o connect4_prog.o connect6_prog.o test_adjust_tail.o \
35 test_btf_haskv.o test_btf_nokv.o test_sockmap_kern.o test_tunnel_kern.o 35 test_btf_haskv.o test_btf_nokv.o test_sockmap_kern.o test_tunnel_kern.o \
36 test_get_stack_rawtp.o
36 37
37# Order correspond to 'make run_tests' order 38# Order correspond to 'make run_tests' order
38TEST_PROGS := test_kmod.sh \ 39TEST_PROGS := test_kmod.sh \
@@ -58,6 +59,7 @@ $(OUTPUT)/test_dev_cgroup: cgroup_helpers.c
58$(OUTPUT)/test_sock: cgroup_helpers.c 59$(OUTPUT)/test_sock: cgroup_helpers.c
59$(OUTPUT)/test_sock_addr: cgroup_helpers.c 60$(OUTPUT)/test_sock_addr: cgroup_helpers.c
60$(OUTPUT)/test_sockmap: cgroup_helpers.c 61$(OUTPUT)/test_sockmap: cgroup_helpers.c
62$(OUTPUT)/test_progs: trace_helpers.c
61 63
62.PHONY: force 64.PHONY: force
63 65
diff --git a/tools/testing/selftests/bpf/bpf_helpers.h b/tools/testing/selftests/bpf/bpf_helpers.h
index 69d7b918e66a..265f8e0e8ada 100644
--- a/tools/testing/selftests/bpf/bpf_helpers.h
+++ b/tools/testing/selftests/bpf/bpf_helpers.h
@@ -101,6 +101,8 @@ static int (*bpf_xdp_adjust_tail)(void *ctx, int offset) =
101static int (*bpf_skb_get_xfrm_state)(void *ctx, int index, void *state, 101static int (*bpf_skb_get_xfrm_state)(void *ctx, int index, void *state,
102 int size, int flags) = 102 int size, int flags) =
103 (void *) BPF_FUNC_skb_get_xfrm_state; 103 (void *) BPF_FUNC_skb_get_xfrm_state;
104static int (*bpf_get_stack)(void *ctx, void *buf, int size, int flags) =
105 (void *) BPF_FUNC_get_stack;
104 106
105/* llvm builtin functions that eBPF C program may use to 107/* llvm builtin functions that eBPF C program may use to
106 * emit BPF_LD_ABS and BPF_LD_IND instructions 108 * emit BPF_LD_ABS and BPF_LD_IND instructions
diff --git a/tools/testing/selftests/bpf/test_get_stack_rawtp.c b/tools/testing/selftests/bpf/test_get_stack_rawtp.c
new file mode 100644
index 000000000000..f6d9f238e00a
--- /dev/null
+++ b/tools/testing/selftests/bpf/test_get_stack_rawtp.c
@@ -0,0 +1,102 @@
1// SPDX-License-Identifier: GPL-2.0
2
3#include <linux/bpf.h>
4#include "bpf_helpers.h"
5
6/* Permit pretty deep stack traces */
7#define MAX_STACK_RAWTP 100
8struct stack_trace_t {
9 int pid;
10 int kern_stack_size;
11 int user_stack_size;
12 int user_stack_buildid_size;
13 __u64 kern_stack[MAX_STACK_RAWTP];
14 __u64 user_stack[MAX_STACK_RAWTP];
15 struct bpf_stack_build_id user_stack_buildid[MAX_STACK_RAWTP];
16};
17
18struct bpf_map_def SEC("maps") perfmap = {
19 .type = BPF_MAP_TYPE_PERF_EVENT_ARRAY,
20 .key_size = sizeof(int),
21 .value_size = sizeof(__u32),
22 .max_entries = 2,
23};
24
25struct bpf_map_def SEC("maps") stackdata_map = {
26 .type = BPF_MAP_TYPE_PERCPU_ARRAY,
27 .key_size = sizeof(__u32),
28 .value_size = sizeof(struct stack_trace_t),
29 .max_entries = 1,
30};
31
32/* Allocate per-cpu space twice the needed. For the code below
33 * usize = bpf_get_stack(ctx, raw_data, max_len, BPF_F_USER_STACK);
34 * if (usize < 0)
35 * return 0;
36 * ksize = bpf_get_stack(ctx, raw_data + usize, max_len - usize, 0);
37 *
38 * If we have value_size = MAX_STACK_RAWTP * sizeof(__u64),
39 * verifier will complain that access "raw_data + usize"
40 * with size "max_len - usize" may be out of bound.
41 * The maximum "raw_data + usize" is "raw_data + max_len"
42 * and the maximum "max_len - usize" is "max_len", verifier
43 * concludes that the maximum buffer access range is
44 * "raw_data[0...max_len * 2 - 1]" and hence reject the program.
45 *
46 * Doubling the to-be-used max buffer size can fix this verifier
47 * issue and avoid complicated C programming massaging.
48 * This is an acceptable workaround since there is one entry here.
49 */
50struct bpf_map_def SEC("maps") rawdata_map = {
51 .type = BPF_MAP_TYPE_PERCPU_ARRAY,
52 .key_size = sizeof(__u32),
53 .value_size = MAX_STACK_RAWTP * sizeof(__u64) * 2,
54 .max_entries = 1,
55};
56
57SEC("tracepoint/raw_syscalls/sys_enter")
58int bpf_prog1(void *ctx)
59{
60 int max_len, max_buildid_len, usize, ksize, total_size;
61 struct stack_trace_t *data;
62 void *raw_data;
63 __u32 key = 0;
64
65 data = bpf_map_lookup_elem(&stackdata_map, &key);
66 if (!data)
67 return 0;
68
69 max_len = MAX_STACK_RAWTP * sizeof(__u64);
70 max_buildid_len = MAX_STACK_RAWTP * sizeof(struct bpf_stack_build_id);
71 data->pid = bpf_get_current_pid_tgid();
72 data->kern_stack_size = bpf_get_stack(ctx, data->kern_stack,
73 max_len, 0);
74 data->user_stack_size = bpf_get_stack(ctx, data->user_stack, max_len,
75 BPF_F_USER_STACK);
76 data->user_stack_buildid_size = bpf_get_stack(
77 ctx, data->user_stack_buildid, max_buildid_len,
78 BPF_F_USER_STACK | BPF_F_USER_BUILD_ID);
79 bpf_perf_event_output(ctx, &perfmap, 0, data, sizeof(*data));
80
81 /* write both kernel and user stacks to the same buffer */
82 raw_data = bpf_map_lookup_elem(&rawdata_map, &key);
83 if (!raw_data)
84 return 0;
85
86 usize = bpf_get_stack(ctx, raw_data, max_len, BPF_F_USER_STACK);
87 if (usize < 0)
88 return 0;
89
90 ksize = bpf_get_stack(ctx, raw_data + usize, max_len - usize, 0);
91 if (ksize < 0)
92 return 0;
93
94 total_size = usize + ksize;
95 if (total_size > 0 && total_size <= max_len)
96 bpf_perf_event_output(ctx, &perfmap, 0, raw_data, total_size);
97
98 return 0;
99}
100
101char _license[] SEC("license") = "GPL";
102__u32 _version SEC("version") = 1; /* ignored by tracepoints, required by libbpf.a */
diff --git a/tools/testing/selftests/bpf/test_progs.c b/tools/testing/selftests/bpf/test_progs.c
index fac581f1c57f..ed197eef1cfc 100644
--- a/tools/testing/selftests/bpf/test_progs.c
+++ b/tools/testing/selftests/bpf/test_progs.c
@@ -38,8 +38,10 @@ typedef __u16 __sum16;
38#include "bpf_util.h" 38#include "bpf_util.h"
39#include "bpf_endian.h" 39#include "bpf_endian.h"
40#include "bpf_rlimit.h" 40#include "bpf_rlimit.h"
41#include "trace_helpers.h"
41 42
42static int error_cnt, pass_cnt; 43static int error_cnt, pass_cnt;
44static bool jit_enabled;
43 45
44#define MAGIC_BYTES 123 46#define MAGIC_BYTES 123
45 47
@@ -391,13 +393,30 @@ static inline __u64 ptr_to_u64(const void *ptr)
391 return (__u64) (unsigned long) ptr; 393 return (__u64) (unsigned long) ptr;
392} 394}
393 395
396static bool is_jit_enabled(void)
397{
398 const char *jit_sysctl = "/proc/sys/net/core/bpf_jit_enable";
399 bool enabled = false;
400 int sysctl_fd;
401
402 sysctl_fd = open(jit_sysctl, 0, O_RDONLY);
403 if (sysctl_fd != -1) {
404 char tmpc;
405
406 if (read(sysctl_fd, &tmpc, sizeof(tmpc)) == 1)
407 enabled = (tmpc != '0');
408 close(sysctl_fd);
409 }
410
411 return enabled;
412}
413
394static void test_bpf_obj_id(void) 414static void test_bpf_obj_id(void)
395{ 415{
396 const __u64 array_magic_value = 0xfaceb00c; 416 const __u64 array_magic_value = 0xfaceb00c;
397 const __u32 array_key = 0; 417 const __u32 array_key = 0;
398 const int nr_iters = 2; 418 const int nr_iters = 2;
399 const char *file = "./test_obj_id.o"; 419 const char *file = "./test_obj_id.o";
400 const char *jit_sysctl = "/proc/sys/net/core/bpf_jit_enable";
401 const char *expected_prog_name = "test_obj_id"; 420 const char *expected_prog_name = "test_obj_id";
402 const char *expected_map_name = "test_map_id"; 421 const char *expected_map_name = "test_map_id";
403 const __u64 nsec_per_sec = 1000000000; 422 const __u64 nsec_per_sec = 1000000000;
@@ -414,20 +433,11 @@ static void test_bpf_obj_id(void)
414 char jited_insns[128], xlated_insns[128], zeros[128]; 433 char jited_insns[128], xlated_insns[128], zeros[128];
415 __u32 i, next_id, info_len, nr_id_found, duration = 0; 434 __u32 i, next_id, info_len, nr_id_found, duration = 0;
416 struct timespec real_time_ts, boot_time_ts; 435 struct timespec real_time_ts, boot_time_ts;
417 int sysctl_fd, jit_enabled = 0, err = 0; 436 int err = 0;
418 __u64 array_value; 437 __u64 array_value;
419 uid_t my_uid = getuid(); 438 uid_t my_uid = getuid();
420 time_t now, load_time; 439 time_t now, load_time;
421 440
422 sysctl_fd = open(jit_sysctl, 0, O_RDONLY);
423 if (sysctl_fd != -1) {
424 char tmpc;
425
426 if (read(sysctl_fd, &tmpc, sizeof(tmpc)) == 1)
427 jit_enabled = (tmpc != '0');
428 close(sysctl_fd);
429 }
430
431 err = bpf_prog_get_fd_by_id(0); 441 err = bpf_prog_get_fd_by_id(0);
432 CHECK(err >= 0 || errno != ENOENT, 442 CHECK(err >= 0 || errno != ENOENT,
433 "get-fd-by-notexist-prog-id", "err %d errno %d\n", err, errno); 443 "get-fd-by-notexist-prog-id", "err %d errno %d\n", err, errno);
@@ -896,11 +906,47 @@ static int compare_map_keys(int map1_fd, int map2_fd)
896 return 0; 906 return 0;
897} 907}
898 908
909static int compare_stack_ips(int smap_fd, int amap_fd, int stack_trace_len)
910{
911 __u32 key, next_key, *cur_key_p, *next_key_p;
912 char *val_buf1, *val_buf2;
913 int i, err = 0;
914
915 val_buf1 = malloc(stack_trace_len);
916 val_buf2 = malloc(stack_trace_len);
917 cur_key_p = NULL;
918 next_key_p = &key;
919 while (bpf_map_get_next_key(smap_fd, cur_key_p, next_key_p) == 0) {
920 err = bpf_map_lookup_elem(smap_fd, next_key_p, val_buf1);
921 if (err)
922 goto out;
923 err = bpf_map_lookup_elem(amap_fd, next_key_p, val_buf2);
924 if (err)
925 goto out;
926 for (i = 0; i < stack_trace_len; i++) {
927 if (val_buf1[i] != val_buf2[i]) {
928 err = -1;
929 goto out;
930 }
931 }
932 key = *next_key_p;
933 cur_key_p = &key;
934 next_key_p = &next_key;
935 }
936 if (errno != ENOENT)
937 err = -1;
938
939out:
940 free(val_buf1);
941 free(val_buf2);
942 return err;
943}
944
899static void test_stacktrace_map() 945static void test_stacktrace_map()
900{ 946{
901 int control_map_fd, stackid_hmap_fd, stackmap_fd; 947 int control_map_fd, stackid_hmap_fd, stackmap_fd, stack_amap_fd;
902 const char *file = "./test_stacktrace_map.o"; 948 const char *file = "./test_stacktrace_map.o";
903 int bytes, efd, err, pmu_fd, prog_fd; 949 int bytes, efd, err, pmu_fd, prog_fd, stack_trace_len;
904 struct perf_event_attr attr = {}; 950 struct perf_event_attr attr = {};
905 __u32 key, val, duration = 0; 951 __u32 key, val, duration = 0;
906 struct bpf_object *obj; 952 struct bpf_object *obj;
@@ -956,6 +1002,10 @@ static void test_stacktrace_map()
956 if (stackmap_fd < 0) 1002 if (stackmap_fd < 0)
957 goto disable_pmu; 1003 goto disable_pmu;
958 1004
1005 stack_amap_fd = bpf_find_map(__func__, obj, "stack_amap");
1006 if (stack_amap_fd < 0)
1007 goto disable_pmu;
1008
959 /* give some time for bpf program run */ 1009 /* give some time for bpf program run */
960 sleep(1); 1010 sleep(1);
961 1011
@@ -977,6 +1027,12 @@ static void test_stacktrace_map()
977 "err %d errno %d\n", err, errno)) 1027 "err %d errno %d\n", err, errno))
978 goto disable_pmu_noerr; 1028 goto disable_pmu_noerr;
979 1029
1030 stack_trace_len = PERF_MAX_STACK_DEPTH * sizeof(__u64);
1031 err = compare_stack_ips(stackmap_fd, stack_amap_fd, stack_trace_len);
1032 if (CHECK(err, "compare_stack_ips stackmap vs. stack_amap",
1033 "err %d errno %d\n", err, errno))
1034 goto disable_pmu_noerr;
1035
980 goto disable_pmu_noerr; 1036 goto disable_pmu_noerr;
981disable_pmu: 1037disable_pmu:
982 error_cnt++; 1038 error_cnt++;
@@ -1070,9 +1126,9 @@ err:
1070 1126
1071static void test_stacktrace_build_id(void) 1127static void test_stacktrace_build_id(void)
1072{ 1128{
1073 int control_map_fd, stackid_hmap_fd, stackmap_fd; 1129 int control_map_fd, stackid_hmap_fd, stackmap_fd, stack_amap_fd;
1074 const char *file = "./test_stacktrace_build_id.o"; 1130 const char *file = "./test_stacktrace_build_id.o";
1075 int bytes, efd, err, pmu_fd, prog_fd; 1131 int bytes, efd, err, pmu_fd, prog_fd, stack_trace_len;
1076 struct perf_event_attr attr = {}; 1132 struct perf_event_attr attr = {};
1077 __u32 key, previous_key, val, duration = 0; 1133 __u32 key, previous_key, val, duration = 0;
1078 struct bpf_object *obj; 1134 struct bpf_object *obj;
@@ -1137,6 +1193,11 @@ static void test_stacktrace_build_id(void)
1137 err, errno)) 1193 err, errno))
1138 goto disable_pmu; 1194 goto disable_pmu;
1139 1195
1196 stack_amap_fd = bpf_find_map(__func__, obj, "stack_amap");
1197 if (CHECK(stack_amap_fd < 0, "bpf_find_map stack_amap",
1198 "err %d errno %d\n", err, errno))
1199 goto disable_pmu;
1200
1140 assert(system("dd if=/dev/urandom of=/dev/zero count=4 2> /dev/null") 1201 assert(system("dd if=/dev/urandom of=/dev/zero count=4 2> /dev/null")
1141 == 0); 1202 == 0);
1142 assert(system("./urandom_read") == 0); 1203 assert(system("./urandom_read") == 0);
@@ -1188,8 +1249,15 @@ static void test_stacktrace_build_id(void)
1188 previous_key = key; 1249 previous_key = key;
1189 } while (bpf_map_get_next_key(stackmap_fd, &previous_key, &key) == 0); 1250 } while (bpf_map_get_next_key(stackmap_fd, &previous_key, &key) == 0);
1190 1251
1191 CHECK(build_id_matches < 1, "build id match", 1252 if (CHECK(build_id_matches < 1, "build id match",
1192 "Didn't find expected build ID from the map\n"); 1253 "Didn't find expected build ID from the map\n"))
1254 goto disable_pmu;
1255
1256 stack_trace_len = PERF_MAX_STACK_DEPTH
1257 * sizeof(struct bpf_stack_build_id);
1258 err = compare_stack_ips(stackmap_fd, stack_amap_fd, stack_trace_len);
1259 CHECK(err, "compare_stack_ips stackmap vs. stack_amap",
1260 "err %d errno %d\n", err, errno);
1193 1261
1194disable_pmu: 1262disable_pmu:
1195 ioctl(pmu_fd, PERF_EVENT_IOC_DISABLE); 1263 ioctl(pmu_fd, PERF_EVENT_IOC_DISABLE);
@@ -1204,8 +1272,147 @@ out:
1204 return; 1272 return;
1205} 1273}
1206 1274
1275#define MAX_CNT_RAWTP 10ull
1276#define MAX_STACK_RAWTP 100
1277struct get_stack_trace_t {
1278 int pid;
1279 int kern_stack_size;
1280 int user_stack_size;
1281 int user_stack_buildid_size;
1282 __u64 kern_stack[MAX_STACK_RAWTP];
1283 __u64 user_stack[MAX_STACK_RAWTP];
1284 struct bpf_stack_build_id user_stack_buildid[MAX_STACK_RAWTP];
1285};
1286
1287static int get_stack_print_output(void *data, int size)
1288{
1289 bool good_kern_stack = false, good_user_stack = false;
1290 const char *nonjit_func = "___bpf_prog_run";
1291 struct get_stack_trace_t *e = data;
1292 int i, num_stack;
1293 static __u64 cnt;
1294 struct ksym *ks;
1295
1296 cnt++;
1297
1298 if (size < sizeof(struct get_stack_trace_t)) {
1299 __u64 *raw_data = data;
1300 bool found = false;
1301
1302 num_stack = size / sizeof(__u64);
1303 /* If jit is enabled, we do not have a good way to
1304 * verify the sanity of the kernel stack. So we
1305 * just assume it is good if the stack is not empty.
1306 * This could be improved in the future.
1307 */
1308 if (jit_enabled) {
1309 found = num_stack > 0;
1310 } else {
1311 for (i = 0; i < num_stack; i++) {
1312 ks = ksym_search(raw_data[i]);
1313 if (strcmp(ks->name, nonjit_func) == 0) {
1314 found = true;
1315 break;
1316 }
1317 }
1318 }
1319 if (found) {
1320 good_kern_stack = true;
1321 good_user_stack = true;
1322 }
1323 } else {
1324 num_stack = e->kern_stack_size / sizeof(__u64);
1325 if (jit_enabled) {
1326 good_kern_stack = num_stack > 0;
1327 } else {
1328 for (i = 0; i < num_stack; i++) {
1329 ks = ksym_search(e->kern_stack[i]);
1330 if (strcmp(ks->name, nonjit_func) == 0) {
1331 good_kern_stack = true;
1332 break;
1333 }
1334 }
1335 }
1336 if (e->user_stack_size > 0 && e->user_stack_buildid_size > 0)
1337 good_user_stack = true;
1338 }
1339 if (!good_kern_stack || !good_user_stack)
1340 return PERF_EVENT_ERROR;
1341
1342 if (cnt == MAX_CNT_RAWTP)
1343 return PERF_EVENT_DONE;
1344
1345 return PERF_EVENT_CONT;
1346}
1347
1348static void test_get_stack_raw_tp(void)
1349{
1350 const char *file = "./test_get_stack_rawtp.o";
1351 int i, efd, err, prog_fd, pmu_fd, perfmap_fd;
1352 struct perf_event_attr attr = {};
1353 struct timespec tv = {0, 10};
1354 __u32 key = 0, duration = 0;
1355 struct bpf_object *obj;
1356
1357 err = bpf_prog_load(file, BPF_PROG_TYPE_RAW_TRACEPOINT, &obj, &prog_fd);
1358 if (CHECK(err, "prog_load raw tp", "err %d errno %d\n", err, errno))
1359 return;
1360
1361 efd = bpf_raw_tracepoint_open("sys_enter", prog_fd);
1362 if (CHECK(efd < 0, "raw_tp_open", "err %d errno %d\n", efd, errno))
1363 goto close_prog;
1364
1365 perfmap_fd = bpf_find_map(__func__, obj, "perfmap");
1366 if (CHECK(perfmap_fd < 0, "bpf_find_map", "err %d errno %d\n",
1367 perfmap_fd, errno))
1368 goto close_prog;
1369
1370 err = load_kallsyms();
1371 if (CHECK(err < 0, "load_kallsyms", "err %d errno %d\n", err, errno))
1372 goto close_prog;
1373
1374 attr.sample_type = PERF_SAMPLE_RAW;
1375 attr.type = PERF_TYPE_SOFTWARE;
1376 attr.config = PERF_COUNT_SW_BPF_OUTPUT;
1377 pmu_fd = syscall(__NR_perf_event_open, &attr, getpid()/*pid*/, -1/*cpu*/,
1378 -1/*group_fd*/, 0);
1379 if (CHECK(pmu_fd < 0, "perf_event_open", "err %d errno %d\n", pmu_fd,
1380 errno))
1381 goto close_prog;
1382
1383 err = bpf_map_update_elem(perfmap_fd, &key, &pmu_fd, BPF_ANY);
1384 if (CHECK(err < 0, "bpf_map_update_elem", "err %d errno %d\n", err,
1385 errno))
1386 goto close_prog;
1387
1388 err = ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0);
1389 if (CHECK(err < 0, "ioctl PERF_EVENT_IOC_ENABLE", "err %d errno %d\n",
1390 err, errno))
1391 goto close_prog;
1392
1393 err = perf_event_mmap(pmu_fd);
1394 if (CHECK(err < 0, "perf_event_mmap", "err %d errno %d\n", err, errno))
1395 goto close_prog;
1396
1397 /* trigger some syscall action */
1398 for (i = 0; i < MAX_CNT_RAWTP; i++)
1399 nanosleep(&tv, NULL);
1400
1401 err = perf_event_poller(pmu_fd, get_stack_print_output);
1402 if (CHECK(err < 0, "perf_event_poller", "err %d errno %d\n", err, errno))
1403 goto close_prog;
1404
1405 goto close_prog_noerr;
1406close_prog:
1407 error_cnt++;
1408close_prog_noerr:
1409 bpf_object__close(obj);
1410}
1411
1207int main(void) 1412int main(void)
1208{ 1413{
1414 jit_enabled = is_jit_enabled();
1415
1209 test_pkt_access(); 1416 test_pkt_access();
1210 test_xdp(); 1417 test_xdp();
1211 test_xdp_adjust_tail(); 1418 test_xdp_adjust_tail();
@@ -1219,6 +1426,7 @@ int main(void)
1219 test_stacktrace_map(); 1426 test_stacktrace_map();
1220 test_stacktrace_build_id(); 1427 test_stacktrace_build_id();
1221 test_stacktrace_map_raw_tp(); 1428 test_stacktrace_map_raw_tp();
1429 test_get_stack_raw_tp();
1222 1430
1223 printf("Summary: %d PASSED, %d FAILED\n", pass_cnt, error_cnt); 1431 printf("Summary: %d PASSED, %d FAILED\n", pass_cnt, error_cnt);
1224 return error_cnt ? EXIT_FAILURE : EXIT_SUCCESS; 1432 return error_cnt ? EXIT_FAILURE : EXIT_SUCCESS;
diff --git a/tools/testing/selftests/bpf/test_stacktrace_build_id.c b/tools/testing/selftests/bpf/test_stacktrace_build_id.c
index b755bd783ce5..d86c281e957f 100644
--- a/tools/testing/selftests/bpf/test_stacktrace_build_id.c
+++ b/tools/testing/selftests/bpf/test_stacktrace_build_id.c
@@ -19,7 +19,7 @@ struct bpf_map_def SEC("maps") stackid_hmap = {
19 .type = BPF_MAP_TYPE_HASH, 19 .type = BPF_MAP_TYPE_HASH,
20 .key_size = sizeof(__u32), 20 .key_size = sizeof(__u32),
21 .value_size = sizeof(__u32), 21 .value_size = sizeof(__u32),
22 .max_entries = 10000, 22 .max_entries = 16384,
23}; 23};
24 24
25struct bpf_map_def SEC("maps") stackmap = { 25struct bpf_map_def SEC("maps") stackmap = {
@@ -31,6 +31,14 @@ struct bpf_map_def SEC("maps") stackmap = {
31 .map_flags = BPF_F_STACK_BUILD_ID, 31 .map_flags = BPF_F_STACK_BUILD_ID,
32}; 32};
33 33
34struct bpf_map_def SEC("maps") stack_amap = {
35 .type = BPF_MAP_TYPE_ARRAY,
36 .key_size = sizeof(__u32),
37 .value_size = sizeof(struct bpf_stack_build_id)
38 * PERF_MAX_STACK_DEPTH,
39 .max_entries = 128,
40};
41
34/* taken from /sys/kernel/debug/tracing/events/random/urandom_read/format */ 42/* taken from /sys/kernel/debug/tracing/events/random/urandom_read/format */
35struct random_urandom_args { 43struct random_urandom_args {
36 unsigned long long pad; 44 unsigned long long pad;
@@ -42,7 +50,10 @@ struct random_urandom_args {
42SEC("tracepoint/random/urandom_read") 50SEC("tracepoint/random/urandom_read")
43int oncpu(struct random_urandom_args *args) 51int oncpu(struct random_urandom_args *args)
44{ 52{
53 __u32 max_len = sizeof(struct bpf_stack_build_id)
54 * PERF_MAX_STACK_DEPTH;
45 __u32 key = 0, val = 0, *value_p; 55 __u32 key = 0, val = 0, *value_p;
56 void *stack_p;
46 57
47 value_p = bpf_map_lookup_elem(&control_map, &key); 58 value_p = bpf_map_lookup_elem(&control_map, &key);
48 if (value_p && *value_p) 59 if (value_p && *value_p)
@@ -50,8 +61,13 @@ int oncpu(struct random_urandom_args *args)
50 61
51 /* The size of stackmap and stackid_hmap should be the same */ 62 /* The size of stackmap and stackid_hmap should be the same */
52 key = bpf_get_stackid(args, &stackmap, BPF_F_USER_STACK); 63 key = bpf_get_stackid(args, &stackmap, BPF_F_USER_STACK);
53 if ((int)key >= 0) 64 if ((int)key >= 0) {
54 bpf_map_update_elem(&stackid_hmap, &key, &val, 0); 65 bpf_map_update_elem(&stackid_hmap, &key, &val, 0);
66 stack_p = bpf_map_lookup_elem(&stack_amap, &key);
67 if (stack_p)
68 bpf_get_stack(args, stack_p, max_len,
69 BPF_F_USER_STACK | BPF_F_USER_BUILD_ID);
70 }
55 71
56 return 0; 72 return 0;
57} 73}
diff --git a/tools/testing/selftests/bpf/test_stacktrace_map.c b/tools/testing/selftests/bpf/test_stacktrace_map.c
index 76d85c5d08bd..af111af7ca1a 100644
--- a/tools/testing/selftests/bpf/test_stacktrace_map.c
+++ b/tools/testing/selftests/bpf/test_stacktrace_map.c
@@ -19,14 +19,21 @@ struct bpf_map_def SEC("maps") stackid_hmap = {
19 .type = BPF_MAP_TYPE_HASH, 19 .type = BPF_MAP_TYPE_HASH,
20 .key_size = sizeof(__u32), 20 .key_size = sizeof(__u32),
21 .value_size = sizeof(__u32), 21 .value_size = sizeof(__u32),
22 .max_entries = 10000, 22 .max_entries = 16384,
23}; 23};
24 24
25struct bpf_map_def SEC("maps") stackmap = { 25struct bpf_map_def SEC("maps") stackmap = {
26 .type = BPF_MAP_TYPE_STACK_TRACE, 26 .type = BPF_MAP_TYPE_STACK_TRACE,
27 .key_size = sizeof(__u32), 27 .key_size = sizeof(__u32),
28 .value_size = sizeof(__u64) * PERF_MAX_STACK_DEPTH, 28 .value_size = sizeof(__u64) * PERF_MAX_STACK_DEPTH,
29 .max_entries = 10000, 29 .max_entries = 16384,
30};
31
32struct bpf_map_def SEC("maps") stack_amap = {
33 .type = BPF_MAP_TYPE_ARRAY,
34 .key_size = sizeof(__u32),
35 .value_size = sizeof(__u64) * PERF_MAX_STACK_DEPTH,
36 .max_entries = 16384,
30}; 37};
31 38
32/* taken from /sys/kernel/debug/tracing/events/sched/sched_switch/format */ 39/* taken from /sys/kernel/debug/tracing/events/sched/sched_switch/format */
@@ -44,7 +51,9 @@ struct sched_switch_args {
44SEC("tracepoint/sched/sched_switch") 51SEC("tracepoint/sched/sched_switch")
45int oncpu(struct sched_switch_args *ctx) 52int oncpu(struct sched_switch_args *ctx)
46{ 53{
54 __u32 max_len = PERF_MAX_STACK_DEPTH * sizeof(__u64);
47 __u32 key = 0, val = 0, *value_p; 55 __u32 key = 0, val = 0, *value_p;
56 void *stack_p;
48 57
49 value_p = bpf_map_lookup_elem(&control_map, &key); 58 value_p = bpf_map_lookup_elem(&control_map, &key);
50 if (value_p && *value_p) 59 if (value_p && *value_p)
@@ -52,8 +61,12 @@ int oncpu(struct sched_switch_args *ctx)
52 61
53 /* The size of stackmap and stackid_hmap should be the same */ 62 /* The size of stackmap and stackid_hmap should be the same */
54 key = bpf_get_stackid(ctx, &stackmap, 0); 63 key = bpf_get_stackid(ctx, &stackmap, 0);
55 if ((int)key >= 0) 64 if ((int)key >= 0) {
56 bpf_map_update_elem(&stackid_hmap, &key, &val, 0); 65 bpf_map_update_elem(&stackid_hmap, &key, &val, 0);
66 stack_p = bpf_map_lookup_elem(&stack_amap, &key);
67 if (stack_p)
68 bpf_get_stack(ctx, stack_p, max_len, 0);
69 }
57 70
58 return 0; 71 return 0;
59} 72}
diff --git a/tools/testing/selftests/bpf/test_verifier.c b/tools/testing/selftests/bpf/test_verifier.c
index 165e9ddfa446..275b4570b5b8 100644
--- a/tools/testing/selftests/bpf/test_verifier.c
+++ b/tools/testing/selftests/bpf/test_verifier.c
@@ -47,7 +47,7 @@
47# define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) 47# define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
48#endif 48#endif
49 49
50#define MAX_INSNS 512 50#define MAX_INSNS BPF_MAXINSNS
51#define MAX_FIXUPS 8 51#define MAX_FIXUPS 8
52#define MAX_NR_MAPS 4 52#define MAX_NR_MAPS 4
53#define POINTER_VALUE 0xcafe4all 53#define POINTER_VALUE 0xcafe4all
@@ -77,6 +77,8 @@ struct bpf_test {
77 } result, result_unpriv; 77 } result, result_unpriv;
78 enum bpf_prog_type prog_type; 78 enum bpf_prog_type prog_type;
79 uint8_t flags; 79 uint8_t flags;
80 __u8 data[TEST_DATA_LEN];
81 void (*fill_helper)(struct bpf_test *self);
80}; 82};
81 83
82/* Note we want this to be 64 bit aligned so that the end of our array is 84/* Note we want this to be 64 bit aligned so that the end of our array is
@@ -94,6 +96,62 @@ struct other_val {
94 long long bar; 96 long long bar;
95}; 97};
96 98
99static void bpf_fill_ld_abs_vlan_push_pop(struct bpf_test *self)
100{
101 /* test: {skb->data[0], vlan_push} x 68 + {skb->data[0], vlan_pop} x 68 */
102#define PUSH_CNT 51
103 unsigned int len = BPF_MAXINSNS;
104 struct bpf_insn *insn = self->insns;
105 int i = 0, j, k = 0;
106
107 insn[i++] = BPF_MOV64_REG(BPF_REG_6, BPF_REG_1);
108loop:
109 for (j = 0; j < PUSH_CNT; j++) {
110 insn[i++] = BPF_LD_ABS(BPF_B, 0);
111 insn[i] = BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0x34, len - i - 2);
112 i++;
113 insn[i++] = BPF_MOV64_REG(BPF_REG_1, BPF_REG_6);
114 insn[i++] = BPF_MOV64_IMM(BPF_REG_2, 1);
115 insn[i++] = BPF_MOV64_IMM(BPF_REG_3, 2);
116 insn[i++] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
117 BPF_FUNC_skb_vlan_push),
118 insn[i] = BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, len - i - 2);
119 i++;
120 }
121
122 for (j = 0; j < PUSH_CNT; j++) {
123 insn[i++] = BPF_LD_ABS(BPF_B, 0);
124 insn[i] = BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0x34, len - i - 2);
125 i++;
126 insn[i++] = BPF_MOV64_REG(BPF_REG_1, BPF_REG_6);
127 insn[i++] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
128 BPF_FUNC_skb_vlan_pop),
129 insn[i] = BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, len - i - 2);
130 i++;
131 }
132 if (++k < 5)
133 goto loop;
134
135 for (; i < len - 1; i++)
136 insn[i] = BPF_ALU32_IMM(BPF_MOV, BPF_REG_0, 0xbef);
137 insn[len - 1] = BPF_EXIT_INSN();
138}
139
140static void bpf_fill_jump_around_ld_abs(struct bpf_test *self)
141{
142 struct bpf_insn *insn = self->insns;
143 unsigned int len = BPF_MAXINSNS;
144 int i = 0;
145
146 insn[i++] = BPF_MOV64_REG(BPF_REG_6, BPF_REG_1);
147 insn[i++] = BPF_LD_ABS(BPF_B, 0);
148 insn[i] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 10, len - i - 2);
149 i++;
150 while (i < len - 1)
151 insn[i++] = BPF_LD_ABS(BPF_B, 1);
152 insn[i] = BPF_EXIT_INSN();
153}
154
97static struct bpf_test tests[] = { 155static struct bpf_test tests[] = {
98 { 156 {
99 "add+sub+mul", 157 "add+sub+mul",
@@ -11680,6 +11738,242 @@ static struct bpf_test tests[] = {
11680 .errstr = "BPF_XADD stores into R2 packet", 11738 .errstr = "BPF_XADD stores into R2 packet",
11681 .prog_type = BPF_PROG_TYPE_XDP, 11739 .prog_type = BPF_PROG_TYPE_XDP,
11682 }, 11740 },
11741 {
11742 "bpf_get_stack return R0 within range",
11743 .insns = {
11744 BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
11745 BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0),
11746 BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
11747 BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
11748 BPF_LD_MAP_FD(BPF_REG_1, 0),
11749 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
11750 BPF_FUNC_map_lookup_elem),
11751 BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 28),
11752 BPF_MOV64_REG(BPF_REG_7, BPF_REG_0),
11753 BPF_MOV64_IMM(BPF_REG_9, sizeof(struct test_val)),
11754 BPF_MOV64_REG(BPF_REG_1, BPF_REG_6),
11755 BPF_MOV64_REG(BPF_REG_2, BPF_REG_7),
11756 BPF_MOV64_IMM(BPF_REG_3, sizeof(struct test_val)),
11757 BPF_MOV64_IMM(BPF_REG_4, 256),
11758 BPF_EMIT_CALL(BPF_FUNC_get_stack),
11759 BPF_MOV64_IMM(BPF_REG_1, 0),
11760 BPF_MOV64_REG(BPF_REG_8, BPF_REG_0),
11761 BPF_ALU64_IMM(BPF_LSH, BPF_REG_8, 32),
11762 BPF_ALU64_IMM(BPF_ARSH, BPF_REG_8, 32),
11763 BPF_JMP_REG(BPF_JSLT, BPF_REG_1, BPF_REG_8, 16),
11764 BPF_ALU64_REG(BPF_SUB, BPF_REG_9, BPF_REG_8),
11765 BPF_MOV64_REG(BPF_REG_2, BPF_REG_7),
11766 BPF_ALU64_REG(BPF_ADD, BPF_REG_2, BPF_REG_8),
11767 BPF_MOV64_REG(BPF_REG_1, BPF_REG_9),
11768 BPF_ALU64_IMM(BPF_LSH, BPF_REG_1, 32),
11769 BPF_ALU64_IMM(BPF_ARSH, BPF_REG_1, 32),
11770 BPF_MOV64_REG(BPF_REG_3, BPF_REG_2),
11771 BPF_ALU64_REG(BPF_ADD, BPF_REG_3, BPF_REG_1),
11772 BPF_MOV64_REG(BPF_REG_1, BPF_REG_7),
11773 BPF_MOV64_IMM(BPF_REG_5, sizeof(struct test_val)),
11774 BPF_ALU64_REG(BPF_ADD, BPF_REG_1, BPF_REG_5),
11775 BPF_JMP_REG(BPF_JGE, BPF_REG_3, BPF_REG_1, 4),
11776 BPF_MOV64_REG(BPF_REG_1, BPF_REG_6),
11777 BPF_MOV64_REG(BPF_REG_3, BPF_REG_9),
11778 BPF_MOV64_IMM(BPF_REG_4, 0),
11779 BPF_EMIT_CALL(BPF_FUNC_get_stack),
11780 BPF_EXIT_INSN(),
11781 },
11782 .fixup_map2 = { 4 },
11783 .result = ACCEPT,
11784 .prog_type = BPF_PROG_TYPE_TRACEPOINT,
11785 },
11786 {
11787 "ld_abs: invalid op 1",
11788 .insns = {
11789 BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
11790 BPF_LD_ABS(BPF_DW, 0),
11791 BPF_EXIT_INSN(),
11792 },
11793 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
11794 .result = REJECT,
11795 .errstr = "unknown opcode",
11796 },
11797 {
11798 "ld_abs: invalid op 2",
11799 .insns = {
11800 BPF_MOV32_IMM(BPF_REG_0, 256),
11801 BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
11802 BPF_LD_IND(BPF_DW, BPF_REG_0, 0),
11803 BPF_EXIT_INSN(),
11804 },
11805 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
11806 .result = REJECT,
11807 .errstr = "unknown opcode",
11808 },
11809 {
11810 "ld_abs: nmap reduced",
11811 .insns = {
11812 BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
11813 BPF_LD_ABS(BPF_H, 12),
11814 BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0x806, 28),
11815 BPF_LD_ABS(BPF_H, 12),
11816 BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0x806, 26),
11817 BPF_MOV32_IMM(BPF_REG_0, 18),
11818 BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_0, -64),
11819 BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_10, -64),
11820 BPF_LD_IND(BPF_W, BPF_REG_7, 14),
11821 BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_0, -60),
11822 BPF_MOV32_IMM(BPF_REG_0, 280971478),
11823 BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_0, -56),
11824 BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_10, -56),
11825 BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_10, -60),
11826 BPF_ALU32_REG(BPF_SUB, BPF_REG_0, BPF_REG_7),
11827 BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 15),
11828 BPF_LD_ABS(BPF_H, 12),
11829 BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0x806, 13),
11830 BPF_MOV32_IMM(BPF_REG_0, 22),
11831 BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_0, -56),
11832 BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_10, -56),
11833 BPF_LD_IND(BPF_H, BPF_REG_7, 14),
11834 BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_0, -52),
11835 BPF_MOV32_IMM(BPF_REG_0, 17366),
11836 BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_0, -48),
11837 BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_10, -48),
11838 BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_10, -52),
11839 BPF_ALU32_REG(BPF_SUB, BPF_REG_0, BPF_REG_7),
11840 BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 2),
11841 BPF_MOV32_IMM(BPF_REG_0, 256),
11842 BPF_EXIT_INSN(),
11843 BPF_MOV32_IMM(BPF_REG_0, 0),
11844 BPF_EXIT_INSN(),
11845 },
11846 .data = {
11847 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0x06, 0,
11848 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
11849 0x10, 0xbf, 0x48, 0xd6, 0x43, 0xd6,
11850 },
11851 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
11852 .result = ACCEPT,
11853 .retval = 256,
11854 },
11855 {
11856 "ld_abs: div + abs, test 1",
11857 .insns = {
11858 BPF_ALU64_REG(BPF_MOV, BPF_REG_6, BPF_REG_1),
11859 BPF_LD_ABS(BPF_B, 3),
11860 BPF_ALU64_IMM(BPF_MOV, BPF_REG_2, 2),
11861 BPF_ALU32_REG(BPF_DIV, BPF_REG_0, BPF_REG_2),
11862 BPF_ALU64_REG(BPF_MOV, BPF_REG_8, BPF_REG_0),
11863 BPF_LD_ABS(BPF_B, 4),
11864 BPF_ALU64_REG(BPF_ADD, BPF_REG_8, BPF_REG_0),
11865 BPF_LD_IND(BPF_B, BPF_REG_8, -70),
11866 BPF_EXIT_INSN(),
11867 },
11868 .data = {
11869 10, 20, 30, 40, 50,
11870 },
11871 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
11872 .result = ACCEPT,
11873 .retval = 10,
11874 },
11875 {
11876 "ld_abs: div + abs, test 2",
11877 .insns = {
11878 BPF_ALU64_REG(BPF_MOV, BPF_REG_6, BPF_REG_1),
11879 BPF_LD_ABS(BPF_B, 3),
11880 BPF_ALU64_IMM(BPF_MOV, BPF_REG_2, 2),
11881 BPF_ALU32_REG(BPF_DIV, BPF_REG_0, BPF_REG_2),
11882 BPF_ALU64_REG(BPF_MOV, BPF_REG_8, BPF_REG_0),
11883 BPF_LD_ABS(BPF_B, 128),
11884 BPF_ALU64_REG(BPF_ADD, BPF_REG_8, BPF_REG_0),
11885 BPF_LD_IND(BPF_B, BPF_REG_8, -70),
11886 BPF_EXIT_INSN(),
11887 },
11888 .data = {
11889 10, 20, 30, 40, 50,
11890 },
11891 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
11892 .result = ACCEPT,
11893 .retval = 0,
11894 },
11895 {
11896 "ld_abs: div + abs, test 3",
11897 .insns = {
11898 BPF_ALU64_REG(BPF_MOV, BPF_REG_6, BPF_REG_1),
11899 BPF_ALU64_IMM(BPF_MOV, BPF_REG_7, 0),
11900 BPF_LD_ABS(BPF_B, 3),
11901 BPF_ALU32_REG(BPF_DIV, BPF_REG_0, BPF_REG_7),
11902 BPF_EXIT_INSN(),
11903 },
11904 .data = {
11905 10, 20, 30, 40, 50,
11906 },
11907 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
11908 .result = ACCEPT,
11909 .retval = 0,
11910 },
11911 {
11912 "ld_abs: div + abs, test 4",
11913 .insns = {
11914 BPF_ALU64_REG(BPF_MOV, BPF_REG_6, BPF_REG_1),
11915 BPF_ALU64_IMM(BPF_MOV, BPF_REG_7, 0),
11916 BPF_LD_ABS(BPF_B, 256),
11917 BPF_ALU32_REG(BPF_DIV, BPF_REG_0, BPF_REG_7),
11918 BPF_EXIT_INSN(),
11919 },
11920 .data = {
11921 10, 20, 30, 40, 50,
11922 },
11923 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
11924 .result = ACCEPT,
11925 .retval = 0,
11926 },
11927 {
11928 "ld_abs: vlan + abs, test 1",
11929 .insns = { },
11930 .data = {
11931 0x34,
11932 },
11933 .fill_helper = bpf_fill_ld_abs_vlan_push_pop,
11934 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
11935 .result = ACCEPT,
11936 .retval = 0xbef,
11937 },
11938 {
11939 "ld_abs: vlan + abs, test 2",
11940 .insns = {
11941 BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
11942 BPF_LD_ABS(BPF_B, 0),
11943 BPF_LD_ABS(BPF_H, 0),
11944 BPF_LD_ABS(BPF_W, 0),
11945 BPF_MOV64_REG(BPF_REG_7, BPF_REG_6),
11946 BPF_MOV64_IMM(BPF_REG_6, 0),
11947 BPF_MOV64_REG(BPF_REG_1, BPF_REG_7),
11948 BPF_MOV64_IMM(BPF_REG_2, 1),
11949 BPF_MOV64_IMM(BPF_REG_3, 2),
11950 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
11951 BPF_FUNC_skb_vlan_push),
11952 BPF_MOV64_REG(BPF_REG_6, BPF_REG_7),
11953 BPF_LD_ABS(BPF_B, 0),
11954 BPF_LD_ABS(BPF_H, 0),
11955 BPF_LD_ABS(BPF_W, 0),
11956 BPF_MOV64_IMM(BPF_REG_0, 42),
11957 BPF_EXIT_INSN(),
11958 },
11959 .data = {
11960 0x34,
11961 },
11962 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
11963 .result = ACCEPT,
11964 .retval = 42,
11965 },
11966 {
11967 "ld_abs: jump around ld_abs",
11968 .insns = { },
11969 .data = {
11970 10, 11,
11971 },
11972 .fill_helper = bpf_fill_jump_around_ld_abs,
11973 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
11974 .result = ACCEPT,
11975 .retval = 10,
11976 },
11683}; 11977};
11684 11978
11685static int probe_filter_length(const struct bpf_insn *fp) 11979static int probe_filter_length(const struct bpf_insn *fp)
@@ -11783,7 +12077,7 @@ static int create_map_in_map(void)
11783 return outer_map_fd; 12077 return outer_map_fd;
11784} 12078}
11785 12079
11786static char bpf_vlog[32768]; 12080static char bpf_vlog[UINT_MAX >> 8];
11787 12081
11788static void do_test_fixup(struct bpf_test *test, struct bpf_insn *prog, 12082static void do_test_fixup(struct bpf_test *test, struct bpf_insn *prog,
11789 int *map_fds) 12083 int *map_fds)
@@ -11794,6 +12088,9 @@ static void do_test_fixup(struct bpf_test *test, struct bpf_insn *prog,
11794 int *fixup_prog = test->fixup_prog; 12088 int *fixup_prog = test->fixup_prog;
11795 int *fixup_map_in_map = test->fixup_map_in_map; 12089 int *fixup_map_in_map = test->fixup_map_in_map;
11796 12090
12091 if (test->fill_helper)
12092 test->fill_helper(test);
12093
11797 /* Allocating HTs with 1 elem is fine here, since we only test 12094 /* Allocating HTs with 1 elem is fine here, since we only test
11798 * for verifier and not do a runtime lookup, so the only thing 12095 * for verifier and not do a runtime lookup, so the only thing
11799 * that really matters is value size in this case. 12096 * that really matters is value size in this case.
@@ -11843,10 +12140,8 @@ static void do_test_single(struct bpf_test *test, bool unpriv,
11843 int *passes, int *errors) 12140 int *passes, int *errors)
11844{ 12141{
11845 int fd_prog, expected_ret, reject_from_alignment; 12142 int fd_prog, expected_ret, reject_from_alignment;
12143 int prog_len, prog_type = test->prog_type;
11846 struct bpf_insn *prog = test->insns; 12144 struct bpf_insn *prog = test->insns;
11847 int prog_len = probe_filter_length(prog);
11848 char data_in[TEST_DATA_LEN] = {};
11849 int prog_type = test->prog_type;
11850 int map_fds[MAX_NR_MAPS]; 12145 int map_fds[MAX_NR_MAPS];
11851 const char *expected_err; 12146 const char *expected_err;
11852 uint32_t retval; 12147 uint32_t retval;
@@ -11856,6 +12151,7 @@ static void do_test_single(struct bpf_test *test, bool unpriv,
11856 map_fds[i] = -1; 12151 map_fds[i] = -1;
11857 12152
11858 do_test_fixup(test, prog, map_fds); 12153 do_test_fixup(test, prog, map_fds);
12154 prog_len = probe_filter_length(prog);
11859 12155
11860 fd_prog = bpf_verify_program(prog_type ? : BPF_PROG_TYPE_SOCKET_FILTER, 12156 fd_prog = bpf_verify_program(prog_type ? : BPF_PROG_TYPE_SOCKET_FILTER,
11861 prog, prog_len, test->flags & F_LOAD_WITH_STRICT_ALIGNMENT, 12157 prog, prog_len, test->flags & F_LOAD_WITH_STRICT_ALIGNMENT,
@@ -11895,8 +12191,9 @@ static void do_test_single(struct bpf_test *test, bool unpriv,
11895 } 12191 }
11896 12192
11897 if (fd_prog >= 0) { 12193 if (fd_prog >= 0) {
11898 err = bpf_prog_test_run(fd_prog, 1, data_in, sizeof(data_in), 12194 err = bpf_prog_test_run(fd_prog, 1, test->data,
11899 NULL, NULL, &retval, NULL); 12195 sizeof(test->data), NULL, NULL,
12196 &retval, NULL);
11900 if (err && errno != 524/*ENOTSUPP*/ && errno != EPERM) { 12197 if (err && errno != 524/*ENOTSUPP*/ && errno != EPERM) {
11901 printf("Unexpected bpf_prog_test_run error\n"); 12198 printf("Unexpected bpf_prog_test_run error\n");
11902 goto fail_log; 12199 goto fail_log;
diff --git a/tools/testing/selftests/bpf/trace_helpers.c b/tools/testing/selftests/bpf/trace_helpers.c
new file mode 100644
index 000000000000..ad025bd75f1c
--- /dev/null
+++ b/tools/testing/selftests/bpf/trace_helpers.c
@@ -0,0 +1,180 @@
1// SPDX-License-Identifier: GPL-2.0
2#include <stdio.h>
3#include <stdlib.h>
4#include <string.h>
5#include <assert.h>
6#include <errno.h>
7#include <poll.h>
8#include <unistd.h>
9#include <linux/perf_event.h>
10#include <sys/mman.h>
11#include "trace_helpers.h"
12
13#define MAX_SYMS 300000
14static struct ksym syms[MAX_SYMS];
15static int sym_cnt;
16
17static int ksym_cmp(const void *p1, const void *p2)
18{
19 return ((struct ksym *)p1)->addr - ((struct ksym *)p2)->addr;
20}
21
22int load_kallsyms(void)
23{
24 FILE *f = fopen("/proc/kallsyms", "r");
25 char func[256], buf[256];
26 char symbol;
27 void *addr;
28 int i = 0;
29
30 if (!f)
31 return -ENOENT;
32
33 while (!feof(f)) {
34 if (!fgets(buf, sizeof(buf), f))
35 break;
36 if (sscanf(buf, "%p %c %s", &addr, &symbol, func) != 3)
37 break;
38 if (!addr)
39 continue;
40 syms[i].addr = (long) addr;
41 syms[i].name = strdup(func);
42 i++;
43 }
44 sym_cnt = i;
45 qsort(syms, sym_cnt, sizeof(struct ksym), ksym_cmp);
46 return 0;
47}
48
49struct ksym *ksym_search(long key)
50{
51 int start = 0, end = sym_cnt;
52 int result;
53
54 while (start < end) {
55 size_t mid = start + (end - start) / 2;
56
57 result = key - syms[mid].addr;
58 if (result < 0)
59 end = mid;
60 else if (result > 0)
61 start = mid + 1;
62 else
63 return &syms[mid];
64 }
65
66 if (start >= 1 && syms[start - 1].addr < key &&
67 key < syms[start].addr)
68 /* valid ksym */
69 return &syms[start - 1];
70
71 /* out of range. return _stext */
72 return &syms[0];
73}
74
75static int page_size;
76static int page_cnt = 8;
77static volatile struct perf_event_mmap_page *header;
78
79int perf_event_mmap(int fd)
80{
81 void *base;
82 int mmap_size;
83
84 page_size = getpagesize();
85 mmap_size = page_size * (page_cnt + 1);
86
87 base = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
88 if (base == MAP_FAILED) {
89 printf("mmap err\n");
90 return -1;
91 }
92
93 header = base;
94 return 0;
95}
96
97static int perf_event_poll(int fd)
98{
99 struct pollfd pfd = { .fd = fd, .events = POLLIN };
100
101 return poll(&pfd, 1, 1000);
102}
103
104struct perf_event_sample {
105 struct perf_event_header header;
106 __u32 size;
107 char data[];
108};
109
110static int perf_event_read(perf_event_print_fn fn)
111{
112 __u64 data_tail = header->data_tail;
113 __u64 data_head = header->data_head;
114 __u64 buffer_size = page_cnt * page_size;
115 void *base, *begin, *end;
116 char buf[256];
117 int ret;
118
119 asm volatile("" ::: "memory"); /* in real code it should be smp_rmb() */
120 if (data_head == data_tail)
121 return PERF_EVENT_CONT;
122
123 base = ((char *)header) + page_size;
124
125 begin = base + data_tail % buffer_size;
126 end = base + data_head % buffer_size;
127
128 while (begin != end) {
129 struct perf_event_sample *e;
130
131 e = begin;
132 if (begin + e->header.size > base + buffer_size) {
133 long len = base + buffer_size - begin;
134
135 assert(len < e->header.size);
136 memcpy(buf, begin, len);
137 memcpy(buf + len, base, e->header.size - len);
138 e = (void *) buf;
139 begin = base + e->header.size - len;
140 } else if (begin + e->header.size == base + buffer_size) {
141 begin = base;
142 } else {
143 begin += e->header.size;
144 }
145
146 if (e->header.type == PERF_RECORD_SAMPLE) {
147 ret = fn(e->data, e->size);
148 if (ret != PERF_EVENT_CONT)
149 return ret;
150 } else if (e->header.type == PERF_RECORD_LOST) {
151 struct {
152 struct perf_event_header header;
153 __u64 id;
154 __u64 lost;
155 } *lost = (void *) e;
156 printf("lost %lld events\n", lost->lost);
157 } else {
158 printf("unknown event type=%d size=%d\n",
159 e->header.type, e->header.size);
160 }
161 }
162
163 __sync_synchronize(); /* smp_mb() */
164 header->data_tail = data_head;
165 return PERF_EVENT_CONT;
166}
167
168int perf_event_poller(int fd, perf_event_print_fn output_fn)
169{
170 int ret;
171
172 for (;;) {
173 perf_event_poll(fd);
174 ret = perf_event_read(output_fn);
175 if (ret != PERF_EVENT_CONT)
176 return ret;
177 }
178
179 return PERF_EVENT_DONE;
180}
diff --git a/tools/testing/selftests/bpf/trace_helpers.h b/tools/testing/selftests/bpf/trace_helpers.h
new file mode 100644
index 000000000000..fe3eefd21e86
--- /dev/null
+++ b/tools/testing/selftests/bpf/trace_helpers.h
@@ -0,0 +1,23 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef __TRACE_HELPER_H
3#define __TRACE_HELPER_H
4
5struct ksym {
6 long addr;
7 char *name;
8};
9
10int load_kallsyms(void);
11struct ksym *ksym_search(long key);
12
13typedef int (*perf_event_print_fn)(void *data, int size);
14
15/* return code for perf_event_print_fn */
16#define PERF_EVENT_DONE 0
17#define PERF_EVENT_ERROR -1
18#define PERF_EVENT_CONT -2
19
20int perf_event_mmap(int fd);
21/* return PERF_EVENT_DONE or PERF_EVENT_ERROR */
22int perf_event_poller(int fd, perf_event_print_fn output_fn);
23#endif
generated by cgit v1.2.2 (git 2.25.0) at 2026-01-05 16:26:47 -0500