aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorDaniel Borkmann <daniel@iogearbox.net>2018-04-19 15:48:19 -0400
committerDaniel Borkmann <daniel@iogearbox.net>2018-04-19 15:48:35 -0400
commit34df9d37abe0b7cd86d08d88b54de9db211d15c0 (patch)
tree59286f44ca694018a22239ed61368230ba8c4040
parent97e19cce05e50055dafd1df71bdcbcdc3a7894c5 (diff)
parentc0fa1b6c3efc64b4ecfec658f95475fda650742e (diff)
Merge branch 'bpf-type-format'
Martin KaFai Lau says: ==================== This patch introduces BPF Type Format (BTF). BTF (BPF Type Format) is the meta data format which describes the data types of BPF program/map. Hence, it basically focus on the C programming language which the modern BPF is primary using. The first use case is to provide a generic pretty print capability for a BPF map. A modified pahole that can convert dwarf to BTF is here: https://github.com/iamkafai/pahole/tree/btf Please see individual patch for details. v5: - Remove BTF_KIND_FLOAT and BTF_KIND_FUNC which are not currently used. They can be added in the future. Some bpf_df_xxx() are removed together. - Add comment in patch 7 to clarify that the new bpffs_map_fops should not be extended further. v4: - Fix warning (remove unneeded semicolon) - Remove a redundant variable (nr_bytes) from btf_int_check_meta() in patch 1. Caught by W=1. v3: - Rebase to bpf-next - Fix sparse warning (by adding static) - Add BTF header logging: btf_verifier_log_hdr() - Fix the alignment test on btf->type_off - Add tests for the BTF header - Lower the max BTF size to 16MB. It should be enough for some time. We could raise it later if it would be needed. v2: - Use kvfree where needed in patch 1 and 2 - Also consider BTF_INT_OFFSET() in the btf_int_check_meta() in patch 1 - Fix an incorrect goto target in map_create() during the btf-error-path in patch 7 - re-org some local vars to keep the rev xmas tree in btf.c ==================== Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Diffstat
-rw-r--r--include/linux/bpf.h20
-rw-r--r--include/linux/btf.h48
-rw-r--r--include/uapi/linux/bpf.h12
-rw-r--r--include/uapi/linux/btf.h130
-rw-r--r--kernel/bpf/Makefile1
-rw-r--r--kernel/bpf/arraymap.c50
-rw-r--r--kernel/bpf/btf.c2064
-rw-r--r--kernel/bpf/inode.c156
-rw-r--r--kernel/bpf/syscall.c51
-rw-r--r--tools/include/uapi/linux/bpf.h12
-rw-r--r--tools/include/uapi/linux/btf.h130
-rw-r--r--tools/lib/bpf/Build2
-rw-r--r--tools/lib/bpf/bpf.c92
-rw-r--r--tools/lib/bpf/bpf.h16
-rw-r--r--tools/lib/bpf/btf.c374
-rw-r--r--tools/lib/bpf/btf.h22
-rw-r--r--tools/lib/bpf/libbpf.c148
-rw-r--r--tools/lib/bpf/libbpf.h3
-rw-r--r--tools/testing/selftests/bpf/Makefile26
-rw-r--r--tools/testing/selftests/bpf/test_btf.c1669
-rw-r--r--tools/testing/selftests/bpf/test_btf_haskv.c48
-rw-r--r--tools/testing/selftests/bpf/test_btf_nokv.c43
22 files changed, 5076 insertions, 41 deletions
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 95a7abd0ee92..ee5275e7d4df 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -22,6 +22,8 @@ struct perf_event;
22struct bpf_prog; 22struct bpf_prog;
23struct bpf_map; 23struct bpf_map;
24struct sock; 24struct sock;
25struct seq_file;
26struct btf;
25 27
26/* map is generic key/value storage optionally accesible by eBPF programs */ 28/* map is generic key/value storage optionally accesible by eBPF programs */
27struct bpf_map_ops { 29struct bpf_map_ops {
@@ -43,10 +45,14 @@ struct bpf_map_ops {
43 void (*map_fd_put_ptr)(void *ptr); 45 void (*map_fd_put_ptr)(void *ptr);
44 u32 (*map_gen_lookup)(struct bpf_map *map, struct bpf_insn *insn_buf); 46 u32 (*map_gen_lookup)(struct bpf_map *map, struct bpf_insn *insn_buf);
45 u32 (*map_fd_sys_lookup_elem)(void *ptr); 47 u32 (*map_fd_sys_lookup_elem)(void *ptr);
48 void (*map_seq_show_elem)(struct bpf_map *map, void *key,
49 struct seq_file *m);
50 int (*map_check_btf)(const struct bpf_map *map, const struct btf *btf,
51 u32 key_type_id, u32 value_type_id);
46}; 52};
47 53
48struct bpf_map { 54struct bpf_map {
49 /* 1st cacheline with read-mostly members of which some 55 /* The first two cachelines with read-mostly members of which some
50 * are also accessed in fast-path (e.g. ops, max_entries). 56 * are also accessed in fast-path (e.g. ops, max_entries).
51 */ 57 */
52 const struct bpf_map_ops *ops ____cacheline_aligned; 58 const struct bpf_map_ops *ops ____cacheline_aligned;
@@ -62,10 +68,13 @@ struct bpf_map {
62 u32 pages; 68 u32 pages;
63 u32 id; 69 u32 id;
64 int numa_node; 70 int numa_node;
71 u32 btf_key_id;
72 u32 btf_value_id;
73 struct btf *btf;
65 bool unpriv_array; 74 bool unpriv_array;
66 /* 7 bytes hole */ 75 /* 55 bytes hole */
67 76
68 /* 2nd cacheline with misc members to avoid false sharing 77 /* The 3rd and 4th cacheline with misc members to avoid false sharing
69 * particularly with refcounting. 78 * particularly with refcounting.
70 */ 79 */
71 struct user_struct *user ____cacheline_aligned; 80 struct user_struct *user ____cacheline_aligned;
@@ -100,6 +109,11 @@ static inline struct bpf_offloaded_map *map_to_offmap(struct bpf_map *map)
100 return container_of(map, struct bpf_offloaded_map, map); 109 return container_of(map, struct bpf_offloaded_map, map);
101} 110}
102 111
112static inline bool bpf_map_support_seq_show(const struct bpf_map *map)
113{
114 return map->ops->map_seq_show_elem && map->ops->map_check_btf;
115}
116
103extern const struct bpf_map_ops bpf_map_offload_ops; 117extern const struct bpf_map_ops bpf_map_offload_ops;
104 118
105/* function argument constraints */ 119/* function argument constraints */
diff --git a/include/linux/btf.h b/include/linux/btf.h
new file mode 100644
index 000000000000..a966dc6d61ee
--- /dev/null
+++ b/include/linux/btf.h
@@ -0,0 +1,48 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/* Copyright (c) 2018 Facebook */
3
4#ifndef _LINUX_BTF_H
5#define _LINUX_BTF_H 1
6
7#include <linux/types.h>
8
9struct btf;
10struct btf_type;
11union bpf_attr;
12
13extern const struct file_operations btf_fops;
14
15void btf_put(struct btf *btf);
16int btf_new_fd(const union bpf_attr *attr);
17struct btf *btf_get_by_fd(int fd);
18int btf_get_info_by_fd(const struct btf *btf,
19 const union bpf_attr *attr,
20 union bpf_attr __user *uattr);
21/* Figure out the size of a type_id. If type_id is a modifier
22 * (e.g. const), it will be resolved to find out the type with size.
23 *
24 * For example:
25 * In describing "const void *", type_id is "const" and "const"
26 * refers to "void *". The return type will be "void *".
27 *
28 * If type_id is a simple "int", then return type will be "int".
29 *
30 * @btf: struct btf object
31 * @type_id: Find out the size of type_id. The type_id of the return
32 * type is set to *type_id.
33 * @ret_size: It can be NULL. If not NULL, the size of the return
34 * type is set to *ret_size.
35 * Return: The btf_type (resolved to another type with size info if needed).
36 * NULL is returned if type_id itself does not have size info
37 * (e.g. void) or it cannot be resolved to another type that
38 * has size info.
39 * *type_id and *ret_size will not be changed in the
40 * NULL return case.
41 */
42const struct btf_type *btf_type_id_size(const struct btf *btf,
43 u32 *type_id,
44 u32 *ret_size);
45void btf_type_seq_show(const struct btf *btf, u32 type_id, void *obj,
46 struct seq_file *m);
47
48#endif
diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index 9a2d1a04eb24..c8383a289f7b 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -95,6 +95,7 @@ enum bpf_cmd {
95 BPF_OBJ_GET_INFO_BY_FD, 95 BPF_OBJ_GET_INFO_BY_FD,
96 BPF_PROG_QUERY, 96 BPF_PROG_QUERY,
97 BPF_RAW_TRACEPOINT_OPEN, 97 BPF_RAW_TRACEPOINT_OPEN,
98 BPF_BTF_LOAD,
98}; 99};
99 100
100enum bpf_map_type { 101enum bpf_map_type {
@@ -279,6 +280,9 @@ union bpf_attr {
279 */ 280 */
280 char map_name[BPF_OBJ_NAME_LEN]; 281 char map_name[BPF_OBJ_NAME_LEN];
281 __u32 map_ifindex; /* ifindex of netdev to create on */ 282 __u32 map_ifindex; /* ifindex of netdev to create on */
283 __u32 btf_fd; /* fd pointing to a BTF type data */
284 __u32 btf_key_id; /* BTF type_id of the key */
285 __u32 btf_value_id; /* BTF type_id of the value */
282 }; 286 };
283 287
284 struct { /* anonymous struct used by BPF_MAP_*_ELEM commands */ 288 struct { /* anonymous struct used by BPF_MAP_*_ELEM commands */
@@ -363,6 +367,14 @@ union bpf_attr {
363 __u64 name; 367 __u64 name;
364 __u32 prog_fd; 368 __u32 prog_fd;
365 } raw_tracepoint; 369 } raw_tracepoint;
370
371 struct { /* anonymous struct for BPF_BTF_LOAD */
372 __aligned_u64 btf;
373 __aligned_u64 btf_log_buf;
374 __u32 btf_size;
375 __u32 btf_log_size;
376 __u32 btf_log_level;
377 };
366} __attribute__((aligned(8))); 378} __attribute__((aligned(8)));
367 379
368/* BPF helper function descriptions: 380/* BPF helper function descriptions:
diff --git a/include/uapi/linux/btf.h b/include/uapi/linux/btf.h
new file mode 100644
index 000000000000..74a30b1090df
--- /dev/null
+++ b/include/uapi/linux/btf.h
@@ -0,0 +1,130 @@
1/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
2/* Copyright (c) 2018 Facebook */
3#ifndef _UAPI__LINUX_BTF_H__
4#define _UAPI__LINUX_BTF_H__
5
6#include <linux/types.h>
7
8#define BTF_MAGIC 0xeB9F
9#define BTF_MAGIC_SWAP 0x9FeB
10#define BTF_VERSION 1
11#define BTF_FLAGS_COMPR 0x01
12
13struct btf_header {
14 __u16 magic;
15 __u8 version;
16 __u8 flags;
17
18 __u32 parent_label;
19 __u32 parent_name;
20
21 /* All offsets are in bytes relative to the end of this header */
22 __u32 label_off; /* offset of label section */
23 __u32 object_off; /* offset of data object section*/
24 __u32 func_off; /* offset of function section */
25 __u32 type_off; /* offset of type section */
26 __u32 str_off; /* offset of string section */
27 __u32 str_len; /* length of string section */
28};
29
30/* Max # of type identifier */
31#define BTF_MAX_TYPE 0x7fffffff
32/* Max offset into the string section */
33#define BTF_MAX_NAME_OFFSET 0x7fffffff
34/* Max # of struct/union/enum members or func args */
35#define BTF_MAX_VLEN 0xffff
36
37/* The type id is referring to a parent BTF */
38#define BTF_TYPE_PARENT(id) (((id) >> 31) & 0x1)
39#define BTF_TYPE_ID(id) ((id) & BTF_MAX_TYPE)
40
41/* String is in the ELF string section */
42#define BTF_STR_TBL_ELF_ID(ref) (((ref) >> 31) & 0x1)
43#define BTF_STR_OFFSET(ref) ((ref) & BTF_MAX_NAME_OFFSET)
44
45struct btf_type {
46 __u32 name;
47 /* "info" bits arrangement
48 * bits 0-15: vlen (e.g. # of struct's members)
49 * bits 16-23: unused
50 * bits 24-28: kind (e.g. int, ptr, array...etc)
51 * bits 29-30: unused
52 * bits 31: root
53 */
54 __u32 info;
55 /* "size" is used by INT, ENUM, STRUCT and UNION.
56 * "size" tells the size of the type it is describing.
57 *
58 * "type" is used by PTR, TYPEDEF, VOLATILE, CONST and RESTRICT.
59 * "type" is a type_id referring to another type.
60 */
61 union {
62 __u32 size;
63 __u32 type;
64 };
65};
66
67#define BTF_INFO_KIND(info) (((info) >> 24) & 0x1f)
68#define BTF_INFO_ISROOT(info) (!!(((info) >> 24) & 0x80))
69#define BTF_INFO_VLEN(info) ((info) & 0xffff)
70
71#define BTF_KIND_UNKN 0 /* Unknown */
72#define BTF_KIND_INT 1 /* Integer */
73#define BTF_KIND_PTR 2 /* Pointer */
74#define BTF_KIND_ARRAY 3 /* Array */
75#define BTF_KIND_STRUCT 4 /* Struct */
76#define BTF_KIND_UNION 5 /* Union */
77#define BTF_KIND_ENUM 6 /* Enumeration */
78#define BTF_KIND_FWD 7 /* Forward */
79#define BTF_KIND_TYPEDEF 8 /* Typedef */
80#define BTF_KIND_VOLATILE 9 /* Volatile */
81#define BTF_KIND_CONST 10 /* Const */
82#define BTF_KIND_RESTRICT 11 /* Restrict */
83#define BTF_KIND_MAX 11
84#define NR_BTF_KINDS 12
85
86/* For some specific BTF_KIND, "struct btf_type" is immediately
87 * followed by extra data.
88 */
89
90/* BTF_KIND_INT is followed by a u32 and the following
91 * is the 32 bits arrangement:
92 */
93#define BTF_INT_ENCODING(VAL) (((VAL) & 0xff000000) >> 24)
94#define BTF_INT_OFFSET(VAL) (((VAL & 0x00ff0000)) >> 16)
95#define BTF_INT_BITS(VAL) ((VAL) & 0x0000ffff)
96
97/* Attributes stored in the BTF_INT_ENCODING */
98#define BTF_INT_SIGNED 0x1
99#define BTF_INT_CHAR 0x2
100#define BTF_INT_BOOL 0x4
101#define BTF_INT_VARARGS 0x8
102
103/* BTF_KIND_ENUM is followed by multiple "struct btf_enum".
104 * The exact number of btf_enum is stored in the vlen (of the
105 * info in "struct btf_type").
106 */
107struct btf_enum {
108 __u32 name;
109 __s32 val;
110};
111
112/* BTF_KIND_ARRAY is followed by one "struct btf_array" */
113struct btf_array {
114 __u32 type;
115 __u32 index_type;
116 __u32 nelems;
117};
118
119/* BTF_KIND_STRUCT and BTF_KIND_UNION are followed
120 * by multiple "struct btf_member". The exact number
121 * of btf_member is stored in the vlen (of the info in
122 * "struct btf_type").
123 */
124struct btf_member {
125 __u32 name;
126 __u32 type;
127 __u32 offset; /* offset in bits */
128};
129
130#endif /* _UAPI__LINUX_BTF_H__ */
diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile
index a713fd23ec88..35c485fa9ea3 100644
--- a/kernel/bpf/Makefile
+++ b/kernel/bpf/Makefile
@@ -4,6 +4,7 @@ obj-y := core.o
4obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o 4obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o
5obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o 5obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o
6obj-$(CONFIG_BPF_SYSCALL) += disasm.o 6obj-$(CONFIG_BPF_SYSCALL) += disasm.o
7obj-$(CONFIG_BPF_SYSCALL) += btf.o
7ifeq ($(CONFIG_NET),y) 8ifeq ($(CONFIG_NET),y)
8obj-$(CONFIG_BPF_SYSCALL) += devmap.o 9obj-$(CONFIG_BPF_SYSCALL) += devmap.o
9obj-$(CONFIG_BPF_SYSCALL) += cpumap.o 10obj-$(CONFIG_BPF_SYSCALL) += cpumap.o
diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c
index 14750e7c5ee4..02a189339381 100644
--- a/kernel/bpf/arraymap.c
+++ b/kernel/bpf/arraymap.c
@@ -11,11 +11,13 @@
11 * General Public License for more details. 11 * General Public License for more details.
12 */ 12 */
13#include <linux/bpf.h> 13#include <linux/bpf.h>
14#include <linux/btf.h>
14#include <linux/err.h> 15#include <linux/err.h>
15#include <linux/slab.h> 16#include <linux/slab.h>
16#include <linux/mm.h> 17#include <linux/mm.h>
17#include <linux/filter.h> 18#include <linux/filter.h>
18#include <linux/perf_event.h> 19#include <linux/perf_event.h>
20#include <uapi/linux/btf.h>
19 21
20#include "map_in_map.h" 22#include "map_in_map.h"
21 23
@@ -336,6 +338,52 @@ static void array_map_free(struct bpf_map *map)
336 bpf_map_area_free(array); 338 bpf_map_area_free(array);
337} 339}
338 340
341static void array_map_seq_show_elem(struct bpf_map *map, void *key,
342 struct seq_file *m)
343{
344 void *value;
345
346 rcu_read_lock();
347
348 value = array_map_lookup_elem(map, key);
349 if (!value) {
350 rcu_read_unlock();
351 return;
352 }
353
354 seq_printf(m, "%u: ", *(u32 *)key);
355 btf_type_seq_show(map->btf, map->btf_value_id, value, m);
356 seq_puts(m, "\n");
357
358 rcu_read_unlock();
359}
360
361static int array_map_check_btf(const struct bpf_map *map, const struct btf *btf,
362 u32 btf_key_id, u32 btf_value_id)
363{
364 const struct btf_type *key_type, *value_type;
365 u32 key_size, value_size;
366 u32 int_data;
367
368 key_type = btf_type_id_size(btf, &btf_key_id, &key_size);
369 if (!key_type || BTF_INFO_KIND(key_type->info) != BTF_KIND_INT)
370 return -EINVAL;
371
372 int_data = *(u32 *)(key_type + 1);
373 /* bpf array can only take a u32 key. This check makes
374 * sure that the btf matches the attr used during map_create.
375 */
376 if (BTF_INT_BITS(int_data) != 32 || key_size != 4 ||
377 BTF_INT_OFFSET(int_data))
378 return -EINVAL;
379
380 value_type = btf_type_id_size(btf, &btf_value_id, &value_size);
381 if (!value_type || value_size > map->value_size)
382 return -EINVAL;
383
384 return 0;
385}
386
339const struct bpf_map_ops array_map_ops = { 387const struct bpf_map_ops array_map_ops = {
340 .map_alloc_check = array_map_alloc_check, 388 .map_alloc_check = array_map_alloc_check,
341 .map_alloc = array_map_alloc, 389 .map_alloc = array_map_alloc,
@@ -345,6 +393,8 @@ const struct bpf_map_ops array_map_ops = {
345 .map_update_elem = array_map_update_elem, 393 .map_update_elem = array_map_update_elem,
346 .map_delete_elem = array_map_delete_elem, 394 .map_delete_elem = array_map_delete_elem,
347 .map_gen_lookup = array_map_gen_lookup, 395 .map_gen_lookup = array_map_gen_lookup,
396 .map_seq_show_elem = array_map_seq_show_elem,
397 .map_check_btf = array_map_check_btf,
348}; 398};
349 399
350const struct bpf_map_ops percpu_array_map_ops = { 400const struct bpf_map_ops percpu_array_map_ops = {
diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c
new file mode 100644
index 000000000000..eb56ac760547
--- /dev/null
+++ b/kernel/bpf/btf.c
@@ -0,0 +1,2064 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/* Copyright (c) 2018 Facebook */
3
4#include <uapi/linux/btf.h>
5#include <uapi/linux/types.h>
6#include <linux/seq_file.h>
7#include <linux/compiler.h>
8#include <linux/errno.h>
9#include <linux/slab.h>
10#include <linux/anon_inodes.h>
11#include <linux/file.h>
12#include <linux/uaccess.h>
13#include <linux/kernel.h>
14#include <linux/bpf_verifier.h>
15#include <linux/btf.h>
16
17/* BTF (BPF Type Format) is the meta data format which describes
18 * the data types of BPF program/map. Hence, it basically focus
19 * on the C programming language which the modern BPF is primary
20 * using.
21 *
22 * ELF Section:
23 * ~~~~~~~~~~~
24 * The BTF data is stored under the ".BTF" ELF section
25 *
26 * struct btf_type:
27 * ~~~~~~~~~~~~~~~
28 * Each 'struct btf_type' object describes a C data type.
29 * Depending on the type it is describing, a 'struct btf_type'
30 * object may be followed by more data. F.e.
31 * To describe an array, 'struct btf_type' is followed by
32 * 'struct btf_array'.
33 *
34 * 'struct btf_type' and any extra data following it are
35 * 4 bytes aligned.
36 *
37 * Type section:
38 * ~~~~~~~~~~~~~
39 * The BTF type section contains a list of 'struct btf_type' objects.
40 * Each one describes a C type. Recall from the above section
41 * that a 'struct btf_type' object could be immediately followed by extra
42 * data in order to desribe some particular C types.
43 *
44 * type_id:
45 * ~~~~~~~
46 * Each btf_type object is identified by a type_id. The type_id
47 * is implicitly implied by the location of the btf_type object in
48 * the BTF type section. The first one has type_id 1. The second
49 * one has type_id 2...etc. Hence, an earlier btf_type has
50 * a smaller type_id.
51 *
52 * A btf_type object may refer to another btf_type object by using
53 * type_id (i.e. the "type" in the "struct btf_type").
54 *
55 * NOTE that we cannot assume any reference-order.
56 * A btf_type object can refer to an earlier btf_type object
57 * but it can also refer to a later btf_type object.
58 *
59 * For example, to describe "const void *". A btf_type
60 * object describing "const" may refer to another btf_type
61 * object describing "void *". This type-reference is done
62 * by specifying type_id:
63 *
64 * [1] CONST (anon) type_id=2
65 * [2] PTR (anon) type_id=0
66 *
67 * The above is the btf_verifier debug log:
68 * - Each line started with "[?]" is a btf_type object
69 * - [?] is the type_id of the btf_type object.
70 * - CONST/PTR is the BTF_KIND_XXX
71 * - "(anon)" is the name of the type. It just
72 * happens that CONST and PTR has no name.
73 * - type_id=XXX is the 'u32 type' in btf_type
74 *
75 * NOTE: "void" has type_id 0
76 *
77 * String section:
78 * ~~~~~~~~~~~~~~
79 * The BTF string section contains the names used by the type section.
80 * Each string is referred by an "offset" from the beginning of the
81 * string section.
82 *
83 * Each string is '\0' terminated.
84 *
85 * The first character in the string section must be '\0'
86 * which is used to mean 'anonymous'. Some btf_type may not
87 * have a name.
88 */
89
90/* BTF verification:
91 *
92 * To verify BTF data, two passes are needed.
93 *
94 * Pass #1
95 * ~~~~~~~
96 * The first pass is to collect all btf_type objects to
97 * an array: "btf->types".
98 *
99 * Depending on the C type that a btf_type is describing,
100 * a btf_type may be followed by extra data. We don't know
101 * how many btf_type is there, and more importantly we don't
102 * know where each btf_type is located in the type section.
103 *
104 * Without knowing the location of each type_id, most verifications
105 * cannot be done. e.g. an earlier btf_type may refer to a later
106 * btf_type (recall the "const void *" above), so we cannot
107 * check this type-reference in the first pass.
108 *
109 * In the first pass, it still does some verifications (e.g.
110 * checking the name is a valid offset to the string section).
111 *
112 * Pass #2
113 * ~~~~~~~
114 * The main focus is to resolve a btf_type that is referring
115 * to another type.
116 *
117 * We have to ensure the referring type:
118 * 1) does exist in the BTF (i.e. in btf->types[])
119 * 2) does not cause a loop:
120 * struct A {
121 * struct B b;
122 * };
123 *
124 * struct B {
125 * struct A a;
126 * };
127 *
128 * btf_type_needs_resolve() decides if a btf_type needs
129 * to be resolved.
130 *
131 * The needs_resolve type implements the "resolve()" ops which
132 * essentially does a DFS and detects backedge.
133 *
134 * During resolve (or DFS), different C types have different
135 * "RESOLVED" conditions.
136 *
137 * When resolving a BTF_KIND_STRUCT, we need to resolve all its
138 * members because a member is always referring to another
139 * type. A struct's member can be treated as "RESOLVED" if
140 * it is referring to a BTF_KIND_PTR. Otherwise, the
141 * following valid C struct would be rejected:
142 *
143 * struct A {
144 * int m;
145 * struct A *a;
146 * };
147 *
148 * When resolving a BTF_KIND_PTR, it needs to keep resolving if
149 * it is referring to another BTF_KIND_PTR. Otherwise, we cannot
150 * detect a pointer loop, e.g.:
151 * BTF_KIND_CONST -> BTF_KIND_PTR -> BTF_KIND_CONST -> BTF_KIND_PTR +
152 * ^ |
153 * +-----------------------------------------+
154 *
155 */
156
157#define BITS_PER_U64 (sizeof(u64) * BITS_PER_BYTE)
158#define BITS_PER_BYTE_MASK (BITS_PER_BYTE - 1)
159#define BITS_PER_BYTE_MASKED(bits) ((bits) & BITS_PER_BYTE_MASK)
160#define BITS_ROUNDDOWN_BYTES(bits) ((bits) >> 3)
161#define BITS_ROUNDUP_BYTES(bits) \
162 (BITS_ROUNDDOWN_BYTES(bits) + !!BITS_PER_BYTE_MASKED(bits))
163
164/* 16MB for 64k structs and each has 16 members and
165 * a few MB spaces for the string section.
166 * The hard limit is S32_MAX.
167 */
168#define BTF_MAX_SIZE (16 * 1024 * 1024)
169/* 64k. We can raise it later. The hard limit is S32_MAX. */
170#define BTF_MAX_NR_TYPES 65535
171
172#define for_each_member(i, struct_type, member) \
173 for (i = 0, member = btf_type_member(struct_type); \
174 i < btf_type_vlen(struct_type); \
175 i++, member++)
176
177#define for_each_member_from(i, from, struct_type, member) \
178 for (i = from, member = btf_type_member(struct_type) + from; \
179 i < btf_type_vlen(struct_type); \
180 i++, member++)
181
182struct btf {
183 union {
184 struct btf_header *hdr;
185 void *data;
186 };
187 struct btf_type **types;
188 u32 *resolved_ids;
189 u32 *resolved_sizes;
190 const char *strings;
191 void *nohdr_data;
192 u32 nr_types;
193 u32 types_size;
194 u32 data_size;
195 refcount_t refcnt;
196};
197
198enum verifier_phase {
199 CHECK_META,
200 CHECK_TYPE,
201};
202
203struct resolve_vertex {
204 const struct btf_type *t;
205 u32 type_id;
206 u16 next_member;
207};
208
209enum visit_state {
210 NOT_VISITED,
211 VISITED,
212 RESOLVED,
213};
214
215enum resolve_mode {
216 RESOLVE_TBD, /* To Be Determined */
217 RESOLVE_PTR, /* Resolving for Pointer */
218 RESOLVE_STRUCT_OR_ARRAY, /* Resolving for struct/union
219 * or array
220 */
221};
222
223#define MAX_RESOLVE_DEPTH 32
224
225struct btf_verifier_env {
226 struct btf *btf;
227 u8 *visit_states;
228 struct resolve_vertex stack[MAX_RESOLVE_DEPTH];
229 struct bpf_verifier_log log;
230 u32 log_type_id;
231 u32 top_stack;
232 enum verifier_phase phase;
233 enum resolve_mode resolve_mode;
234};
235
236static const char * const btf_kind_str[NR_BTF_KINDS] = {
237 [BTF_KIND_UNKN] = "UNKNOWN",
238 [BTF_KIND_INT] = "INT",
239 [BTF_KIND_PTR] = "PTR",
240 [BTF_KIND_ARRAY] = "ARRAY",
241 [BTF_KIND_STRUCT] = "STRUCT",
242 [BTF_KIND_UNION] = "UNION",
243 [BTF_KIND_ENUM] = "ENUM",
244 [BTF_KIND_FWD] = "FWD",
245 [BTF_KIND_TYPEDEF] = "TYPEDEF",
246 [BTF_KIND_VOLATILE] = "VOLATILE",
247 [BTF_KIND_CONST] = "CONST",
248 [BTF_KIND_RESTRICT] = "RESTRICT",
249};
250
251struct btf_kind_operations {
252 s32 (*check_meta)(struct btf_verifier_env *env,
253 const struct btf_type *t,
254 u32 meta_left);
255 int (*resolve)(struct btf_verifier_env *env,
256 const struct resolve_vertex *v);
257 int (*check_member)(struct btf_verifier_env *env,
258 const struct btf_type *struct_type,
259 const struct btf_member *member,
260 const struct btf_type *member_type);
261 void (*log_details)(struct btf_verifier_env *env,
262 const struct btf_type *t);
263 void (*seq_show)(const struct btf *btf, const struct btf_type *t,
264 u32 type_id, void *data, u8 bits_offsets,
265 struct seq_file *m);
266};
267
268static const struct btf_kind_operations * const kind_ops[NR_BTF_KINDS];
269static struct btf_type btf_void;
270
271static bool btf_type_is_modifier(const struct btf_type *t)
272{
273 /* Some of them is not strictly a C modifier
274 * but they are grouped into the same bucket
275 * for BTF concern:
276 * A type (t) that refers to another
277 * type through t->type AND its size cannot
278 * be determined without following the t->type.
279 *
280 * ptr does not fall into this bucket
281 * because its size is always sizeof(void *).
282 */
283 switch (BTF_INFO_KIND(t->info)) {
284 case BTF_KIND_TYPEDEF:
285 case BTF_KIND_VOLATILE:
286 case BTF_KIND_CONST:
287 case BTF_KIND_RESTRICT:
288 return true;
289 }
290
291 return false;
292}
293
294static bool btf_type_is_void(const struct btf_type *t)
295{
296 /* void => no type and size info.
297 * Hence, FWD is also treated as void.
298 */
299 return t == &btf_void || BTF_INFO_KIND(t->info) == BTF_KIND_FWD;
300}
301
302static bool btf_type_is_void_or_null(const struct btf_type *t)
303{
304 return !t || btf_type_is_void(t);
305}
306
307/* union is only a special case of struct:
308 * all its offsetof(member) == 0
309 */
310static bool btf_type_is_struct(const struct btf_type *t)
311{
312 u8 kind = BTF_INFO_KIND(t->info);
313
314 return kind == BTF_KIND_STRUCT || kind == BTF_KIND_UNION;
315}
316
317static bool btf_type_is_array(const struct btf_type *t)
318{
319 return BTF_INFO_KIND(t->info) == BTF_KIND_ARRAY;
320}
321
322static bool btf_type_is_ptr(const struct btf_type *t)
323{
324 return BTF_INFO_KIND(t->info) == BTF_KIND_PTR;
325}
326
327static bool btf_type_is_int(const struct btf_type *t)
328{
329 return BTF_INFO_KIND(t->info) == BTF_KIND_INT;
330}
331
332/* What types need to be resolved?
333 *
334 * btf_type_is_modifier() is an obvious one.
335 *
336 * btf_type_is_struct() because its member refers to
337 * another type (through member->type).
338
339 * btf_type_is_array() because its element (array->type)
340 * refers to another type. Array can be thought of a
341 * special case of struct while array just has the same
342 * member-type repeated by array->nelems of times.
343 */
344static bool btf_type_needs_resolve(const struct btf_type *t)
345{
346 return btf_type_is_modifier(t) ||
347 btf_type_is_ptr(t) ||
348 btf_type_is_struct(t) ||
349 btf_type_is_array(t);
350}
351
352/* t->size can be used */
353static bool btf_type_has_size(const struct btf_type *t)
354{
355 switch (BTF_INFO_KIND(t->info)) {
356 case BTF_KIND_INT:
357 case BTF_KIND_STRUCT:
358 case BTF_KIND_UNION:
359 case BTF_KIND_ENUM:
360 return true;
361 }
362
363 return false;
364}
365
366static const char *btf_int_encoding_str(u8 encoding)
367{
368 if (encoding == 0)
369 return "(none)";
370 else if (encoding == BTF_INT_SIGNED)
371 return "SIGNED";
372 else if (encoding == BTF_INT_CHAR)
373 return "CHAR";
374 else if (encoding == BTF_INT_BOOL)
375 return "BOOL";
376 else if (encoding == BTF_INT_VARARGS)
377 return "VARARGS";
378 else
379 return "UNKN";
380}
381
382static u16 btf_type_vlen(const struct btf_type *t)
383{
384 return BTF_INFO_VLEN(t->info);
385}
386
387static u32 btf_type_int(const struct btf_type *t)
388{
389 return *(u32 *)(t + 1);
390}
391
392static const struct btf_array *btf_type_array(const struct btf_type *t)
393{
394 return (const struct btf_array *)(t + 1);
395}
396
397static const struct btf_member *btf_type_member(const struct btf_type *t)
398{
399 return (const struct btf_member *)(t + 1);
400}
401
402static const struct btf_enum *btf_type_enum(const struct btf_type *t)
403{
404 return (const struct btf_enum *)(t + 1);
405}
406
407static const struct btf_kind_operations *btf_type_ops(const struct btf_type *t)
408{
409 return kind_ops[BTF_INFO_KIND(t->info)];
410}
411
412static bool btf_name_offset_valid(const struct btf *btf, u32 offset)
413{
414 return !BTF_STR_TBL_ELF_ID(offset) &&
415 BTF_STR_OFFSET(offset) < btf->hdr->str_len;
416}
417
418static const char *btf_name_by_offset(const struct btf *btf, u32 offset)
419{
420 if (!BTF_STR_OFFSET(offset))
421 return "(anon)";
422 else if (BTF_STR_OFFSET(offset) < btf->hdr->str_len)
423 return &btf->strings[BTF_STR_OFFSET(offset)];
424 else
425 return "(invalid-name-offset)";
426}
427
428static const struct btf_type *btf_type_by_id(const struct btf *btf, u32 type_id)
429{
430 if (type_id > btf->nr_types)
431 return NULL;
432
433 return btf->types[type_id];
434}
435
436__printf(2, 3) static void __btf_verifier_log(struct bpf_verifier_log *log,
437 const char *fmt, ...)
438{
439 va_list args;
440
441 va_start(args, fmt);
442 bpf_verifier_vlog(log, fmt, args);
443 va_end(args);
444}
445
446__printf(2, 3) static void btf_verifier_log(struct btf_verifier_env *env,
447 const char *fmt, ...)
448{
449 struct bpf_verifier_log *log = &env->log;
450 va_list args;
451
452 if (!bpf_verifier_log_needed(log))
453 return;
454
455 va_start(args, fmt);
456 bpf_verifier_vlog(log, fmt, args);
457 va_end(args);
458}
459
460__printf(4, 5) static void __btf_verifier_log_type(struct btf_verifier_env *env,
461 const struct btf_type *t,
462 bool log_details,
463 const char *fmt, ...)
464{
465 struct bpf_verifier_log *log = &env->log;
466 u8 kind = BTF_INFO_KIND(t->info);
467 struct btf *btf = env->btf;
468 va_list args;
469
470 if (!bpf_verifier_log_needed(log))
471 return;
472
473 __btf_verifier_log(log, "[%u] %s %s%s",
474 env->log_type_id,
475 btf_kind_str[kind],
476 btf_name_by_offset(btf, t->name),
477 log_details ? " " : "");
478
479 if (log_details)
480 btf_type_ops(t)->log_details(env, t);
481
482 if (fmt && *fmt) {
483 __btf_verifier_log(log, " ");
484 va_start(args, fmt);
485 bpf_verifier_vlog(log, fmt, args);
486 va_end(args);
487 }
488
489 __btf_verifier_log(log, "\n");
490}
491
492#define btf_verifier_log_type(env, t, ...) \
493 __btf_verifier_log_type((env), (t), true, __VA_ARGS__)
494#define btf_verifier_log_basic(env, t, ...) \
495 __btf_verifier_log_type((env), (t), false, __VA_ARGS__)
496
497__printf(4, 5)
498static void btf_verifier_log_member(struct btf_verifier_env *env,
499 const struct btf_type *struct_type,
500 const struct btf_member *member,
501 const char *fmt, ...)
502{
503 struct bpf_verifier_log *log = &env->log;
504 struct btf *btf = env->btf;
505 va_list args;
506
507 if (!bpf_verifier_log_needed(log))
508 return;
509
510 /* The CHECK_META phase already did a btf dump.
511 *
512 * If member is logged again, it must hit an error in
513 * parsing this member. It is useful to print out which
514 * struct this member belongs to.
515 */
516 if (env->phase != CHECK_META)
517 btf_verifier_log_type(env, struct_type, NULL);
518
519 __btf_verifier_log(log, "\t%s type_id=%u bits_offset=%u",
520 btf_name_by_offset(btf, member->name),
521 member->type, member->offset);
522
523 if (fmt && *fmt) {
524 __btf_verifier_log(log, " ");
525 va_start(args, fmt);
526 bpf_verifier_vlog(log, fmt, args);
527 va_end(args);
528 }
529
530 __btf_verifier_log(log, "\n");
531}
532
533static void btf_verifier_log_hdr(struct btf_verifier_env *env)
534{
535 struct bpf_verifier_log *log = &env->log;
536 const struct btf *btf = env->btf;
537 const struct btf_header *hdr;
538
539 if (!bpf_verifier_log_needed(log))
540 return;
541
542 hdr = btf->hdr;
543 __btf_verifier_log(log, "magic: 0x%x\n", hdr->magic);
544 __btf_verifier_log(log, "version: %u\n", hdr->version);
545 __btf_verifier_log(log, "flags: 0x%x\n", hdr->flags);
546 __btf_verifier_log(log, "parent_label: %u\n", hdr->parent_label);
547 __btf_verifier_log(log, "parent_name: %u\n", hdr->parent_name);
548 __btf_verifier_log(log, "label_off: %u\n", hdr->label_off);
549 __btf_verifier_log(log, "object_off: %u\n", hdr->object_off);
550 __btf_verifier_log(log, "func_off: %u\n", hdr->func_off);
551 __btf_verifier_log(log, "type_off: %u\n", hdr->type_off);
552 __btf_verifier_log(log, "str_off: %u\n", hdr->str_off);
553 __btf_verifier_log(log, "str_len: %u\n", hdr->str_len);
554 __btf_verifier_log(log, "btf_total_size: %u\n", btf->data_size);
555}
556
557static int btf_add_type(struct btf_verifier_env *env, struct btf_type *t)
558{
559 struct btf *btf = env->btf;
560
561 /* < 2 because +1 for btf_void which is always in btf->types[0].
562 * btf_void is not accounted in btf->nr_types because btf_void
563 * does not come from the BTF file.
564 */
565 if (btf->types_size - btf->nr_types < 2) {
566 /* Expand 'types' array */
567
568 struct btf_type **new_types;
569 u32 expand_by, new_size;
570
571 if (btf->types_size == BTF_MAX_NR_TYPES) {
572 btf_verifier_log(env, "Exceeded max num of types");
573 return -E2BIG;
574 }
575
576 expand_by = max_t(u32, btf->types_size >> 2, 16);
577 new_size = min_t(u32, BTF_MAX_NR_TYPES,
578 btf->types_size + expand_by);
579
580 new_types = kvzalloc(new_size * sizeof(*new_types),
581 GFP_KERNEL | __GFP_NOWARN);
582 if (!new_types)
583 return -ENOMEM;
584
585 if (btf->nr_types == 0)
586 new_types[0] = &btf_void;
587 else
588 memcpy(new_types, btf->types,
589 sizeof(*btf->types) * (btf->nr_types + 1));
590
591 kvfree(btf->types);
592 btf->types = new_types;
593 btf->types_size = new_size;
594 }
595
596 btf->types[++(btf->nr_types)] = t;
597
598 return 0;
599}
600
601static void btf_free(struct btf *btf)
602{
603 kvfree(btf->types);
604 kvfree(btf->resolved_sizes);
605 kvfree(btf->resolved_ids);
606 kvfree(btf->data);
607 kfree(btf);
608}
609
610static void btf_get(struct btf *btf)
611{
612 refcount_inc(&btf->refcnt);
613}
614
615void btf_put(struct btf *btf)
616{
617 if (btf && refcount_dec_and_test(&btf->refcnt))
618 btf_free(btf);
619}
620
621static int env_resolve_init(struct btf_verifier_env *env)
622{
623 struct btf *btf = env->btf;
624 u32 nr_types = btf->nr_types;
625 u32 *resolved_sizes = NULL;
626 u32 *resolved_ids = NULL;
627 u8 *visit_states = NULL;
628
629 /* +1 for btf_void */
630 resolved_sizes = kvzalloc((nr_types + 1) * sizeof(*resolved_sizes),
631 GFP_KERNEL | __GFP_NOWARN);
632 if (!resolved_sizes)
633 goto nomem;
634
635 resolved_ids = kvzalloc((nr_types + 1) * sizeof(*resolved_ids),
636 GFP_KERNEL | __GFP_NOWARN);
637 if (!resolved_ids)
638 goto nomem;
639
640 visit_states = kvzalloc((nr_types + 1) * sizeof(*visit_states),
641 GFP_KERNEL | __GFP_NOWARN);
642 if (!visit_states)
643 goto nomem;
644
645 btf->resolved_sizes = resolved_sizes;
646 btf->resolved_ids = resolved_ids;
647 env->visit_states = visit_states;
648
649 return 0;
650
651nomem:
652 kvfree(resolved_sizes);
653 kvfree(resolved_ids);
654 kvfree(visit_states);
655 return -ENOMEM;
656}
657
658static void btf_verifier_env_free(struct btf_verifier_env *env)
659{
660 kvfree(env->visit_states);
661 kfree(env);
662}
663
664static bool env_type_is_resolve_sink(const struct btf_verifier_env *env,
665 const struct btf_type *next_type)
666{
667 switch (env->resolve_mode) {
668 case RESOLVE_TBD:
669 /* int, enum or void is a sink */
670 return !btf_type_needs_resolve(next_type);
671 case RESOLVE_PTR:
672 /* int, enum, void, struct or array is a sink for ptr */
673 return !btf_type_is_modifier(next_type) &&
674 !btf_type_is_ptr(next_type);
675 case RESOLVE_STRUCT_OR_ARRAY:
676 /* int, enum, void or ptr is a sink for struct and array */
677 return !btf_type_is_modifier(next_type) &&
678 !btf_type_is_array(next_type) &&
679 !btf_type_is_struct(next_type);
680 default:
681 BUG_ON(1);
682 }
683}
684
685static bool env_type_is_resolved(const struct btf_verifier_env *env,
686 u32 type_id)
687{
688 return env->visit_states[type_id] == RESOLVED;
689}
690
691static int env_stack_push(struct btf_verifier_env *env,
692 const struct btf_type *t, u32 type_id)
693{
694 struct resolve_vertex *v;
695
696 if (env->top_stack == MAX_RESOLVE_DEPTH)
697 return -E2BIG;
698
699 if (env->visit_states[type_id] != NOT_VISITED)
700 return -EEXIST;
701
702 env->visit_states[type_id] = VISITED;
703
704 v = &env->stack[env->top_stack++];
705 v->t = t;
706 v->type_id = type_id;
707 v->next_member = 0;
708
709 if (env->resolve_mode == RESOLVE_TBD) {
710 if (btf_type_is_ptr(t))
711 env->resolve_mode = RESOLVE_PTR;
712 else if (btf_type_is_struct(t) || btf_type_is_array(t))
713 env->resolve_mode = RESOLVE_STRUCT_OR_ARRAY;
714 }
715
716 return 0;
717}
718
719static void env_stack_set_next_member(struct btf_verifier_env *env,
720 u16 next_member)
721{
722 env->stack[env->top_stack - 1].next_member = next_member;
723}
724
725static void env_stack_pop_resolved(struct btf_verifier_env *env,
726 u32 resolved_type_id,
727 u32 resolved_size)
728{
729 u32 type_id = env->stack[--(env->top_stack)].type_id;
730 struct btf *btf = env->btf;
731
732 btf->resolved_sizes[type_id] = resolved_size;
733 btf->resolved_ids[type_id] = resolved_type_id;
734 env->visit_states[type_id] = RESOLVED;
735}
736
737static const struct resolve_vertex *env_stack_peak(struct btf_verifier_env *env)
738{
739 return env->top_stack ? &env->stack[env->top_stack - 1] : NULL;
740}
741
742/* The input param "type_id" must point to a needs_resolve type */
743static const struct btf_type *btf_type_id_resolve(const struct btf *btf,
744 u32 *type_id)
745{
746 *type_id = btf->resolved_ids[*type_id];
747 return btf_type_by_id(btf, *type_id);
748}
749
750const struct btf_type *btf_type_id_size(const struct btf *btf,
751 u32 *type_id, u32 *ret_size)
752{
753 const struct btf_type *size_type;
754 u32 size_type_id = *type_id;
755 u32 size = 0;
756
757 size_type = btf_type_by_id(btf, size_type_id);
758 if (btf_type_is_void_or_null(size_type))
759 return NULL;
760
761 if (btf_type_has_size(size_type)) {
762 size = size_type->size;
763 } else if (btf_type_is_array(size_type)) {
764 size = btf->resolved_sizes[size_type_id];
765 } else if (btf_type_is_ptr(size_type)) {
766 size = sizeof(void *);
767 } else {
768 if (WARN_ON_ONCE(!btf_type_is_modifier(size_type)))
769 return NULL;
770
771 size = btf->resolved_sizes[size_type_id];
772 size_type_id = btf->resolved_ids[size_type_id];
773 size_type = btf_type_by_id(btf, size_type_id);
774 if (btf_type_is_void(size_type))
775 return NULL;
776 }
777
778 *type_id = size_type_id;
779 if (ret_size)
780 *ret_size = size;
781
782 return size_type;
783}
784
785static int btf_df_check_member(struct btf_verifier_env *env,
786 const struct btf_type *struct_type,
787 const struct btf_member *member,
788 const struct btf_type *member_type)
789{
790 btf_verifier_log_basic(env, struct_type,
791 "Unsupported check_member");
792 return -EINVAL;
793}
794
795static int btf_df_resolve(struct btf_verifier_env *env,
796 const struct resolve_vertex *v)
797{
798 btf_verifier_log_basic(env, v->t, "Unsupported resolve");
799 return -EINVAL;
800}
801
802static void btf_df_seq_show(const struct btf *btf, const struct btf_type *t,
803 u32 type_id, void *data, u8 bits_offsets,
804 struct seq_file *m)
805{
806 seq_printf(m, "<unsupported kind:%u>", BTF_INFO_KIND(t->info));
807}
808
809static int btf_int_check_member(struct btf_verifier_env *env,
810 const struct btf_type *struct_type,
811 const struct btf_member *member,
812 const struct btf_type *member_type)
813{
814 u32 int_data = btf_type_int(member_type);
815 u32 struct_bits_off = member->offset;
816 u32 struct_size = struct_type->size;
817 u32 nr_copy_bits;
818 u32 bytes_offset;
819
820 if (U32_MAX - struct_bits_off < BTF_INT_OFFSET(int_data)) {
821 btf_verifier_log_member(env, struct_type, member,
822 "bits_offset exceeds U32_MAX");
823 return -EINVAL;
824 }
825
826 struct_bits_off += BTF_INT_OFFSET(int_data);
827 bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off);
828 nr_copy_bits = BTF_INT_BITS(int_data) +
829 BITS_PER_BYTE_MASKED(struct_bits_off);
830
831 if (nr_copy_bits > BITS_PER_U64) {
832 btf_verifier_log_member(env, struct_type, member,
833 "nr_copy_bits exceeds 64");
834 return -EINVAL;
835 }
836
837 if (struct_size < bytes_offset ||
838 struct_size - bytes_offset < BITS_ROUNDUP_BYTES(nr_copy_bits)) {
839 btf_verifier_log_member(env, struct_type, member,
840 "Member exceeds struct_size");
841 return -EINVAL;
842 }
843
844 return 0;
845}
846
847static s32 btf_int_check_meta(struct btf_verifier_env *env,
848 const struct btf_type *t,
849 u32 meta_left)
850{
851 u32 int_data, nr_bits, meta_needed = sizeof(int_data);
852 u16 encoding;
853
854 if (meta_left < meta_needed) {
855 btf_verifier_log_basic(env, t,
856 "meta_left:%u meta_needed:%u",
857 meta_left, meta_needed);
858 return -EINVAL;
859 }
860
861 if (btf_type_vlen(t)) {
862 btf_verifier_log_type(env, t, "vlen != 0");
863 return -EINVAL;
864 }
865
866 int_data = btf_type_int(t);
867 nr_bits = BTF_INT_BITS(int_data) + BTF_INT_OFFSET(int_data);
868
869 if (nr_bits > BITS_PER_U64) {
870 btf_verifier_log_type(env, t, "nr_bits exceeds %zu",
871 BITS_PER_U64);
872 return -EINVAL;
873 }
874
875 if (BITS_ROUNDUP_BYTES(nr_bits) > t->size) {
876 btf_verifier_log_type(env, t, "nr_bits exceeds type_size");
877 return -EINVAL;
878 }
879
880 encoding = BTF_INT_ENCODING(int_data);
881 if (encoding &&
882 encoding != BTF_INT_SIGNED &&
883 encoding != BTF_INT_CHAR &&
884 encoding != BTF_INT_BOOL &&
885 encoding != BTF_INT_VARARGS) {
886 btf_verifier_log_type(env, t, "Unsupported encoding");
887 return -ENOTSUPP;
888 }
889
890 btf_verifier_log_type(env, t, NULL);
891
892 return meta_needed;
893}
894
895static void btf_int_log(struct btf_verifier_env *env,
896 const struct btf_type *t)
897{
898 int int_data = btf_type_int(t);
899
900 btf_verifier_log(env,
901 "size=%u bits_offset=%u nr_bits=%u encoding=%s",
902 t->size, BTF_INT_OFFSET(int_data),
903 BTF_INT_BITS(int_data),
904 btf_int_encoding_str(BTF_INT_ENCODING(int_data)));
905}
906
907static void btf_int_bits_seq_show(const struct btf *btf,
908 const struct btf_type *t,
909 void *data, u8 bits_offset,
910 struct seq_file *m)
911{
912 u32 int_data = btf_type_int(t);
913 u16 nr_bits = BTF_INT_BITS(int_data);
914 u16 total_bits_offset;
915 u16 nr_copy_bytes;
916 u16 nr_copy_bits;
917 u8 nr_upper_bits;
918 union {
919 u64 u64_num;
920 u8 u8_nums[8];
921 } print_num;
922
923 total_bits_offset = bits_offset + BTF_INT_OFFSET(int_data);
924 data += BITS_ROUNDDOWN_BYTES(total_bits_offset);
925 bits_offset = BITS_PER_BYTE_MASKED(total_bits_offset);
926 nr_copy_bits = nr_bits + bits_offset;
927 nr_copy_bytes = BITS_ROUNDUP_BYTES(nr_copy_bits);
928
929 print_num.u64_num = 0;
930 memcpy(&print_num.u64_num, data, nr_copy_bytes);
931
932 /* Ditch the higher order bits */
933 nr_upper_bits = BITS_PER_BYTE_MASKED(nr_copy_bits);
934 if (nr_upper_bits) {
935 /* We need to mask out some bits of the upper byte. */
936 u8 mask = (1 << nr_upper_bits) - 1;
937
938 print_num.u8_nums[nr_copy_bytes - 1] &= mask;
939 }
940
941 print_num.u64_num >>= bits_offset;
942
943 seq_printf(m, "0x%llx", print_num.u64_num);
944}
945
946static void btf_int_seq_show(const struct btf *btf, const struct btf_type *t,
947 u32 type_id, void *data, u8 bits_offset,
948 struct seq_file *m)
949{
950 u32 int_data = btf_type_int(t);
951 u8 encoding = BTF_INT_ENCODING(int_data);
952 bool sign = encoding & BTF_INT_SIGNED;
953 u32 nr_bits = BTF_INT_BITS(int_data);
954
955 if (bits_offset || BTF_INT_OFFSET(int_data) ||
956 BITS_PER_BYTE_MASKED(nr_bits)) {
957 btf_int_bits_seq_show(btf, t, data, bits_offset, m);
958 return;
959 }
960
961 switch (nr_bits) {
962 case 64:
963 if (sign)
964 seq_printf(m, "%lld", *(s64 *)data);
965 else
966 seq_printf(m, "%llu", *(u64 *)data);
967 break;
968 case 32:
969 if (sign)
970 seq_printf(m, "%d", *(s32 *)data);
971 else
972 seq_printf(m, "%u", *(u32 *)data);
973 break;
974 case 16:
975 if (sign)
976 seq_printf(m, "%d", *(s16 *)data);
977 else
978 seq_printf(m, "%u", *(u16 *)data);
979 break;
980 case 8:
981 if (sign)
982 seq_printf(m, "%d", *(s8 *)data);
983 else
984 seq_printf(m, "%u", *(u8 *)data);
985 break;
986 default:
987 btf_int_bits_seq_show(btf, t, data, bits_offset, m);
988 }
989}
990
991static const struct btf_kind_operations int_ops = {
992 .check_meta = btf_int_check_meta,
993 .resolve = btf_df_resolve,
994 .check_member = btf_int_check_member,
995 .log_details = btf_int_log,
996 .seq_show = btf_int_seq_show,
997};
998
999static int btf_modifier_check_member(struct btf_verifier_env *env,
1000 const struct btf_type *struct_type,
1001 const struct btf_member *member,
1002 const struct btf_type *member_type)
1003{
1004 const struct btf_type *resolved_type;
1005 u32 resolved_type_id = member->type;
1006 struct btf_member resolved_member;
1007 struct btf *btf = env->btf;
1008
1009 resolved_type = btf_type_id_size(btf, &resolved_type_id, NULL);
1010 if (!resolved_type) {
1011 btf_verifier_log_member(env, struct_type, member,
1012 "Invalid member");
1013 return -EINVAL;
1014 }
1015
1016 resolved_member = *member;
1017 resolved_member.type = resolved_type_id;
1018
1019 return btf_type_ops(resolved_type)->check_member(env, struct_type,
1020 &resolved_member,
1021 resolved_type);
1022}
1023
1024static int btf_ptr_check_member(struct btf_verifier_env *env,
1025 const struct btf_type *struct_type,
1026 const struct btf_member *member,
1027 const struct btf_type *member_type)
1028{
1029 u32 struct_size, struct_bits_off, bytes_offset;
1030
1031 struct_size = struct_type->size;
1032 struct_bits_off = member->offset;
1033 bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off);
1034
1035 if (BITS_PER_BYTE_MASKED(struct_bits_off)) {
1036 btf_verifier_log_member(env, struct_type, member,
1037 "Member is not byte aligned");
1038 return -EINVAL;
1039 }
1040
1041 if (struct_size - bytes_offset < sizeof(void *)) {
1042 btf_verifier_log_member(env, struct_type, member,
1043 "Member exceeds struct_size");
1044 return -EINVAL;
1045 }
1046
1047 return 0;
1048}
1049
1050static int btf_ref_type_check_meta(struct btf_verifier_env *env,
1051 const struct btf_type *t,
1052 u32 meta_left)
1053{
1054 if (btf_type_vlen(t)) {
1055 btf_verifier_log_type(env, t, "vlen != 0");
1056 return -EINVAL;
1057 }
1058
1059 if (BTF_TYPE_PARENT(t->type)) {
1060 btf_verifier_log_type(env, t, "Invalid type_id");
1061 return -EINVAL;
1062 }
1063
1064 btf_verifier_log_type(env, t, NULL);
1065
1066 return 0;
1067}
1068
1069static int btf_modifier_resolve(struct btf_verifier_env *env,
1070 const struct resolve_vertex *v)
1071{
1072 const struct btf_type *t = v->t;
1073 const struct btf_type *next_type;
1074 u32 next_type_id = t->type;
1075 struct btf *btf = env->btf;
1076 u32 next_type_size = 0;
1077
1078 next_type = btf_type_by_id(btf, next_type_id);
1079 if (!next_type) {
1080 btf_verifier_log_type(env, v->t, "Invalid type_id");
1081 return -EINVAL;
1082 }
1083
1084 /* "typedef void new_void", "const void"...etc */
1085 if (btf_type_is_void(next_type))
1086 goto resolved;
1087
1088 if (!env_type_is_resolve_sink(env, next_type) &&
1089 !env_type_is_resolved(env, next_type_id))
1090 return env_stack_push(env, next_type, next_type_id);
1091
1092 /* Figure out the resolved next_type_id with size.
1093 * They will be stored in the current modifier's
1094 * resolved_ids and resolved_sizes such that it can
1095 * save us a few type-following when we use it later (e.g. in
1096 * pretty print).
1097 */
1098 if (!btf_type_id_size(btf, &next_type_id, &next_type_size) &&
1099 !btf_type_is_void(btf_type_id_resolve(btf, &next_type_id))) {
1100 btf_verifier_log_type(env, v->t, "Invalid type_id");
1101 return -EINVAL;
1102 }
1103
1104resolved:
1105 env_stack_pop_resolved(env, next_type_id, next_type_size);
1106
1107 return 0;
1108}
1109
1110static int btf_ptr_resolve(struct btf_verifier_env *env,
1111 const struct resolve_vertex *v)
1112{
1113 const struct btf_type *next_type;
1114 const struct btf_type *t = v->t;
1115 u32 next_type_id = t->type;
1116 struct btf *btf = env->btf;
1117 u32 next_type_size = 0;
1118
1119 next_type = btf_type_by_id(btf, next_type_id);
1120 if (!next_type) {
1121 btf_verifier_log_type(env, v->t, "Invalid type_id");
1122 return -EINVAL;
1123 }
1124
1125 /* "void *" */
1126 if (btf_type_is_void(next_type))
1127 goto resolved;
1128
1129 if (!env_type_is_resolve_sink(env, next_type) &&
1130 !env_type_is_resolved(env, next_type_id))
1131 return env_stack_push(env, next_type, next_type_id);
1132
1133 /* If the modifier was RESOLVED during RESOLVE_STRUCT_OR_ARRAY,
1134 * the modifier may have stopped resolving when it was resolved
1135 * to a ptr (last-resolved-ptr).
1136 *
1137 * We now need to continue from the last-resolved-ptr to
1138 * ensure the last-resolved-ptr will not referring back to
1139 * the currenct ptr (t).
1140 */
1141 if (btf_type_is_modifier(next_type)) {
1142 const struct btf_type *resolved_type;
1143 u32 resolved_type_id;
1144
1145 resolved_type_id = next_type_id;
1146 resolved_type = btf_type_id_resolve(btf, &resolved_type_id);
1147
1148 if (btf_type_is_ptr(resolved_type) &&
1149 !env_type_is_resolve_sink(env, resolved_type) &&
1150 !env_type_is_resolved(env, resolved_type_id))
1151 return env_stack_push(env, resolved_type,
1152 resolved_type_id);
1153 }
1154
1155 if (!btf_type_id_size(btf, &next_type_id, &next_type_size) &&
1156 !btf_type_is_void(btf_type_id_resolve(btf, &next_type_id))) {
1157 btf_verifier_log_type(env, v->t, "Invalid type_id");
1158 return -EINVAL;
1159 }
1160
1161resolved:
1162 env_stack_pop_resolved(env, next_type_id, 0);
1163
1164 return 0;
1165}
1166
1167static void btf_modifier_seq_show(const struct btf *btf,
1168 const struct btf_type *t,
1169 u32 type_id, void *data,
1170 u8 bits_offset, struct seq_file *m)
1171{
1172 t = btf_type_id_resolve(btf, &type_id);
1173
1174 btf_type_ops(t)->seq_show(btf, t, type_id, data, bits_offset, m);
1175}
1176
1177static void btf_ptr_seq_show(const struct btf *btf, const struct btf_type *t,
1178 u32 type_id, void *data, u8 bits_offset,
1179 struct seq_file *m)
1180{
1181 /* It is a hashed value */
1182 seq_printf(m, "%p", *(void **)data);
1183}
1184
1185static void btf_ref_type_log(struct btf_verifier_env *env,
1186 const struct btf_type *t)
1187{
1188 btf_verifier_log(env, "type_id=%u", t->type);
1189}
1190
1191static struct btf_kind_operations modifier_ops = {
1192 .check_meta = btf_ref_type_check_meta,
1193 .resolve = btf_modifier_resolve,
1194 .check_member = btf_modifier_check_member,
1195 .log_details = btf_ref_type_log,
1196 .seq_show = btf_modifier_seq_show,
1197};
1198
1199static struct btf_kind_operations ptr_ops = {
1200 .check_meta = btf_ref_type_check_meta,
1201 .resolve = btf_ptr_resolve,
1202 .check_member = btf_ptr_check_member,
1203 .log_details = btf_ref_type_log,
1204 .seq_show = btf_ptr_seq_show,
1205};
1206
1207static struct btf_kind_operations fwd_ops = {
1208 .check_meta = btf_ref_type_check_meta,
1209 .resolve = btf_df_resolve,
1210 .check_member = btf_df_check_member,
1211 .log_details = btf_ref_type_log,
1212 .seq_show = btf_df_seq_show,
1213};
1214
1215static int btf_array_check_member(struct btf_verifier_env *env,
1216 const struct btf_type *struct_type,
1217 const struct btf_member *member,
1218 const struct btf_type *member_type)
1219{
1220 u32 struct_bits_off = member->offset;
1221 u32 struct_size, bytes_offset;
1222 u32 array_type_id, array_size;
1223 struct btf *btf = env->btf;
1224
1225 if (BITS_PER_BYTE_MASKED(struct_bits_off)) {
1226 btf_verifier_log_member(env, struct_type, member,
1227 "Member is not byte aligned");
1228 return -EINVAL;
1229 }
1230
1231 array_type_id = member->type;
1232 btf_type_id_size(btf, &array_type_id, &array_size);
1233 struct_size = struct_type->size;
1234 bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off);
1235 if (struct_size - bytes_offset < array_size) {
1236 btf_verifier_log_member(env, struct_type, member,
1237 "Member exceeds struct_size");
1238 return -EINVAL;
1239 }
1240
1241 return 0;
1242}
1243
1244static s32 btf_array_check_meta(struct btf_verifier_env *env,
1245 const struct btf_type *t,
1246 u32 meta_left)
1247{
1248 const struct btf_array *array = btf_type_array(t);
1249 u32 meta_needed = sizeof(*array);
1250
1251 if (meta_left < meta_needed) {
1252 btf_verifier_log_basic(env, t,
1253 "meta_left:%u meta_needed:%u",
1254 meta_left, meta_needed);
1255 return -EINVAL;
1256 }
1257
1258 if (btf_type_vlen(t)) {
1259 btf_verifier_log_type(env, t, "vlen != 0");
1260 return -EINVAL;
1261 }
1262
1263 /* We are a little forgiving on array->index_type since
1264 * the kernel is not using it.
1265 */
1266 /* Array elem cannot be in type void,
1267 * so !array->type is not allowed.
1268 */
1269 if (!array->type || BTF_TYPE_PARENT(array->type)) {
1270 btf_verifier_log_type(env, t, "Invalid type_id");
1271 return -EINVAL;
1272 }
1273
1274 btf_verifier_log_type(env, t, NULL);
1275
1276 return meta_needed;
1277}
1278
1279static int btf_array_resolve(struct btf_verifier_env *env,
1280 const struct resolve_vertex *v)
1281{
1282 const struct btf_array *array = btf_type_array(v->t);
1283 const struct btf_type *elem_type;
1284 u32 elem_type_id = array->type;
1285 struct btf *btf = env->btf;
1286 u32 elem_size;
1287
1288 elem_type = btf_type_by_id(btf, elem_type_id);
1289 if (btf_type_is_void_or_null(elem_type)) {
1290 btf_verifier_log_type(env, v->t,
1291 "Invalid elem");
1292 return -EINVAL;
1293 }
1294
1295 if (!env_type_is_resolve_sink(env, elem_type) &&
1296 !env_type_is_resolved(env, elem_type_id))
1297 return env_stack_push(env, elem_type, elem_type_id);
1298
1299 elem_type = btf_type_id_size(btf, &elem_type_id, &elem_size);
1300 if (!elem_type) {
1301 btf_verifier_log_type(env, v->t, "Invalid elem");
1302 return -EINVAL;
1303 }
1304
1305 if (btf_type_is_int(elem_type)) {
1306 int int_type_data = btf_type_int(elem_type);
1307 u16 nr_bits = BTF_INT_BITS(int_type_data);
1308 u16 nr_bytes = BITS_ROUNDUP_BYTES(nr_bits);
1309
1310 /* Put more restriction on array of int. The int cannot
1311 * be a bit field and it must be either u8/u16/u32/u64.
1312 */
1313 if (BITS_PER_BYTE_MASKED(nr_bits) ||
1314 BTF_INT_OFFSET(int_type_data) ||
1315 (nr_bytes != sizeof(u8) && nr_bytes != sizeof(u16) &&
1316 nr_bytes != sizeof(u32) && nr_bytes != sizeof(u64))) {
1317 btf_verifier_log_type(env, v->t,
1318 "Invalid array of int");
1319 return -EINVAL;
1320 }
1321 }
1322
1323 if (array->nelems && elem_size > U32_MAX / array->nelems) {
1324 btf_verifier_log_type(env, v->t,
1325 "Array size overflows U32_MAX");
1326 return -EINVAL;
1327 }
1328
1329 env_stack_pop_resolved(env, elem_type_id, elem_size * array->nelems);
1330
1331 return 0;
1332}
1333
1334static void btf_array_log(struct btf_verifier_env *env,
1335 const struct btf_type *t)
1336{
1337 const struct btf_array *array = btf_type_array(t);
1338
1339 btf_verifier_log(env, "type_id=%u index_type_id=%u nr_elems=%u",
1340 array->type, array->index_type, array->nelems);
1341}
1342
1343static void btf_array_seq_show(const struct btf *btf, const struct btf_type *t,
1344 u32 type_id, void *data, u8 bits_offset,
1345 struct seq_file *m)
1346{
1347 const struct btf_array *array = btf_type_array(t);
1348 const struct btf_kind_operations *elem_ops;
1349 const struct btf_type *elem_type;
1350 u32 i, elem_size, elem_type_id;
1351
1352 elem_type_id = array->type;
1353 elem_type = btf_type_id_size(btf, &elem_type_id, &elem_size);
1354 elem_ops = btf_type_ops(elem_type);
1355 seq_puts(m, "[");
1356 for (i = 0; i < array->nelems; i++) {
1357 if (i)
1358 seq_puts(m, ",");
1359
1360 elem_ops->seq_show(btf, elem_type, elem_type_id, data,
1361 bits_offset, m);
1362 data += elem_size;
1363 }
1364 seq_puts(m, "]");
1365}
1366
1367static struct btf_kind_operations array_ops = {
1368 .check_meta = btf_array_check_meta,
1369 .resolve = btf_array_resolve,
1370 .check_member = btf_array_check_member,
1371 .log_details = btf_array_log,
1372 .seq_show = btf_array_seq_show,
1373};
1374
1375static int btf_struct_check_member(struct btf_verifier_env *env,
1376 const struct btf_type *struct_type,
1377 const struct btf_member *member,
1378 const struct btf_type *member_type)
1379{
1380 u32 struct_bits_off = member->offset;
1381 u32 struct_size, bytes_offset;
1382
1383 if (BITS_PER_BYTE_MASKED(struct_bits_off)) {
1384 btf_verifier_log_member(env, struct_type, member,
1385 "Member is not byte aligned");
1386 return -EINVAL;
1387 }
1388
1389 struct_size = struct_type->size;
1390 bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off);
1391 if (struct_size - bytes_offset < member_type->size) {
1392 btf_verifier_log_member(env, struct_type, member,
1393 "Member exceeds struct_size");
1394 return -EINVAL;
1395 }
1396
1397 return 0;
1398}
1399
1400static s32 btf_struct_check_meta(struct btf_verifier_env *env,
1401 const struct btf_type *t,
1402 u32 meta_left)
1403{
1404 bool is_union = BTF_INFO_KIND(t->info) == BTF_KIND_UNION;
1405 const struct btf_member *member;
1406 struct btf *btf = env->btf;
1407 u32 struct_size = t->size;
1408 u32 meta_needed;
1409 u16 i;
1410
1411 meta_needed = btf_type_vlen(t) * sizeof(*member);
1412 if (meta_left < meta_needed) {
1413 btf_verifier_log_basic(env, t,
1414 "meta_left:%u meta_needed:%u",
1415 meta_left, meta_needed);
1416 return -EINVAL;
1417 }
1418
1419 btf_verifier_log_type(env, t, NULL);
1420
1421 for_each_member(i, t, member) {
1422 if (!btf_name_offset_valid(btf, member->name)) {
1423 btf_verifier_log_member(env, t, member,
1424 "Invalid member name_offset:%u",
1425 member->name);
1426 return -EINVAL;
1427 }
1428
1429 /* A member cannot be in type void */
1430 if (!member->type || BTF_TYPE_PARENT(member->type)) {
1431 btf_verifier_log_member(env, t, member,
1432 "Invalid type_id");
1433 return -EINVAL;
1434 }
1435
1436 if (is_union && member->offset) {
1437 btf_verifier_log_member(env, t, member,
1438 "Invalid member bits_offset");
1439 return -EINVAL;
1440 }
1441
1442 if (BITS_ROUNDUP_BYTES(member->offset) > struct_size) {
1443 btf_verifier_log_member(env, t, member,
1444 "Memmber bits_offset exceeds its struct size");
1445 return -EINVAL;
1446 }
1447
1448 btf_verifier_log_member(env, t, member, NULL);
1449 }
1450
1451 return meta_needed;
1452}
1453
1454static int btf_struct_resolve(struct btf_verifier_env *env,
1455 const struct resolve_vertex *v)
1456{
1457 const struct btf_member *member;
1458 int err;
1459 u16 i;
1460
1461 /* Before continue resolving the next_member,
1462 * ensure the last member is indeed resolved to a
1463 * type with size info.
1464 */
1465 if (v->next_member) {
1466 const struct btf_type *last_member_type;
1467 const struct btf_member *last_member;
1468 u16 last_member_type_id;
1469
1470 last_member = btf_type_member(v->t) + v->next_member - 1;
1471 last_member_type_id = last_member->type;
1472 if (WARN_ON_ONCE(!env_type_is_resolved(env,
1473 last_member_type_id)))
1474 return -EINVAL;
1475
1476 last_member_type = btf_type_by_id(env->btf,
1477 last_member_type_id);
1478 err = btf_type_ops(last_member_type)->check_member(env, v->t,
1479 last_member,
1480 last_member_type);
1481 if (err)
1482 return err;
1483 }
1484
1485 for_each_member_from(i, v->next_member, v->t, member) {
1486 u32 member_type_id = member->type;
1487 const struct btf_type *member_type = btf_type_by_id(env->btf,
1488 member_type_id);
1489
1490 if (btf_type_is_void_or_null(member_type)) {
1491 btf_verifier_log_member(env, v->t, member,
1492 "Invalid member");
1493 return -EINVAL;
1494 }
1495
1496 if (!env_type_is_resolve_sink(env, member_type) &&
1497 !env_type_is_resolved(env, member_type_id)) {
1498 env_stack_set_next_member(env, i + 1);
1499 return env_stack_push(env, member_type, member_type_id);
1500 }
1501
1502 err = btf_type_ops(member_type)->check_member(env, v->t,
1503 member,
1504 member_type);
1505 if (err)
1506 return err;
1507 }
1508
1509 env_stack_pop_resolved(env, 0, 0);
1510
1511 return 0;
1512}
1513
1514static void btf_struct_log(struct btf_verifier_env *env,
1515 const struct btf_type *t)
1516{
1517 btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t));
1518}
1519
1520static void btf_struct_seq_show(const struct btf *btf, const struct btf_type *t,
1521 u32 type_id, void *data, u8 bits_offset,
1522 struct seq_file *m)
1523{
1524 const char *seq = BTF_INFO_KIND(t->info) == BTF_KIND_UNION ? "|" : ",";
1525 const struct btf_member *member;
1526 u32 i;
1527
1528 seq_puts(m, "{");
1529 for_each_member(i, t, member) {
1530 const struct btf_type *member_type = btf_type_by_id(btf,
1531 member->type);
1532 u32 member_offset = member->offset;
1533 u32 bytes_offset = BITS_ROUNDDOWN_BYTES(member_offset);
1534 u8 bits8_offset = BITS_PER_BYTE_MASKED(member_offset);
1535 const struct btf_kind_operations *ops;
1536
1537 if (i)
1538 seq_puts(m, seq);
1539
1540 ops = btf_type_ops(member_type);
1541 ops->seq_show(btf, member_type, member->type,
1542 data + bytes_offset, bits8_offset, m);
1543 }
1544 seq_puts(m, "}");
1545}
1546
1547static struct btf_kind_operations struct_ops = {
1548 .check_meta = btf_struct_check_meta,
1549 .resolve = btf_struct_resolve,
1550 .check_member = btf_struct_check_member,
1551 .log_details = btf_struct_log,
1552 .seq_show = btf_struct_seq_show,
1553};
1554
1555static int btf_enum_check_member(struct btf_verifier_env *env,
1556 const struct btf_type *struct_type,
1557 const struct btf_member *member,
1558 const struct btf_type *member_type)
1559{
1560 u32 struct_bits_off = member->offset;
1561 u32 struct_size, bytes_offset;
1562
1563 if (BITS_PER_BYTE_MASKED(struct_bits_off)) {
1564 btf_verifier_log_member(env, struct_type, member,
1565 "Member is not byte aligned");
1566 return -EINVAL;
1567 }
1568
1569 struct_size = struct_type->size;
1570 bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off);
1571 if (struct_size - bytes_offset < sizeof(int)) {
1572 btf_verifier_log_member(env, struct_type, member,
1573 "Member exceeds struct_size");
1574 return -EINVAL;
1575 }
1576
1577 return 0;
1578}
1579
1580static s32 btf_enum_check_meta(struct btf_verifier_env *env,
1581 const struct btf_type *t,
1582 u32 meta_left)
1583{
1584 const struct btf_enum *enums = btf_type_enum(t);
1585 struct btf *btf = env->btf;
1586 u16 i, nr_enums;
1587 u32 meta_needed;
1588
1589 nr_enums = btf_type_vlen(t);
1590 meta_needed = nr_enums * sizeof(*enums);
1591
1592 if (meta_left < meta_needed) {
1593 btf_verifier_log_basic(env, t,
1594 "meta_left:%u meta_needed:%u",
1595 meta_left, meta_needed);
1596 return -EINVAL;
1597 }
1598
1599 if (t->size != sizeof(int)) {
1600 btf_verifier_log_type(env, t, "Expected size:%zu",
1601 sizeof(int));
1602 return -EINVAL;
1603 }
1604
1605 btf_verifier_log_type(env, t, NULL);
1606
1607 for (i = 0; i < nr_enums; i++) {
1608 if (!btf_name_offset_valid(btf, enums[i].name)) {
1609 btf_verifier_log(env, "\tInvalid name_offset:%u",
1610 enums[i].name);
1611 return -EINVAL;
1612 }
1613
1614 btf_verifier_log(env, "\t%s val=%d\n",
1615 btf_name_by_offset(btf, enums[i].name),
1616 enums[i].val);
1617 }
1618
1619 return meta_needed;
1620}
1621
1622static void btf_enum_log(struct btf_verifier_env *env,
1623 const struct btf_type *t)
1624{
1625 btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t));
1626}
1627
1628static void btf_enum_seq_show(const struct btf *btf, const struct btf_type *t,
1629 u32 type_id, void *data, u8 bits_offset,
1630 struct seq_file *m)
1631{
1632 const struct btf_enum *enums = btf_type_enum(t);
1633 u32 i, nr_enums = btf_type_vlen(t);
1634 int v = *(int *)data;
1635
1636 for (i = 0; i < nr_enums; i++) {
1637 if (v == enums[i].val) {
1638 seq_printf(m, "%s",
1639 btf_name_by_offset(btf, enums[i].name));
1640 return;
1641 }
1642 }
1643
1644 seq_printf(m, "%d", v);
1645}
1646
1647static struct btf_kind_operations enum_ops = {
1648 .check_meta = btf_enum_check_meta,
1649 .resolve = btf_df_resolve,
1650 .check_member = btf_enum_check_member,
1651 .log_details = btf_enum_log,
1652 .seq_show = btf_enum_seq_show,
1653};
1654
1655static const struct btf_kind_operations * const kind_ops[NR_BTF_KINDS] = {
1656 [BTF_KIND_INT] = &int_ops,
1657 [BTF_KIND_PTR] = &ptr_ops,
1658 [BTF_KIND_ARRAY] = &array_ops,
1659 [BTF_KIND_STRUCT] = &struct_ops,
1660 [BTF_KIND_UNION] = &struct_ops,
1661 [BTF_KIND_ENUM] = &enum_ops,
1662 [BTF_KIND_FWD] = &fwd_ops,
1663 [BTF_KIND_TYPEDEF] = &modifier_ops,
1664 [BTF_KIND_VOLATILE] = &modifier_ops,
1665 [BTF_KIND_CONST] = &modifier_ops,
1666 [BTF_KIND_RESTRICT] = &modifier_ops,
1667};
1668
1669static s32 btf_check_meta(struct btf_verifier_env *env,
1670 const struct btf_type *t,
1671 u32 meta_left)
1672{
1673 u32 saved_meta_left = meta_left;
1674 s32 var_meta_size;
1675
1676 if (meta_left < sizeof(*t)) {
1677 btf_verifier_log(env, "[%u] meta_left:%u meta_needed:%zu",
1678 env->log_type_id, meta_left, sizeof(*t));
1679 return -EINVAL;
1680 }
1681 meta_left -= sizeof(*t);
1682
1683 if (BTF_INFO_KIND(t->info) > BTF_KIND_MAX ||
1684 BTF_INFO_KIND(t->info) == BTF_KIND_UNKN) {
1685 btf_verifier_log(env, "[%u] Invalid kind:%u",
1686 env->log_type_id, BTF_INFO_KIND(t->info));
1687 return -EINVAL;
1688 }
1689
1690 if (!btf_name_offset_valid(env->btf, t->name)) {
1691 btf_verifier_log(env, "[%u] Invalid name_offset:%u",
1692 env->log_type_id, t->name);
1693 return -EINVAL;
1694 }
1695
1696 var_meta_size = btf_type_ops(t)->check_meta(env, t, meta_left);
1697 if (var_meta_size < 0)
1698 return var_meta_size;
1699
1700 meta_left -= var_meta_size;
1701
1702 return saved_meta_left - meta_left;
1703}
1704
1705static int btf_check_all_metas(struct btf_verifier_env *env)
1706{
1707 struct btf *btf = env->btf;
1708 struct btf_header *hdr;
1709 void *cur, *end;
1710
1711 hdr = btf->hdr;
1712 cur = btf->nohdr_data + hdr->type_off;
1713 end = btf->nohdr_data + hdr->str_off;
1714
1715 env->log_type_id = 1;
1716 while (cur < end) {
1717 struct btf_type *t = cur;
1718 s32 meta_size;
1719
1720 meta_size = btf_check_meta(env, t, end - cur);
1721 if (meta_size < 0)
1722 return meta_size;
1723
1724 btf_add_type(env, t);
1725 cur += meta_size;
1726 env->log_type_id++;
1727 }
1728
1729 return 0;
1730}
1731
1732static int btf_resolve(struct btf_verifier_env *env,
1733 const struct btf_type *t, u32 type_id)
1734{
1735 const struct resolve_vertex *v;
1736 int err = 0;
1737
1738 env->resolve_mode = RESOLVE_TBD;
1739 env_stack_push(env, t, type_id);
1740 while (!err && (v = env_stack_peak(env))) {
1741 env->log_type_id = v->type_id;
1742 err = btf_type_ops(v->t)->resolve(env, v);
1743 }
1744
1745 env->log_type_id = type_id;
1746 if (err == -E2BIG)
1747 btf_verifier_log_type(env, t,
1748 "Exceeded max resolving depth:%u",
1749 MAX_RESOLVE_DEPTH);
1750 else if (err == -EEXIST)
1751 btf_verifier_log_type(env, t, "Loop detected");
1752
1753 return err;
1754}
1755
1756static bool btf_resolve_valid(struct btf_verifier_env *env,
1757 const struct btf_type *t,
1758 u32 type_id)
1759{
1760 struct btf *btf = env->btf;
1761
1762 if (!env_type_is_resolved(env, type_id))
1763 return false;
1764
1765 if (btf_type_is_struct(t))
1766 return !btf->resolved_ids[type_id] &&
1767 !btf->resolved_sizes[type_id];
1768
1769 if (btf_type_is_modifier(t) || btf_type_is_ptr(t)) {
1770 t = btf_type_id_resolve(btf, &type_id);
1771 return t && !btf_type_is_modifier(t);
1772 }
1773
1774 if (btf_type_is_array(t)) {
1775 const struct btf_array *array = btf_type_array(t);
1776 const struct btf_type *elem_type;
1777 u32 elem_type_id = array->type;
1778 u32 elem_size;
1779
1780 elem_type = btf_type_id_size(btf, &elem_type_id, &elem_size);
1781 return elem_type && !btf_type_is_modifier(elem_type) &&
1782 (array->nelems * elem_size ==
1783 btf->resolved_sizes[type_id]);
1784 }
1785
1786 return false;
1787}
1788
1789static int btf_check_all_types(struct btf_verifier_env *env)
1790{
1791 struct btf *btf = env->btf;
1792 u32 type_id;
1793 int err;
1794
1795 err = env_resolve_init(env);
1796 if (err)
1797 return err;
1798
1799 env->phase++;
1800 for (type_id = 1; type_id <= btf->nr_types; type_id++) {
1801 const struct btf_type *t = btf_type_by_id(btf, type_id);
1802
1803 env->log_type_id = type_id;
1804 if (btf_type_needs_resolve(t) &&
1805 !env_type_is_resolved(env, type_id)) {
1806 err = btf_resolve(env, t, type_id);
1807 if (err)
1808 return err;
1809 }
1810
1811 if (btf_type_needs_resolve(t) &&
1812 !btf_resolve_valid(env, t, type_id)) {
1813 btf_verifier_log_type(env, t, "Invalid resolve state");
1814 return -EINVAL;
1815 }
1816 }
1817
1818 return 0;
1819}
1820
1821static int btf_parse_type_sec(struct btf_verifier_env *env)
1822{
1823 int err;
1824
1825 err = btf_check_all_metas(env);
1826 if (err)
1827 return err;
1828
1829 return btf_check_all_types(env);
1830}
1831
1832static int btf_parse_str_sec(struct btf_verifier_env *env)
1833{
1834 const struct btf_header *hdr;
1835 struct btf *btf = env->btf;
1836 const char *start, *end;
1837
1838 hdr = btf->hdr;
1839 start = btf->nohdr_data + hdr->str_off;
1840 end = start + hdr->str_len;
1841
1842 if (!hdr->str_len || hdr->str_len - 1 > BTF_MAX_NAME_OFFSET ||
1843 start[0] || end[-1]) {
1844 btf_verifier_log(env, "Invalid string section");
1845 return -EINVAL;
1846 }
1847
1848 btf->strings = start;
1849
1850 return 0;
1851}
1852
1853static int btf_parse_hdr(struct btf_verifier_env *env)
1854{
1855 const struct btf_header *hdr;
1856 struct btf *btf = env->btf;
1857 u32 meta_left;
1858
1859 if (btf->data_size < sizeof(*hdr)) {
1860 btf_verifier_log(env, "btf_header not found");
1861 return -EINVAL;
1862 }
1863
1864 btf_verifier_log_hdr(env);
1865
1866 hdr = btf->hdr;
1867 if (hdr->magic != BTF_MAGIC) {
1868 btf_verifier_log(env, "Invalid magic");
1869 return -EINVAL;
1870 }
1871
1872 if (hdr->version != BTF_VERSION) {
1873 btf_verifier_log(env, "Unsupported version");
1874 return -ENOTSUPP;
1875 }
1876
1877 if (hdr->flags) {
1878 btf_verifier_log(env, "Unsupported flags");
1879 return -ENOTSUPP;
1880 }
1881
1882 meta_left = btf->data_size - sizeof(*hdr);
1883 if (!meta_left) {
1884 btf_verifier_log(env, "No data");
1885 return -EINVAL;
1886 }
1887
1888 if (meta_left < hdr->type_off || hdr->str_off <= hdr->type_off ||
1889 /* Type section must align to 4 bytes */
1890 hdr->type_off & (sizeof(u32) - 1)) {
1891 btf_verifier_log(env, "Invalid type_off");
1892 return -EINVAL;
1893 }
1894
1895 if (meta_left < hdr->str_off ||
1896 meta_left - hdr->str_off < hdr->str_len) {
1897 btf_verifier_log(env, "Invalid str_off or str_len");
1898 return -EINVAL;
1899 }
1900
1901 btf->nohdr_data = btf->hdr + 1;
1902
1903 return 0;
1904}
1905
1906static struct btf *btf_parse(void __user *btf_data, u32 btf_data_size,
1907 u32 log_level, char __user *log_ubuf, u32 log_size)
1908{
1909 struct btf_verifier_env *env = NULL;
1910 struct bpf_verifier_log *log;
1911 struct btf *btf = NULL;
1912 u8 *data;
1913 int err;
1914
1915 if (btf_data_size > BTF_MAX_SIZE)
1916 return ERR_PTR(-E2BIG);
1917
1918 env = kzalloc(sizeof(*env), GFP_KERNEL | __GFP_NOWARN);
1919 if (!env)
1920 return ERR_PTR(-ENOMEM);
1921
1922 log = &env->log;
1923 if (log_level || log_ubuf || log_size) {
1924 /* user requested verbose verifier output
1925 * and supplied buffer to store the verification trace
1926 */
1927 log->level = log_level;
1928 log->ubuf = log_ubuf;
1929 log->len_total = log_size;
1930
1931 /* log attributes have to be sane */
1932 if (log->len_total < 128 || log->len_total > UINT_MAX >> 8 ||
1933 !log->level || !log->ubuf) {
1934 err = -EINVAL;
1935 goto errout;
1936 }
1937 }
1938
1939 btf = kzalloc(sizeof(*btf), GFP_KERNEL | __GFP_NOWARN);
1940 if (!btf) {
1941 err = -ENOMEM;
1942 goto errout;
1943 }
1944
1945 data = kvmalloc(btf_data_size, GFP_KERNEL | __GFP_NOWARN);
1946 if (!data) {
1947 err = -ENOMEM;
1948 goto errout;
1949 }
1950
1951 btf->data = data;
1952 btf->data_size = btf_data_size;
1953
1954 if (copy_from_user(data, btf_data, btf_data_size)) {
1955 err = -EFAULT;
1956 goto errout;
1957 }
1958
1959 env->btf = btf;
1960
1961 err = btf_parse_hdr(env);
1962 if (err)
1963 goto errout;
1964
1965 err = btf_parse_str_sec(env);
1966 if (err)
1967 goto errout;
1968
1969 err = btf_parse_type_sec(env);
1970 if (err)
1971 goto errout;
1972
1973 if (!err && log->level && bpf_verifier_log_full(log)) {
1974 err = -ENOSPC;
1975 goto errout;
1976 }
1977
1978 if (!err) {
1979 btf_verifier_env_free(env);
1980 btf_get(btf);
1981 return btf;
1982 }
1983
1984errout:
1985 btf_verifier_env_free(env);
1986 if (btf)
1987 btf_free(btf);
1988 return ERR_PTR(err);
1989}
1990
1991void btf_type_seq_show(const struct btf *btf, u32 type_id, void *obj,
1992 struct seq_file *m)
1993{
1994 const struct btf_type *t = btf_type_by_id(btf, type_id);
1995
1996 btf_type_ops(t)->seq_show(btf, t, type_id, obj, 0, m);
1997}
1998
1999static int btf_release(struct inode *inode, struct file *filp)
2000{
2001 btf_put(filp->private_data);
2002 return 0;
2003}
2004
2005const struct file_operations btf_fops = {
2006 .release = btf_release,
2007};
2008
2009int btf_new_fd(const union bpf_attr *attr)
2010{
2011 struct btf *btf;
2012 int fd;
2013
2014 btf = btf_parse(u64_to_user_ptr(attr->btf),
2015 attr->btf_size, attr->btf_log_level,
2016 u64_to_user_ptr(attr->btf_log_buf),
2017 attr->btf_log_size);
2018 if (IS_ERR(btf))
2019 return PTR_ERR(btf);
2020
2021 fd = anon_inode_getfd("btf", &btf_fops, btf,
2022 O_RDONLY | O_CLOEXEC);
2023 if (fd < 0)
2024 btf_put(btf);
2025
2026 return fd;
2027}
2028
2029struct btf *btf_get_by_fd(int fd)
2030{
2031 struct btf *btf;
2032 struct fd f;
2033
2034 f = fdget(fd);
2035
2036 if (!f.file)
2037 return ERR_PTR(-EBADF);
2038
2039 if (f.file->f_op != &btf_fops) {
2040 fdput(f);
2041 return ERR_PTR(-EINVAL);
2042 }
2043
2044 btf = f.file->private_data;
2045 btf_get(btf);
2046 fdput(f);
2047
2048 return btf;
2049}
2050
2051int btf_get_info_by_fd(const struct btf *btf,
2052 const union bpf_attr *attr,
2053 union bpf_attr __user *uattr)
2054{
2055 void __user *udata = u64_to_user_ptr(attr->info.info);
2056 u32 copy_len = min_t(u32, btf->data_size,
2057 attr->info.info_len);
2058
2059 if (copy_to_user(udata, btf->data, copy_len) ||
2060 put_user(btf->data_size, &uattr->info.info_len))
2061 return -EFAULT;
2062
2063 return 0;
2064}
diff --git a/kernel/bpf/inode.c b/kernel/bpf/inode.c
index bf6da59ae0d0..a41343009ccc 100644
--- a/kernel/bpf/inode.c
+++ b/kernel/bpf/inode.c
@@ -150,8 +150,154 @@ static int bpf_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
150 return 0; 150 return 0;
151} 151}
152 152
153struct map_iter {
154 void *key;
155 bool done;
156};
157
158static struct map_iter *map_iter(struct seq_file *m)
159{
160 return m->private;
161}
162
163static struct bpf_map *seq_file_to_map(struct seq_file *m)
164{
165 return file_inode(m->file)->i_private;
166}
167
168static void map_iter_free(struct map_iter *iter)
169{
170 if (iter) {
171 kfree(iter->key);
172 kfree(iter);
173 }
174}
175
176static struct map_iter *map_iter_alloc(struct bpf_map *map)
177{
178 struct map_iter *iter;
179
180 iter = kzalloc(sizeof(*iter), GFP_KERNEL | __GFP_NOWARN);
181 if (!iter)
182 goto error;
183
184 iter->key = kzalloc(map->key_size, GFP_KERNEL | __GFP_NOWARN);
185 if (!iter->key)
186 goto error;
187
188 return iter;
189
190error:
191 map_iter_free(iter);
192 return NULL;
193}
194
195static void *map_seq_next(struct seq_file *m, void *v, loff_t *pos)
196{
197 struct bpf_map *map = seq_file_to_map(m);
198 void *key = map_iter(m)->key;
199
200 if (map_iter(m)->done)
201 return NULL;
202
203 if (unlikely(v == SEQ_START_TOKEN))
204 goto done;
205
206 if (map->ops->map_get_next_key(map, key, key)) {
207 map_iter(m)->done = true;
208 return NULL;
209 }
210
211done:
212 ++(*pos);
213 return key;
214}
215
216static void *map_seq_start(struct seq_file *m, loff_t *pos)
217{
218 if (map_iter(m)->done)
219 return NULL;
220
221 return *pos ? map_iter(m)->key : SEQ_START_TOKEN;
222}
223
224static void map_seq_stop(struct seq_file *m, void *v)
225{
226}
227
228static int map_seq_show(struct seq_file *m, void *v)
229{
230 struct bpf_map *map = seq_file_to_map(m);
231 void *key = map_iter(m)->key;
232
233 if (unlikely(v == SEQ_START_TOKEN)) {
234 seq_puts(m, "# WARNING!! The output is for debug purpose only\n");
235 seq_puts(m, "# WARNING!! The output format will change\n");
236 } else {
237 map->ops->map_seq_show_elem(map, key, m);
238 }
239
240 return 0;
241}
242
243static const struct seq_operations bpffs_map_seq_ops = {
244 .start = map_seq_start,
245 .next = map_seq_next,
246 .show = map_seq_show,
247 .stop = map_seq_stop,
248};
249
250static int bpffs_map_open(struct inode *inode, struct file *file)
251{
252 struct bpf_map *map = inode->i_private;
253 struct map_iter *iter;
254 struct seq_file *m;
255 int err;
256
257 iter = map_iter_alloc(map);
258 if (!iter)
259 return -ENOMEM;
260
261 err = seq_open(file, &bpffs_map_seq_ops);
262 if (err) {
263 map_iter_free(iter);
264 return err;
265 }
266
267 m = file->private_data;
268 m->private = iter;
269
270 return 0;
271}
272
273static int bpffs_map_release(struct inode *inode, struct file *file)
274{
275 struct seq_file *m = file->private_data;
276
277 map_iter_free(map_iter(m));
278
279 return seq_release(inode, file);
280}
281
282/* bpffs_map_fops should only implement the basic
283 * read operation for a BPF map. The purpose is to
284 * provide a simple user intuitive way to do
285 * "cat bpffs/pathto/a-pinned-map".
286 *
287 * Other operations (e.g. write, lookup...) should be realized by
288 * the userspace tools (e.g. bpftool) through the
289 * BPF_OBJ_GET_INFO_BY_FD and the map's lookup/update
290 * interface.
291 */
292static const struct file_operations bpffs_map_fops = {
293 .open = bpffs_map_open,
294 .read = seq_read,
295 .release = bpffs_map_release,
296};
297
153static int bpf_mkobj_ops(struct dentry *dentry, umode_t mode, void *raw, 298static int bpf_mkobj_ops(struct dentry *dentry, umode_t mode, void *raw,
154 const struct inode_operations *iops) 299 const struct inode_operations *iops,
300 const struct file_operations *fops)
155{ 301{
156 struct inode *dir = dentry->d_parent->d_inode; 302 struct inode *dir = dentry->d_parent->d_inode;
157 struct inode *inode = bpf_get_inode(dir->i_sb, dir, mode); 303 struct inode *inode = bpf_get_inode(dir->i_sb, dir, mode);
@@ -159,6 +305,7 @@ static int bpf_mkobj_ops(struct dentry *dentry, umode_t mode, void *raw,
159 return PTR_ERR(inode); 305 return PTR_ERR(inode);
160 306
161 inode->i_op = iops; 307 inode->i_op = iops;
308 inode->i_fop = fops;
162 inode->i_private = raw; 309 inode->i_private = raw;
163 310
164 bpf_dentry_finalize(dentry, inode, dir); 311 bpf_dentry_finalize(dentry, inode, dir);
@@ -167,12 +314,15 @@ static int bpf_mkobj_ops(struct dentry *dentry, umode_t mode, void *raw,
167 314
168static int bpf_mkprog(struct dentry *dentry, umode_t mode, void *arg) 315static int bpf_mkprog(struct dentry *dentry, umode_t mode, void *arg)
169{ 316{
170 return bpf_mkobj_ops(dentry, mode, arg, &bpf_prog_iops); 317 return bpf_mkobj_ops(dentry, mode, arg, &bpf_prog_iops, NULL);
171} 318}
172 319
173static int bpf_mkmap(struct dentry *dentry, umode_t mode, void *arg) 320static int bpf_mkmap(struct dentry *dentry, umode_t mode, void *arg)
174{ 321{
175 return bpf_mkobj_ops(dentry, mode, arg, &bpf_map_iops); 322 struct bpf_map *map = arg;
323
324 return bpf_mkobj_ops(dentry, mode, arg, &bpf_map_iops,
325 map->btf ? &bpffs_map_fops : NULL);
176} 326}
177 327
178static struct dentry * 328static struct dentry *
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 4ca46df19c9a..fe23dc5a3ec4 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -11,6 +11,7 @@
11 */ 11 */
12#include <linux/bpf.h> 12#include <linux/bpf.h>
13#include <linux/bpf_trace.h> 13#include <linux/bpf_trace.h>
14#include <linux/btf.h>
14#include <linux/syscalls.h> 15#include <linux/syscalls.h>
15#include <linux/slab.h> 16#include <linux/slab.h>
16#include <linux/sched/signal.h> 17#include <linux/sched/signal.h>
@@ -26,6 +27,7 @@
26#include <linux/cred.h> 27#include <linux/cred.h>
27#include <linux/timekeeping.h> 28#include <linux/timekeeping.h>
28#include <linux/ctype.h> 29#include <linux/ctype.h>
30#include <linux/btf.h>
29 31
30#define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY || \ 32#define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY || \
31 (map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \ 33 (map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \
@@ -250,6 +252,7 @@ static void bpf_map_free_deferred(struct work_struct *work)
250 252
251 bpf_map_uncharge_memlock(map); 253 bpf_map_uncharge_memlock(map);
252 security_bpf_map_free(map); 254 security_bpf_map_free(map);
255 btf_put(map->btf);
253 /* implementation dependent freeing */ 256 /* implementation dependent freeing */
254 map->ops->map_free(map); 257 map->ops->map_free(map);
255} 258}
@@ -415,7 +418,7 @@ static int bpf_obj_name_cpy(char *dst, const char *src)
415 return 0; 418 return 0;
416} 419}
417 420
418#define BPF_MAP_CREATE_LAST_FIELD map_ifindex 421#define BPF_MAP_CREATE_LAST_FIELD btf_value_id
419/* called via syscall */ 422/* called via syscall */
420static int map_create(union bpf_attr *attr) 423static int map_create(union bpf_attr *attr)
421{ 424{
@@ -449,6 +452,33 @@ static int map_create(union bpf_attr *attr)
449 atomic_set(&map->refcnt, 1); 452 atomic_set(&map->refcnt, 1);
450 atomic_set(&map->usercnt, 1); 453 atomic_set(&map->usercnt, 1);
451 454
455 if (bpf_map_support_seq_show(map) &&
456 (attr->btf_key_id || attr->btf_value_id)) {
457 struct btf *btf;
458
459 if (!attr->btf_key_id || !attr->btf_value_id) {
460 err = -EINVAL;
461 goto free_map_nouncharge;
462 }
463
464 btf = btf_get_by_fd(attr->btf_fd);
465 if (IS_ERR(btf)) {
466 err = PTR_ERR(btf);
467 goto free_map_nouncharge;
468 }
469
470 err = map->ops->map_check_btf(map, btf, attr->btf_key_id,
471 attr->btf_value_id);
472 if (err) {
473 btf_put(btf);
474 goto free_map_nouncharge;
475 }
476
477 map->btf = btf;
478 map->btf_key_id = attr->btf_key_id;
479 map->btf_value_id = attr->btf_value_id;
480 }
481
452 err = security_bpf_map_alloc(map); 482 err = security_bpf_map_alloc(map);
453 if (err) 483 if (err)
454 goto free_map_nouncharge; 484 goto free_map_nouncharge;
@@ -481,6 +511,7 @@ free_map:
481free_map_sec: 511free_map_sec:
482 security_bpf_map_free(map); 512 security_bpf_map_free(map);
483free_map_nouncharge: 513free_map_nouncharge:
514 btf_put(map->btf);
484 map->ops->map_free(map); 515 map->ops->map_free(map);
485 return err; 516 return err;
486} 517}
@@ -2016,6 +2047,8 @@ static int bpf_obj_get_info_by_fd(const union bpf_attr *attr,
2016 else if (f.file->f_op == &bpf_map_fops) 2047 else if (f.file->f_op == &bpf_map_fops)
2017 err = bpf_map_get_info_by_fd(f.file->private_data, attr, 2048 err = bpf_map_get_info_by_fd(f.file->private_data, attr,
2018 uattr); 2049 uattr);
2050 else if (f.file->f_op == &btf_fops)
2051 err = btf_get_info_by_fd(f.file->private_data, attr, uattr);
2019 else 2052 else
2020 err = -EINVAL; 2053 err = -EINVAL;
2021 2054
@@ -2023,6 +2056,19 @@ static int bpf_obj_get_info_by_fd(const union bpf_attr *attr,
2023 return err; 2056 return err;
2024} 2057}
2025 2058
2059#define BPF_BTF_LOAD_LAST_FIELD btf_log_level
2060
2061static int bpf_btf_load(const union bpf_attr *attr)
2062{
2063 if (CHECK_ATTR(BPF_BTF_LOAD))
2064 return -EINVAL;
2065
2066 if (!capable(CAP_SYS_ADMIN))
2067 return -EPERM;
2068
2069 return btf_new_fd(attr);
2070}
2071
2026SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size) 2072SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size)
2027{ 2073{
2028 union bpf_attr attr = {}; 2074 union bpf_attr attr = {};
@@ -2103,6 +2149,9 @@ SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, siz
2103 case BPF_RAW_TRACEPOINT_OPEN: 2149 case BPF_RAW_TRACEPOINT_OPEN:
2104 err = bpf_raw_tracepoint_open(&attr); 2150 err = bpf_raw_tracepoint_open(&attr);
2105 break; 2151 break;
2152 case BPF_BTF_LOAD:
2153 err = bpf_btf_load(&attr);
2154 break;
2106 default: 2155 default:
2107 err = -EINVAL; 2156 err = -EINVAL;
2108 break; 2157 break;
diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
index 56bf493ba7ed..7f7fbb9d0253 100644
--- a/tools/include/uapi/linux/bpf.h
+++ b/tools/include/uapi/linux/bpf.h
@@ -95,6 +95,7 @@ enum bpf_cmd {
95 BPF_OBJ_GET_INFO_BY_FD, 95 BPF_OBJ_GET_INFO_BY_FD,
96 BPF_PROG_QUERY, 96 BPF_PROG_QUERY,
97 BPF_RAW_TRACEPOINT_OPEN, 97 BPF_RAW_TRACEPOINT_OPEN,
98 BPF_BTF_LOAD,
98}; 99};
99 100
100enum bpf_map_type { 101enum bpf_map_type {
@@ -279,6 +280,9 @@ union bpf_attr {
279 */ 280 */
280 char map_name[BPF_OBJ_NAME_LEN]; 281 char map_name[BPF_OBJ_NAME_LEN];
281 __u32 map_ifindex; /* ifindex of netdev to create on */ 282 __u32 map_ifindex; /* ifindex of netdev to create on */
283 __u32 btf_fd; /* fd pointing to a BTF type data */
284 __u32 btf_key_id; /* BTF type_id of the key */
285 __u32 btf_value_id; /* BTF type_id of the value */
282 }; 286 };
283 287
284 struct { /* anonymous struct used by BPF_MAP_*_ELEM commands */ 288 struct { /* anonymous struct used by BPF_MAP_*_ELEM commands */
@@ -363,6 +367,14 @@ union bpf_attr {
363 __u64 name; 367 __u64 name;
364 __u32 prog_fd; 368 __u32 prog_fd;
365 } raw_tracepoint; 369 } raw_tracepoint;
370
371 struct { /* anonymous struct for BPF_BTF_LOAD */
372 __aligned_u64 btf;
373 __aligned_u64 btf_log_buf;
374 __u32 btf_size;
375 __u32 btf_log_size;
376 __u32 btf_log_level;
377 };
366} __attribute__((aligned(8))); 378} __attribute__((aligned(8)));
367 379
368/* BPF helper function descriptions: 380/* BPF helper function descriptions:
diff --git a/tools/include/uapi/linux/btf.h b/tools/include/uapi/linux/btf.h
new file mode 100644
index 000000000000..74a30b1090df
--- /dev/null
+++ b/tools/include/uapi/linux/btf.h
@@ -0,0 +1,130 @@
1/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
2/* Copyright (c) 2018 Facebook */
3#ifndef _UAPI__LINUX_BTF_H__
4#define _UAPI__LINUX_BTF_H__
5
6#include <linux/types.h>
7
8#define BTF_MAGIC 0xeB9F
9#define BTF_MAGIC_SWAP 0x9FeB
10#define BTF_VERSION 1
11#define BTF_FLAGS_COMPR 0x01
12
13struct btf_header {
14 __u16 magic;
15 __u8 version;
16 __u8 flags;
17
18 __u32 parent_label;
19 __u32 parent_name;
20
21 /* All offsets are in bytes relative to the end of this header */
22 __u32 label_off; /* offset of label section */
23 __u32 object_off; /* offset of data object section*/
24 __u32 func_off; /* offset of function section */
25 __u32 type_off; /* offset of type section */
26 __u32 str_off; /* offset of string section */
27 __u32 str_len; /* length of string section */
28};
29
30/* Max # of type identifier */
31#define BTF_MAX_TYPE 0x7fffffff
32/* Max offset into the string section */
33#define BTF_MAX_NAME_OFFSET 0x7fffffff
34/* Max # of struct/union/enum members or func args */
35#define BTF_MAX_VLEN 0xffff
36
37/* The type id is referring to a parent BTF */
38#define BTF_TYPE_PARENT(id) (((id) >> 31) & 0x1)
39#define BTF_TYPE_ID(id) ((id) & BTF_MAX_TYPE)
40
41/* String is in the ELF string section */
42#define BTF_STR_TBL_ELF_ID(ref) (((ref) >> 31) & 0x1)
43#define BTF_STR_OFFSET(ref) ((ref) & BTF_MAX_NAME_OFFSET)
44
45struct btf_type {
46 __u32 name;
47 /* "info" bits arrangement
48 * bits 0-15: vlen (e.g. # of struct's members)
49 * bits 16-23: unused
50 * bits 24-28: kind (e.g. int, ptr, array...etc)
51 * bits 29-30: unused
52 * bits 31: root
53 */
54 __u32 info;
55 /* "size" is used by INT, ENUM, STRUCT and UNION.
56 * "size" tells the size of the type it is describing.
57 *
58 * "type" is used by PTR, TYPEDEF, VOLATILE, CONST and RESTRICT.
59 * "type" is a type_id referring to another type.
60 */
61 union {
62 __u32 size;
63 __u32 type;
64 };
65};
66
67#define BTF_INFO_KIND(info) (((info) >> 24) & 0x1f)
68#define BTF_INFO_ISROOT(info) (!!(((info) >> 24) & 0x80))
69#define BTF_INFO_VLEN(info) ((info) & 0xffff)
70
71#define BTF_KIND_UNKN 0 /* Unknown */
72#define BTF_KIND_INT 1 /* Integer */
73#define BTF_KIND_PTR 2 /* Pointer */
74#define BTF_KIND_ARRAY 3 /* Array */
75#define BTF_KIND_STRUCT 4 /* Struct */
76#define BTF_KIND_UNION 5 /* Union */
77#define BTF_KIND_ENUM 6 /* Enumeration */
78#define BTF_KIND_FWD 7 /* Forward */
79#define BTF_KIND_TYPEDEF 8 /* Typedef */
80#define BTF_KIND_VOLATILE 9 /* Volatile */
81#define BTF_KIND_CONST 10 /* Const */
82#define BTF_KIND_RESTRICT 11 /* Restrict */
83#define BTF_KIND_MAX 11
84#define NR_BTF_KINDS 12
85
86/* For some specific BTF_KIND, "struct btf_type" is immediately
87 * followed by extra data.
88 */
89
90/* BTF_KIND_INT is followed by a u32 and the following
91 * is the 32 bits arrangement:
92 */
93#define BTF_INT_ENCODING(VAL) (((VAL) & 0xff000000) >> 24)
94#define BTF_INT_OFFSET(VAL) (((VAL & 0x00ff0000)) >> 16)
95#define BTF_INT_BITS(VAL) ((VAL) & 0x0000ffff)
96
97/* Attributes stored in the BTF_INT_ENCODING */
98#define BTF_INT_SIGNED 0x1
99#define BTF_INT_CHAR 0x2
100#define BTF_INT_BOOL 0x4
101#define BTF_INT_VARARGS 0x8
102
103/* BTF_KIND_ENUM is followed by multiple "struct btf_enum".
104 * The exact number of btf_enum is stored in the vlen (of the
105 * info in "struct btf_type").
106 */
107struct btf_enum {
108 __u32 name;
109 __s32 val;
110};
111
112/* BTF_KIND_ARRAY is followed by one "struct btf_array" */
113struct btf_array {
114 __u32 type;
115 __u32 index_type;
116 __u32 nelems;
117};
118
119/* BTF_KIND_STRUCT and BTF_KIND_UNION are followed
120 * by multiple "struct btf_member". The exact number
121 * of btf_member is stored in the vlen (of the info in
122 * "struct btf_type").
123 */
124struct btf_member {
125 __u32 name;
126 __u32 type;
127 __u32 offset; /* offset in bits */
128};
129
130#endif /* _UAPI__LINUX_BTF_H__ */
diff --git a/tools/lib/bpf/Build b/tools/lib/bpf/Build
index 64c679d67109..6070e655042d 100644
--- a/tools/lib/bpf/Build
+++ b/tools/lib/bpf/Build
@@ -1 +1 @@
libbpf-y := libbpf.o bpf.o nlattr.o libbpf-y := libbpf.o bpf.o nlattr.o btf.o
diff --git a/tools/lib/bpf/bpf.c b/tools/lib/bpf/bpf.c
index acbb3f8b3bec..76b36cc16e7f 100644
--- a/tools/lib/bpf/bpf.c
+++ b/tools/lib/bpf/bpf.c
@@ -73,43 +73,76 @@ static inline int sys_bpf(enum bpf_cmd cmd, union bpf_attr *attr,
73 return syscall(__NR_bpf, cmd, attr, size); 73 return syscall(__NR_bpf, cmd, attr, size);
74} 74}
75 75
76int bpf_create_map_node(enum bpf_map_type map_type, const char *name, 76int bpf_create_map_xattr(const struct bpf_create_map_attr *create_attr)
77 int key_size, int value_size, int max_entries,
78 __u32 map_flags, int node)
79{ 77{
80 __u32 name_len = name ? strlen(name) : 0; 78 __u32 name_len = create_attr->name ? strlen(create_attr->name) : 0;
81 union bpf_attr attr; 79 union bpf_attr attr;
82 80
83 memset(&attr, '\0', sizeof(attr)); 81 memset(&attr, '\0', sizeof(attr));
84 82
85 attr.map_type = map_type; 83 attr.map_type = create_attr->map_type;
86 attr.key_size = key_size; 84 attr.key_size = create_attr->key_size;
87 attr.value_size = value_size; 85 attr.value_size = create_attr->value_size;
88 attr.max_entries = max_entries; 86 attr.max_entries = create_attr->max_entries;
89 attr.map_flags = map_flags; 87 attr.map_flags = create_attr->map_flags;
90 memcpy(attr.map_name, name, min(name_len, BPF_OBJ_NAME_LEN - 1)); 88 memcpy(attr.map_name, create_attr->name,
89 min(name_len, BPF_OBJ_NAME_LEN - 1));
90 attr.numa_node = create_attr->numa_node;
91 attr.btf_fd = create_attr->btf_fd;
92 attr.btf_key_id = create_attr->btf_key_id;
93 attr.btf_value_id = create_attr->btf_value_id;
94
95 return sys_bpf(BPF_MAP_CREATE, &attr, sizeof(attr));
96}
91 97
98int bpf_create_map_node(enum bpf_map_type map_type, const char *name,
99 int key_size, int value_size, int max_entries,
100 __u32 map_flags, int node)
101{
102 struct bpf_create_map_attr map_attr = {};
103
104 map_attr.name = name;
105 map_attr.map_type = map_type;
106 map_attr.map_flags = map_flags;
107 map_attr.key_size = key_size;
108 map_attr.value_size = value_size;
109 map_attr.max_entries = max_entries;
92 if (node >= 0) { 110 if (node >= 0) {
93 attr.map_flags |= BPF_F_NUMA_NODE; 111 map_attr.numa_node = node;
94 attr.numa_node = node; 112 map_attr.map_flags |= BPF_F_NUMA_NODE;
95 } 113 }
96 114
97 return sys_bpf(BPF_MAP_CREATE, &attr, sizeof(attr)); 115 return bpf_create_map_xattr(&map_attr);
98} 116}
99 117
100int bpf_create_map(enum bpf_map_type map_type, int key_size, 118int bpf_create_map(enum bpf_map_type map_type, int key_size,
101 int value_size, int max_entries, __u32 map_flags) 119 int value_size, int max_entries, __u32 map_flags)
102{ 120{
103 return bpf_create_map_node(map_type, NULL, key_size, value_size, 121 struct bpf_create_map_attr map_attr = {};
104 max_entries, map_flags, -1); 122
123 map_attr.map_type = map_type;
124 map_attr.map_flags = map_flags;
125 map_attr.key_size = key_size;
126 map_attr.value_size = value_size;
127 map_attr.max_entries = max_entries;
128
129 return bpf_create_map_xattr(&map_attr);
105} 130}
106 131
107int bpf_create_map_name(enum bpf_map_type map_type, const char *name, 132int bpf_create_map_name(enum bpf_map_type map_type, const char *name,
108 int key_size, int value_size, int max_entries, 133 int key_size, int value_size, int max_entries,
109 __u32 map_flags) 134 __u32 map_flags)
110{ 135{
111 return bpf_create_map_node(map_type, name, key_size, value_size, 136 struct bpf_create_map_attr map_attr = {};
112 max_entries, map_flags, -1); 137
138 map_attr.name = name;
139 map_attr.map_type = map_type;
140 map_attr.map_flags = map_flags;
141 map_attr.key_size = key_size;
142 map_attr.value_size = value_size;
143 map_attr.max_entries = max_entries;
144
145 return bpf_create_map_xattr(&map_attr);
113} 146}
114 147
115int bpf_create_map_in_map_node(enum bpf_map_type map_type, const char *name, 148int bpf_create_map_in_map_node(enum bpf_map_type map_type, const char *name,
@@ -573,3 +606,28 @@ cleanup:
573 close(sock); 606 close(sock);
574 return ret; 607 return ret;
575} 608}
609
610int bpf_load_btf(void *btf, __u32 btf_size, char *log_buf, __u32 log_buf_size,
611 bool do_log)
612{
613 union bpf_attr attr = {};
614 int fd;
615
616 attr.btf = ptr_to_u64(btf);
617 attr.btf_size = btf_size;
618
619retry:
620 if (do_log && log_buf && log_buf_size) {
621 attr.btf_log_level = 1;
622 attr.btf_log_size = log_buf_size;
623 attr.btf_log_buf = ptr_to_u64(log_buf);
624 }
625
626 fd = sys_bpf(BPF_BTF_LOAD, &attr, sizeof(attr));
627 if (fd == -1 && !do_log && log_buf && log_buf_size) {
628 do_log = true;
629 goto retry;
630 }
631
632 return fd;
633}
diff --git a/tools/lib/bpf/bpf.h b/tools/lib/bpf/bpf.h
index 39f6a0d64a3b..01bda076310f 100644
--- a/tools/lib/bpf/bpf.h
+++ b/tools/lib/bpf/bpf.h
@@ -26,6 +26,20 @@
26#include <linux/bpf.h> 26#include <linux/bpf.h>
27#include <stddef.h> 27#include <stddef.h>
28 28
29struct bpf_create_map_attr {
30 const char *name;
31 enum bpf_map_type map_type;
32 __u32 map_flags;
33 __u32 key_size;
34 __u32 value_size;
35 __u32 max_entries;
36 __u32 numa_node;
37 __u32 btf_fd;
38 __u32 btf_key_id;
39 __u32 btf_value_id;
40};
41
42int bpf_create_map_xattr(const struct bpf_create_map_attr *create_attr);
29int bpf_create_map_node(enum bpf_map_type map_type, const char *name, 43int bpf_create_map_node(enum bpf_map_type map_type, const char *name,
30 int key_size, int value_size, int max_entries, 44 int key_size, int value_size, int max_entries,
31 __u32 map_flags, int node); 45 __u32 map_flags, int node);
@@ -87,4 +101,6 @@ int bpf_obj_get_info_by_fd(int prog_fd, void *info, __u32 *info_len);
87int bpf_prog_query(int target_fd, enum bpf_attach_type type, __u32 query_flags, 101int bpf_prog_query(int target_fd, enum bpf_attach_type type, __u32 query_flags,
88 __u32 *attach_flags, __u32 *prog_ids, __u32 *prog_cnt); 102 __u32 *attach_flags, __u32 *prog_ids, __u32 *prog_cnt);
89int bpf_raw_tracepoint_open(const char *name, int prog_fd); 103int bpf_raw_tracepoint_open(const char *name, int prog_fd);
104int bpf_load_btf(void *btf, __u32 btf_size, char *log_buf, __u32 log_buf_size,
105 bool do_log);
90#endif 106#endif
diff --git a/tools/lib/bpf/btf.c b/tools/lib/bpf/btf.c
new file mode 100644
index 000000000000..58b6255abc7a
--- /dev/null
+++ b/tools/lib/bpf/btf.c
@@ -0,0 +1,374 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/* Copyright (c) 2018 Facebook */
3
4#include <stdlib.h>
5#include <stdint.h>
6#include <string.h>
7#include <unistd.h>
8#include <errno.h>
9#include <linux/err.h>
10#include <linux/btf.h>
11#include "btf.h"
12#include "bpf.h"
13
14#define elog(fmt, ...) { if (err_log) err_log(fmt, ##__VA_ARGS__); }
15#define max(a, b) ((a) > (b) ? (a) : (b))
16#define min(a, b) ((a) < (b) ? (a) : (b))
17
18#define BTF_MAX_NR_TYPES 65535
19
20static struct btf_type btf_void;
21
22struct btf {
23 union {
24 struct btf_header *hdr;
25 void *data;
26 };
27 struct btf_type **types;
28 const char *strings;
29 void *nohdr_data;
30 uint32_t nr_types;
31 uint32_t types_size;
32 uint32_t data_size;
33 int fd;
34};
35
36static const char *btf_name_by_offset(const struct btf *btf, uint32_t offset)
37{
38 if (!BTF_STR_TBL_ELF_ID(offset) &&
39 BTF_STR_OFFSET(offset) < btf->hdr->str_len)
40 return &btf->strings[BTF_STR_OFFSET(offset)];
41 else
42 return NULL;
43}
44
45static int btf_add_type(struct btf *btf, struct btf_type *t)
46{
47 if (btf->types_size - btf->nr_types < 2) {
48 struct btf_type **new_types;
49 u32 expand_by, new_size;
50
51 if (btf->types_size == BTF_MAX_NR_TYPES)
52 return -E2BIG;
53
54 expand_by = max(btf->types_size >> 2, 16);
55 new_size = min(BTF_MAX_NR_TYPES, btf->types_size + expand_by);
56
57 new_types = realloc(btf->types, sizeof(*new_types) * new_size);
58 if (!new_types)
59 return -ENOMEM;
60
61 if (btf->nr_types == 0)
62 new_types[0] = &btf_void;
63
64 btf->types = new_types;
65 btf->types_size = new_size;
66 }
67
68 btf->types[++(btf->nr_types)] = t;
69
70 return 0;
71}
72
73static int btf_parse_hdr(struct btf *btf, btf_print_fn_t err_log)
74{
75 const struct btf_header *hdr = btf->hdr;
76 u32 meta_left;
77
78 if (btf->data_size < sizeof(struct btf_header)) {
79 elog("BTF header not found\n");
80 return -EINVAL;
81 }
82
83 if (hdr->magic != BTF_MAGIC) {
84 elog("Invalid BTF magic:%x\n", hdr->magic);
85 return -EINVAL;
86 }
87
88 if (hdr->version != BTF_VERSION) {
89 elog("Unsupported BTF version:%u\n", hdr->version);
90 return -ENOTSUP;
91 }
92
93 if (hdr->flags) {
94 elog("Unsupported BTF flags:%x\n", hdr->flags);
95 return -ENOTSUP;
96 }
97
98 meta_left = btf->data_size - sizeof(*hdr);
99 if (!meta_left) {
100 elog("BTF has no data\n");
101 return -EINVAL;
102 }
103
104 if (meta_left < hdr->type_off) {
105 elog("Invalid BTF type section offset:%u\n", hdr->type_off);
106 return -EINVAL;
107 }
108
109 if (meta_left < hdr->str_off) {
110 elog("Invalid BTF string section offset:%u\n", hdr->str_off);
111 return -EINVAL;
112 }
113
114 if (hdr->type_off >= hdr->str_off) {
115 elog("BTF type section offset >= string section offset. No type?\n");
116 return -EINVAL;
117 }
118
119 if (hdr->type_off & 0x02) {
120 elog("BTF type section is not aligned to 4 bytes\n");
121 return -EINVAL;
122 }
123
124 btf->nohdr_data = btf->hdr + 1;
125
126 return 0;
127}
128
129static int btf_parse_str_sec(struct btf *btf, btf_print_fn_t err_log)
130{
131 const struct btf_header *hdr = btf->hdr;
132 const char *start = btf->nohdr_data + hdr->str_off;
133 const char *end = start + btf->hdr->str_len;
134
135 if (!hdr->str_len || hdr->str_len - 1 > BTF_MAX_NAME_OFFSET ||
136 start[0] || end[-1]) {
137 elog("Invalid BTF string section\n");
138 return -EINVAL;
139 }
140
141 btf->strings = start;
142
143 return 0;
144}
145
146static int btf_parse_type_sec(struct btf *btf, btf_print_fn_t err_log)
147{
148 struct btf_header *hdr = btf->hdr;
149 void *nohdr_data = btf->nohdr_data;
150 void *next_type = nohdr_data + hdr->type_off;
151 void *end_type = nohdr_data + hdr->str_off;
152
153 while (next_type < end_type) {
154 struct btf_type *t = next_type;
155 uint16_t vlen = BTF_INFO_VLEN(t->info);
156 int err;
157
158 next_type += sizeof(*t);
159 switch (BTF_INFO_KIND(t->info)) {
160 case BTF_KIND_INT:
161 next_type += sizeof(int);
162 break;
163 case BTF_KIND_ARRAY:
164 next_type += sizeof(struct btf_array);
165 break;
166 case BTF_KIND_STRUCT:
167 case BTF_KIND_UNION:
168 next_type += vlen * sizeof(struct btf_member);
169 break;
170 case BTF_KIND_ENUM:
171 next_type += vlen * sizeof(struct btf_enum);
172 break;
173 case BTF_KIND_TYPEDEF:
174 case BTF_KIND_PTR:
175 case BTF_KIND_FWD:
176 case BTF_KIND_VOLATILE:
177 case BTF_KIND_CONST:
178 case BTF_KIND_RESTRICT:
179 break;
180 default:
181 elog("Unsupported BTF_KIND:%u\n",
182 BTF_INFO_KIND(t->info));
183 return -EINVAL;
184 }
185
186 err = btf_add_type(btf, t);
187 if (err)
188 return err;
189 }
190
191 return 0;
192}
193
194static const struct btf_type *btf_type_by_id(const struct btf *btf,
195 uint32_t type_id)
196{
197 if (type_id > btf->nr_types)
198 return NULL;
199
200 return btf->types[type_id];
201}
202
203static bool btf_type_is_void(const struct btf_type *t)
204{
205 return t == &btf_void || BTF_INFO_KIND(t->info) == BTF_KIND_FWD;
206}
207
208static bool btf_type_is_void_or_null(const struct btf_type *t)
209{
210 return !t || btf_type_is_void(t);
211}
212
213static int64_t btf_type_size(const struct btf_type *t)
214{
215 switch (BTF_INFO_KIND(t->info)) {
216 case BTF_KIND_INT:
217 case BTF_KIND_STRUCT:
218 case BTF_KIND_UNION:
219 case BTF_KIND_ENUM:
220 return t->size;
221 case BTF_KIND_PTR:
222 return sizeof(void *);
223 default:
224 return -EINVAL;
225 }
226}
227
228#define MAX_RESOLVE_DEPTH 32
229
230int64_t btf__resolve_size(const struct btf *btf, uint32_t type_id)
231{
232 const struct btf_array *array;
233 const struct btf_type *t;
234 uint32_t nelems = 1;
235 int64_t size = -1;
236 int i;
237
238 t = btf_type_by_id(btf, type_id);
239 for (i = 0; i < MAX_RESOLVE_DEPTH && !btf_type_is_void_or_null(t);
240 i++) {
241 size = btf_type_size(t);
242 if (size >= 0)
243 break;
244
245 switch (BTF_INFO_KIND(t->info)) {
246 case BTF_KIND_TYPEDEF:
247 case BTF_KIND_VOLATILE:
248 case BTF_KIND_CONST:
249 case BTF_KIND_RESTRICT:
250 type_id = t->type;
251 break;
252 case BTF_KIND_ARRAY:
253 array = (const struct btf_array *)(t + 1);
254 if (nelems && array->nelems > UINT32_MAX / nelems)
255 return -E2BIG;
256 nelems *= array->nelems;
257 type_id = array->type;
258 break;
259 default:
260 return -EINVAL;
261 }
262
263 t = btf_type_by_id(btf, type_id);
264 }
265
266 if (size < 0)
267 return -EINVAL;
268
269 if (nelems && size > UINT32_MAX / nelems)
270 return -E2BIG;
271
272 return nelems * size;
273}
274
275int32_t btf__find_by_name(const struct btf *btf, const char *type_name)
276{
277 uint32_t i;
278
279 if (!strcmp(type_name, "void"))
280 return 0;
281
282 for (i = 1; i <= btf->nr_types; i++) {
283 const struct btf_type *t = btf->types[i];
284 const char *name = btf_name_by_offset(btf, t->name);
285
286 if (name && !strcmp(type_name, name))
287 return i;
288 }
289
290 return -ENOENT;
291}
292
293void btf__free(struct btf *btf)
294{
295 if (!btf)
296 return;
297
298 if (btf->fd != -1)
299 close(btf->fd);
300
301 free(btf->data);
302 free(btf->types);
303 free(btf);
304}
305
306struct btf *btf__new(uint8_t *data, uint32_t size,
307 btf_print_fn_t err_log)
308{
309 uint32_t log_buf_size = 0;
310 char *log_buf = NULL;
311 struct btf *btf;
312 int err;
313
314 btf = calloc(1, sizeof(struct btf));
315 if (!btf)
316 return ERR_PTR(-ENOMEM);
317
318 btf->fd = -1;
319
320 if (err_log) {
321 log_buf = malloc(BPF_LOG_BUF_SIZE);
322 if (!log_buf) {
323 err = -ENOMEM;
324 goto done;
325 }
326 *log_buf = 0;
327 log_buf_size = BPF_LOG_BUF_SIZE;
328 }
329
330 btf->data = malloc(size);
331 if (!btf->data) {
332 err = -ENOMEM;
333 goto done;
334 }
335
336 memcpy(btf->data, data, size);
337 btf->data_size = size;
338
339 btf->fd = bpf_load_btf(btf->data, btf->data_size,
340 log_buf, log_buf_size, false);
341
342 if (btf->fd == -1) {
343 err = -errno;
344 elog("Error loading BTF: %s(%d)\n", strerror(errno), errno);
345 if (log_buf && *log_buf)
346 elog("%s\n", log_buf);
347 goto done;
348 }
349
350 err = btf_parse_hdr(btf, err_log);
351 if (err)
352 goto done;
353
354 err = btf_parse_str_sec(btf, err_log);
355 if (err)
356 goto done;
357
358 err = btf_parse_type_sec(btf, err_log);
359
360done:
361 free(log_buf);
362
363 if (err) {
364 btf__free(btf);
365 return ERR_PTR(err);
366 }
367
368 return btf;
369}
370
371int btf__fd(const struct btf *btf)
372{
373 return btf->fd;
374}
diff --git a/tools/lib/bpf/btf.h b/tools/lib/bpf/btf.h
new file mode 100644
index 000000000000..74bb344035bb
--- /dev/null
+++ b/tools/lib/bpf/btf.h
@@ -0,0 +1,22 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/* Copyright (c) 2018 Facebook */
3
4#ifndef __BPF_BTF_H
5#define __BPF_BTF_H
6
7#include <stdint.h>
8
9#define BTF_ELF_SEC ".BTF"
10
11struct btf;
12
13typedef int (*btf_print_fn_t)(const char *, ...)
14 __attribute__((format(printf, 1, 2)));
15
16void btf__free(struct btf *btf);
17struct btf *btf__new(uint8_t *data, uint32_t size, btf_print_fn_t err_log);
18int32_t btf__find_by_name(const struct btf *btf, const char *type_name);
19int64_t btf__resolve_size(const struct btf *btf, uint32_t type_id);
20int btf__fd(const struct btf *btf);
21
22#endif
diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c
index 3d35bacf656f..6513e0b08795 100644
--- a/tools/lib/bpf/libbpf.c
+++ b/tools/lib/bpf/libbpf.c
@@ -45,6 +45,7 @@
45 45
46#include "libbpf.h" 46#include "libbpf.h"
47#include "bpf.h" 47#include "bpf.h"
48#include "btf.h"
48 49
49#ifndef EM_BPF 50#ifndef EM_BPF
50#define EM_BPF 247 51#define EM_BPF 247
@@ -212,6 +213,8 @@ struct bpf_map {
212 char *name; 213 char *name;
213 size_t offset; 214 size_t offset;
214 struct bpf_map_def def; 215 struct bpf_map_def def;
216 uint32_t btf_key_id;
217 uint32_t btf_value_id;
215 void *priv; 218 void *priv;
216 bpf_map_clear_priv_t clear_priv; 219 bpf_map_clear_priv_t clear_priv;
217}; 220};
@@ -256,6 +259,8 @@ struct bpf_object {
256 */ 259 */
257 struct list_head list; 260 struct list_head list;
258 261
262 struct btf *btf;
263
259 void *priv; 264 void *priv;
260 bpf_object_clear_priv_t clear_priv; 265 bpf_object_clear_priv_t clear_priv;
261 266
@@ -819,7 +824,15 @@ static int bpf_object__elf_collect(struct bpf_object *obj)
819 data->d_size); 824 data->d_size);
820 else if (strcmp(name, "maps") == 0) 825 else if (strcmp(name, "maps") == 0)
821 obj->efile.maps_shndx = idx; 826 obj->efile.maps_shndx = idx;
822 else if (sh.sh_type == SHT_SYMTAB) { 827 else if (strcmp(name, BTF_ELF_SEC) == 0) {
828 obj->btf = btf__new(data->d_buf, data->d_size,
829 __pr_debug);
830 if (IS_ERR(obj->btf)) {
831 pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n",
832 BTF_ELF_SEC, PTR_ERR(obj->btf));
833 obj->btf = NULL;
834 }
835 } else if (sh.sh_type == SHT_SYMTAB) {
823 if (obj->efile.symbols) { 836 if (obj->efile.symbols) {
824 pr_warning("bpf: multiple SYMTAB in %s\n", 837 pr_warning("bpf: multiple SYMTAB in %s\n",
825 obj->path); 838 obj->path);
@@ -996,33 +1009,126 @@ bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr,
996 return 0; 1009 return 0;
997} 1010}
998 1011
1012static int bpf_map_find_btf_info(struct bpf_map *map, const struct btf *btf)
1013{
1014 struct bpf_map_def *def = &map->def;
1015 const size_t max_name = 256;
1016 int64_t key_size, value_size;
1017 int32_t key_id, value_id;
1018 char name[max_name];
1019
1020 /* Find key type by name from BTF */
1021 if (snprintf(name, max_name, "%s_key", map->name) == max_name) {
1022 pr_warning("map:%s length of BTF key_type:%s_key is too long\n",
1023 map->name, map->name);
1024 return -EINVAL;
1025 }
1026
1027 key_id = btf__find_by_name(btf, name);
1028 if (key_id < 0) {
1029 pr_debug("map:%s key_type:%s cannot be found in BTF\n",
1030 map->name, name);
1031 return key_id;
1032 }
1033
1034 key_size = btf__resolve_size(btf, key_id);
1035 if (key_size < 0) {
1036 pr_warning("map:%s key_type:%s cannot get the BTF type_size\n",
1037 map->name, name);
1038 return key_size;
1039 }
1040
1041 if (def->key_size != key_size) {
1042 pr_warning("map:%s key_type:%s has BTF type_size:%ld != key_size:%u\n",
1043 map->name, name, key_size, def->key_size);
1044 return -EINVAL;
1045 }
1046
1047 /* Find value type from BTF */
1048 if (snprintf(name, max_name, "%s_value", map->name) == max_name) {
1049 pr_warning("map:%s length of BTF value_type:%s_value is too long\n",
1050 map->name, map->name);
1051 return -EINVAL;
1052 }
1053
1054 value_id = btf__find_by_name(btf, name);
1055 if (value_id < 0) {
1056 pr_debug("map:%s value_type:%s cannot be found in BTF\n",
1057 map->name, name);
1058 return value_id;
1059 }
1060
1061 value_size = btf__resolve_size(btf, value_id);
1062 if (value_size < 0) {
1063 pr_warning("map:%s value_type:%s cannot get the BTF type_size\n",
1064 map->name, name);
1065 return value_size;
1066 }
1067
1068 if (def->value_size != value_size) {
1069 pr_warning("map:%s value_type:%s has BTF type_size:%ld != value_size:%u\n",
1070 map->name, name, value_size, def->value_size);
1071 return -EINVAL;
1072 }
1073
1074 map->btf_key_id = key_id;
1075 map->btf_value_id = value_id;
1076
1077 return 0;
1078}
1079
999static int 1080static int
1000bpf_object__create_maps(struct bpf_object *obj) 1081bpf_object__create_maps(struct bpf_object *obj)
1001{ 1082{
1083 struct bpf_create_map_attr create_attr = {};
1002 unsigned int i; 1084 unsigned int i;
1085 int err;
1003 1086
1004 for (i = 0; i < obj->nr_maps; i++) { 1087 for (i = 0; i < obj->nr_maps; i++) {
1005 struct bpf_map_def *def = &obj->maps[i].def; 1088 struct bpf_map *map = &obj->maps[i];
1006 int *pfd = &obj->maps[i].fd; 1089 struct bpf_map_def *def = &map->def;
1007 1090 int *pfd = &map->fd;
1008 *pfd = bpf_create_map_name(def->type, 1091
1009 obj->maps[i].name, 1092 create_attr.name = map->name;
1010 def->key_size, 1093 create_attr.map_type = def->type;
1011 def->value_size, 1094 create_attr.map_flags = def->map_flags;
1012 def->max_entries, 1095 create_attr.key_size = def->key_size;
1013 def->map_flags); 1096 create_attr.value_size = def->value_size;
1097 create_attr.max_entries = def->max_entries;
1098 create_attr.btf_fd = 0;
1099 create_attr.btf_key_id = 0;
1100 create_attr.btf_value_id = 0;
1101
1102 if (obj->btf && !bpf_map_find_btf_info(map, obj->btf)) {
1103 create_attr.btf_fd = btf__fd(obj->btf);
1104 create_attr.btf_key_id = map->btf_key_id;
1105 create_attr.btf_value_id = map->btf_value_id;
1106 }
1107
1108 *pfd = bpf_create_map_xattr(&create_attr);
1109 if (*pfd < 0 && create_attr.btf_key_id) {
1110 pr_warning("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n",
1111 map->name, strerror(errno), errno);
1112 create_attr.btf_fd = 0;
1113 create_attr.btf_key_id = 0;
1114 create_attr.btf_value_id = 0;
1115 map->btf_key_id = 0;
1116 map->btf_value_id = 0;
1117 *pfd = bpf_create_map_xattr(&create_attr);
1118 }
1119
1014 if (*pfd < 0) { 1120 if (*pfd < 0) {
1015 size_t j; 1121 size_t j;
1016 int err = *pfd;
1017 1122
1123 err = *pfd;
1018 pr_warning("failed to create map (name: '%s'): %s\n", 1124 pr_warning("failed to create map (name: '%s'): %s\n",
1019 obj->maps[i].name, 1125 map->name,
1020 strerror(errno)); 1126 strerror(errno));
1021 for (j = 0; j < i; j++) 1127 for (j = 0; j < i; j++)
1022 zclose(obj->maps[j].fd); 1128 zclose(obj->maps[j].fd);
1023 return err; 1129 return err;
1024 } 1130 }
1025 pr_debug("create map %s: fd=%d\n", obj->maps[i].name, *pfd); 1131 pr_debug("create map %s: fd=%d\n", map->name, *pfd);
1026 } 1132 }
1027 1133
1028 return 0; 1134 return 0;
@@ -1641,6 +1747,7 @@ void bpf_object__close(struct bpf_object *obj)
1641 1747
1642 bpf_object__elf_finish(obj); 1748 bpf_object__elf_finish(obj);
1643 bpf_object__unload(obj); 1749 bpf_object__unload(obj);
1750 btf__free(obj->btf);
1644 1751
1645 for (i = 0; i < obj->nr_maps; i++) { 1752 for (i = 0; i < obj->nr_maps; i++) {
1646 zfree(&obj->maps[i].name); 1753 zfree(&obj->maps[i].name);
@@ -1692,6 +1799,11 @@ unsigned int bpf_object__kversion(struct bpf_object *obj)
1692 return obj ? obj->kern_version : 0; 1799 return obj ? obj->kern_version : 0;
1693} 1800}
1694 1801
1802int bpf_object__btf_fd(const struct bpf_object *obj)
1803{
1804 return obj->btf ? btf__fd(obj->btf) : -1;
1805}
1806
1695int bpf_object__set_priv(struct bpf_object *obj, void *priv, 1807int bpf_object__set_priv(struct bpf_object *obj, void *priv,
1696 bpf_object_clear_priv_t clear_priv) 1808 bpf_object_clear_priv_t clear_priv)
1697{ 1809{
@@ -1937,6 +2049,16 @@ const char *bpf_map__name(struct bpf_map *map)
1937 return map ? map->name : NULL; 2049 return map ? map->name : NULL;
1938} 2050}
1939 2051
2052uint32_t bpf_map__btf_key_id(const struct bpf_map *map)
2053{
2054 return map ? map->btf_key_id : 0;
2055}
2056
2057uint32_t bpf_map__btf_value_id(const struct bpf_map *map)
2058{
2059 return map ? map->btf_value_id : 0;
2060}
2061
1940int bpf_map__set_priv(struct bpf_map *map, void *priv, 2062int bpf_map__set_priv(struct bpf_map *map, void *priv,
1941 bpf_map_clear_priv_t clear_priv) 2063 bpf_map_clear_priv_t clear_priv)
1942{ 2064{
diff --git a/tools/lib/bpf/libbpf.h b/tools/lib/bpf/libbpf.h
index 8b242486b464..d6ac4fa6f472 100644
--- a/tools/lib/bpf/libbpf.h
+++ b/tools/lib/bpf/libbpf.h
@@ -78,6 +78,7 @@ int bpf_object__load(struct bpf_object *obj);
78int bpf_object__unload(struct bpf_object *obj); 78int bpf_object__unload(struct bpf_object *obj);
79const char *bpf_object__name(struct bpf_object *obj); 79const char *bpf_object__name(struct bpf_object *obj);
80unsigned int bpf_object__kversion(struct bpf_object *obj); 80unsigned int bpf_object__kversion(struct bpf_object *obj);
81int bpf_object__btf_fd(const struct bpf_object *obj);
81 82
82struct bpf_object *bpf_object__next(struct bpf_object *prev); 83struct bpf_object *bpf_object__next(struct bpf_object *prev);
83#define bpf_object__for_each_safe(pos, tmp) \ 84#define bpf_object__for_each_safe(pos, tmp) \
@@ -241,6 +242,8 @@ bpf_map__next(struct bpf_map *map, struct bpf_object *obj);
241int bpf_map__fd(struct bpf_map *map); 242int bpf_map__fd(struct bpf_map *map);
242const struct bpf_map_def *bpf_map__def(struct bpf_map *map); 243const struct bpf_map_def *bpf_map__def(struct bpf_map *map);
243const char *bpf_map__name(struct bpf_map *map); 244const char *bpf_map__name(struct bpf_map *map);
245uint32_t bpf_map__btf_key_id(const struct bpf_map *map);
246uint32_t bpf_map__btf_value_id(const struct bpf_map *map);
244 247
245typedef void (*bpf_map_clear_priv_t)(struct bpf_map *, void *); 248typedef void (*bpf_map_clear_priv_t)(struct bpf_map *, void *);
246int bpf_map__set_priv(struct bpf_map *map, void *priv, 249int bpf_map__set_priv(struct bpf_map *map, void *priv,
diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile
index 3e819dc70bee..0b72cc7596f1 100644
--- a/tools/testing/selftests/bpf/Makefile
+++ b/tools/testing/selftests/bpf/Makefile
@@ -24,14 +24,15 @@ urandom_read: urandom_read.c
24# Order correspond to 'make run_tests' order 24# Order correspond to 'make run_tests' order
25TEST_GEN_PROGS = test_verifier test_tag test_maps test_lru_map test_lpm_map test_progs \ 25TEST_GEN_PROGS = test_verifier test_tag test_maps test_lru_map test_lpm_map test_progs \
26 test_align test_verifier_log test_dev_cgroup test_tcpbpf_user \ 26 test_align test_verifier_log test_dev_cgroup test_tcpbpf_user \
27 test_sock test_sock_addr 27 test_sock test_sock_addr test_btf
28 28
29TEST_GEN_FILES = test_pkt_access.o test_xdp.o test_l4lb.o test_tcp_estats.o test_obj_id.o \ 29TEST_GEN_FILES = test_pkt_access.o test_xdp.o test_l4lb.o test_tcp_estats.o test_obj_id.o \
30 test_pkt_md_access.o test_xdp_redirect.o test_xdp_meta.o sockmap_parse_prog.o \ 30 test_pkt_md_access.o test_xdp_redirect.o test_xdp_meta.o sockmap_parse_prog.o \
31 sockmap_verdict_prog.o dev_cgroup.o sample_ret0.o test_tracepoint.o \ 31 sockmap_verdict_prog.o dev_cgroup.o sample_ret0.o test_tracepoint.o \
32 test_l4lb_noinline.o test_xdp_noinline.o test_stacktrace_map.o \ 32 test_l4lb_noinline.o test_xdp_noinline.o test_stacktrace_map.o \
33 sample_map_ret0.o test_tcpbpf_kern.o test_stacktrace_build_id.o \ 33 sample_map_ret0.o test_tcpbpf_kern.o test_stacktrace_build_id.o \
34 sockmap_tcp_msg_prog.o connect4_prog.o connect6_prog.o test_adjust_tail.o 34 sockmap_tcp_msg_prog.o connect4_prog.o connect6_prog.o test_adjust_tail.o \
35 test_btf_haskv.o test_btf_nokv.o
35 36
36# Order correspond to 'make run_tests' order 37# Order correspond to 'make run_tests' order
37TEST_PROGS := test_kmod.sh \ 38TEST_PROGS := test_kmod.sh \
@@ -66,6 +67,8 @@ $(BPFOBJ): force
66 67
67CLANG ?= clang 68CLANG ?= clang
68LLC ?= llc 69LLC ?= llc
70LLVM_OBJCOPY ?= llvm-objcopy
71BTF_PAHOLE ?= pahole
69 72
70PROBE := $(shell $(LLC) -march=bpf -mcpu=probe -filetype=null /dev/null 2>&1) 73PROBE := $(shell $(LLC) -march=bpf -mcpu=probe -filetype=null /dev/null 2>&1)
71 74
@@ -83,9 +86,26 @@ CLANG_FLAGS = -I. -I./include/uapi -I../../../include/uapi \
83$(OUTPUT)/test_l4lb_noinline.o: CLANG_FLAGS += -fno-inline 86$(OUTPUT)/test_l4lb_noinline.o: CLANG_FLAGS += -fno-inline
84$(OUTPUT)/test_xdp_noinline.o: CLANG_FLAGS += -fno-inline 87$(OUTPUT)/test_xdp_noinline.o: CLANG_FLAGS += -fno-inline
85 88
89BTF_LLC_PROBE := $(shell $(LLC) -march=bpf -mattr=help |& grep dwarfris)
90BTF_PAHOLE_PROBE := $(shell $(BTF_PAHOLE) --help |& grep BTF)
91BTF_OBJCOPY_PROBE := $(shell $(LLVM_OBJCOPY) --version |& grep LLVM)
92
93ifneq ($(BTF_LLC_PROBE),)
94ifneq ($(BTF_PAHOLE_PROBE),)
95ifneq ($(BTF_OBJCOPY_PROBE),)
96 CLANG_FLAGS += -g
97 LLC_FLAGS += -mattr=dwarfris
98 DWARF2BTF = y
99endif
100endif
101endif
102
86$(OUTPUT)/%.o: %.c 103$(OUTPUT)/%.o: %.c
87 $(CLANG) $(CLANG_FLAGS) \ 104 $(CLANG) $(CLANG_FLAGS) \
88 -O2 -target bpf -emit-llvm -c $< -o - | \ 105 -O2 -target bpf -emit-llvm -c $< -o - | \
89 $(LLC) -march=bpf -mcpu=$(CPU) -filetype=obj -o $@ 106 $(LLC) -march=bpf -mcpu=$(CPU) $(LLC_FLAGS) -filetype=obj -o $@
107ifeq ($(DWARF2BTF),y)
108 $(BTF_PAHOLE) -J $@
109endif
90 110
91EXTRA_CLEAN := $(TEST_CUSTOM_PROGS) 111EXTRA_CLEAN := $(TEST_CUSTOM_PROGS)
diff --git a/tools/testing/selftests/bpf/test_btf.c b/tools/testing/selftests/bpf/test_btf.c
new file mode 100644
index 000000000000..7b39b1f712a1
--- /dev/null
+++ b/tools/testing/selftests/bpf/test_btf.c
@@ -0,0 +1,1669 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/* Copyright (c) 2018 Facebook */
3
4#include <linux/bpf.h>
5#include <linux/btf.h>
6#include <linux/err.h>
7#include <bpf/bpf.h>
8#include <sys/resource.h>
9#include <libelf.h>
10#include <gelf.h>
11#include <string.h>
12#include <stdlib.h>
13#include <stdio.h>
14#include <stdarg.h>
15#include <unistd.h>
16#include <fcntl.h>
17#include <errno.h>
18#include <bpf/libbpf.h>
19#include <bpf/btf.h>
20
21#include "bpf_rlimit.h"
22
23#define min(a, b) ((a) < (b) ? (a) : (b))
24#define __printf(a, b) __attribute__((format(printf, a, b)))
25
26__printf(1, 2)
27static int __base_pr(const char *format, ...)
28{
29 va_list args;
30 int err;
31
32 va_start(args, format);
33 err = vfprintf(stderr, format, args);
34 va_end(args);
35 return err;
36}
37
38#define BTF_INFO_ENC(kind, root, vlen) \
39 ((!!(root) << 31) | ((kind) << 24) | ((vlen) & BTF_MAX_VLEN))
40
41#define BTF_TYPE_ENC(name, info, size_or_type) \
42 (name), (info), (size_or_type)
43
44#define BTF_INT_ENC(encoding, bits_offset, nr_bits) \
45 ((encoding) << 24 | (bits_offset) << 16 | (nr_bits))
46#define BTF_TYPE_INT_ENC(name, encoding, bits_offset, bits, sz) \
47 BTF_TYPE_ENC(name, BTF_INFO_ENC(BTF_KIND_INT, 0, 0), sz), \
48 BTF_INT_ENC(encoding, bits_offset, bits)
49
50#define BTF_ARRAY_ENC(type, index_type, nr_elems) \
51 (type), (index_type), (nr_elems)
52#define BTF_TYPE_ARRAY_ENC(type, index_type, nr_elems) \
53 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_ARRAY, 0, 0), 0), \
54 BTF_ARRAY_ENC(type, index_type, nr_elems)
55
56#define BTF_MEMBER_ENC(name, type, bits_offset) \
57 (name), (type), (bits_offset)
58#define BTF_ENUM_ENC(name, val) (name), (val)
59
60#define BTF_TYPEDEF_ENC(name, type) \
61 BTF_TYPE_ENC(name, BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0), type)
62
63#define BTF_PTR_ENC(name, type) \
64 BTF_TYPE_ENC(name, BTF_INFO_ENC(BTF_KIND_PTR, 0, 0), type)
65
66#define BTF_END_RAW 0xdeadbeef
67#define NAME_TBD 0xdeadb33f
68
69#define MAX_NR_RAW_TYPES 1024
70#define BTF_LOG_BUF_SIZE 65535
71
72#ifndef ARRAY_SIZE
73# define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
74#endif
75
76static struct args {
77 unsigned int raw_test_num;
78 unsigned int file_test_num;
79 unsigned int get_info_test_num;
80 bool raw_test;
81 bool file_test;
82 bool get_info_test;
83 bool pprint_test;
84 bool always_log;
85} args;
86
87static char btf_log_buf[BTF_LOG_BUF_SIZE];
88
89static struct btf_header hdr_tmpl = {
90 .magic = BTF_MAGIC,
91 .version = BTF_VERSION,
92};
93
94struct btf_raw_test {
95 const char *descr;
96 const char *str_sec;
97 const char *map_name;
98 __u32 raw_types[MAX_NR_RAW_TYPES];
99 __u32 str_sec_size;
100 enum bpf_map_type map_type;
101 __u32 key_size;
102 __u32 value_size;
103 __u32 key_id;
104 __u32 value_id;
105 __u32 max_entries;
106 bool btf_load_err;
107 bool map_create_err;
108 int type_off_delta;
109 int str_off_delta;
110 int str_len_delta;
111};
112
113static struct btf_raw_test raw_tests[] = {
114/* enum E {
115 * E0,
116 * E1,
117 * };
118 *
119 * struct A {
120 * int m;
121 * unsigned long long n;
122 * char o;
123 * [3 bytes hole]
124 * int p[8];
125 * int q[4][8];
126 * enum E r;
127 * };
128 */
129{
130 .descr = "struct test #1",
131 .raw_types = {
132 /* int */
133 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
134 /* unsigned long long */
135 BTF_TYPE_INT_ENC(0, 0, 0, 64, 8), /* [2] */
136 /* char */
137 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 8, 1), /* [3] */
138 /* int[8] */
139 BTF_TYPE_ARRAY_ENC(1, 1, 8), /* [4] */
140 /* struct A { */ /* [5] */
141 BTF_TYPE_ENC(NAME_TBD, BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 6), 180),
142 BTF_MEMBER_ENC(NAME_TBD, 1, 0), /* int m; */
143 BTF_MEMBER_ENC(NAME_TBD, 2, 32),/* unsigned long long n;*/
144 BTF_MEMBER_ENC(NAME_TBD, 3, 96),/* char o; */
145 BTF_MEMBER_ENC(NAME_TBD, 4, 128),/* int p[8] */
146 BTF_MEMBER_ENC(NAME_TBD, 6, 384),/* int q[4][8] */
147 BTF_MEMBER_ENC(NAME_TBD, 7, 1408), /* enum E r */
148 /* } */
149 /* int[4][8] */
150 BTF_TYPE_ARRAY_ENC(4, 1, 4), /* [6] */
151 BTF_TYPE_ENC(NAME_TBD, BTF_INFO_ENC(BTF_KIND_ENUM, 0, 2), sizeof(int)),
152 BTF_ENUM_ENC(NAME_TBD, 0),
153 BTF_ENUM_ENC(NAME_TBD, 1),
154 BTF_END_RAW,
155 },
156 .str_sec = "\0A\0m\0n\0o\0p\0q\0r\0E\0E0\0E1",
157 .str_sec_size = sizeof("\0A\0m\0n\0o\0p\0q\0r\0E\0E0\0E1"),
158 .map_type = BPF_MAP_TYPE_ARRAY,
159 .map_name = "struct_test1_map",
160 .key_size = sizeof(int),
161 .value_size = 180,
162 .key_id = 1,
163 .value_id = 5,
164 .max_entries = 4,
165},
166
167/* typedef struct b Struct_B;
168 *
169 * struct A {
170 * int m;
171 * struct b n[4];
172 * const Struct_B o[4];
173 * };
174 *
175 * struct B {
176 * int m;
177 * int n;
178 * };
179 */
180{
181 .descr = "struct test #2",
182 .raw_types = {
183 /* int */ /* [1] */
184 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4),
185 /* struct b [4] */ /* [2] */
186 BTF_TYPE_ARRAY_ENC(4, 1, 4),
187
188 /* struct A { */ /* [3] */
189 BTF_TYPE_ENC(NAME_TBD, BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 3), 68),
190 BTF_MEMBER_ENC(NAME_TBD, 1, 0), /* int m; */
191 BTF_MEMBER_ENC(NAME_TBD, 2, 32),/* struct B n[4] */
192 BTF_MEMBER_ENC(NAME_TBD, 8, 288),/* const Struct_B o[4];*/
193 /* } */
194
195 /* struct B { */ /* [4] */
196 BTF_TYPE_ENC(NAME_TBD, BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 2), 8),
197 BTF_MEMBER_ENC(NAME_TBD, 1, 0), /* int m; */
198 BTF_MEMBER_ENC(NAME_TBD, 1, 32),/* int n; */
199 /* } */
200
201 /* const int */ /* [5] */
202 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_CONST, 0, 0), 1),
203 /* typedef struct b Struct_B */ /* [6] */
204 BTF_TYPE_ENC(NAME_TBD, BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0), 4),
205 /* const Struct_B */ /* [7] */
206 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_CONST, 0, 0), 6),
207 /* const Struct_B [4] */ /* [8] */
208 BTF_TYPE_ARRAY_ENC(7, 1, 4),
209 BTF_END_RAW,
210 },
211 .str_sec = "\0A\0m\0n\0o\0B\0m\0n\0Struct_B",
212 .str_sec_size = sizeof("\0A\0m\0n\0o\0B\0m\0n\0Struct_B"),
213 .map_type = BPF_MAP_TYPE_ARRAY,
214 .map_name = "struct_test2_map",
215 .key_size = sizeof(int),
216 .value_size = 68,
217 .key_id = 1,
218 .value_id = 3,
219 .max_entries = 4,
220},
221
222/* Test member exceeds the size of struct.
223 *
224 * struct A {
225 * int m;
226 * int n;
227 * };
228 */
229{
230 .descr = "size check test #1",
231 .raw_types = {
232 /* int */ /* [1] */
233 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4),
234 /* struct A { */ /* [2] */
235 BTF_TYPE_ENC(NAME_TBD, BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 2), sizeof(int) * 2 - 1),
236 BTF_MEMBER_ENC(NAME_TBD, 1, 0), /* int m; */
237 BTF_MEMBER_ENC(NAME_TBD, 2, 32),/* int n; */
238 /* } */
239 BTF_END_RAW,
240 },
241 .str_sec = "\0A\0m\0n",
242 .str_sec_size = sizeof("\0A\0m\0n"),
243 .map_type = BPF_MAP_TYPE_ARRAY,
244 .map_name = "size_check1_map",
245 .key_size = sizeof(int),
246 .value_size = 1,
247 .key_id = 1,
248 .value_id = 2,
249 .max_entries = 4,
250 .btf_load_err = true,
251},
252
253/* Test member exeeds the size of struct
254 *
255 * struct A {
256 * int m;
257 * int n[2];
258 * };
259 */
260{
261 .descr = "size check test #2",
262 .raw_types = {
263 /* int */ /* [1] */
264 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, sizeof(int)),
265 /* int[2] */ /* [2] */
266 BTF_TYPE_ARRAY_ENC(1, 1, 2),
267 /* struct A { */ /* [3] */
268 BTF_TYPE_ENC(NAME_TBD, BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 2), sizeof(int) * 3 - 1),
269 BTF_MEMBER_ENC(NAME_TBD, 1, 0), /* int m; */
270 BTF_MEMBER_ENC(NAME_TBD, 2, 32),/* int n[2]; */
271 /* } */
272 BTF_END_RAW,
273 },
274 .str_sec = "\0A\0m\0n",
275 .str_sec_size = sizeof("\0A\0m\0n"),
276 .map_type = BPF_MAP_TYPE_ARRAY,
277 .map_name = "size_check2_map",
278 .key_size = sizeof(int),
279 .value_size = 1,
280 .key_id = 1,
281 .value_id = 3,
282 .max_entries = 4,
283 .btf_load_err = true,
284
285},
286
287/* Test member exeeds the size of struct
288 *
289 * struct A {
290 * int m;
291 * void *n;
292 * };
293 */
294{
295 .descr = "size check test #3",
296 .raw_types = {
297 /* int */ /* [1] */
298 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, sizeof(int)),
299 /* void* */ /* [2] */
300 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_PTR, 0, 0), 0),
301 /* struct A { */ /* [3] */
302 BTF_TYPE_ENC(NAME_TBD, BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 2), sizeof(int) + sizeof(void *) - 1),
303 BTF_MEMBER_ENC(NAME_TBD, 1, 0), /* int m; */
304 BTF_MEMBER_ENC(NAME_TBD, 2, 32),/* void *n; */
305 /* } */
306 BTF_END_RAW,
307 },
308 .str_sec = "\0A\0m\0n",
309 .str_sec_size = sizeof("\0A\0m\0n"),
310 .map_type = BPF_MAP_TYPE_ARRAY,
311 .map_name = "size_check3_map",
312 .key_size = sizeof(int),
313 .value_size = 1,
314 .key_id = 1,
315 .value_id = 3,
316 .max_entries = 4,
317 .btf_load_err = true,
318},
319
320/* Test member exceeds the size of struct
321 *
322 * enum E {
323 * E0,
324 * E1,
325 * };
326 *
327 * struct A {
328 * int m;
329 * enum E n;
330 * };
331 */
332{
333 .descr = "size check test #4",
334 .raw_types = {
335 /* int */ /* [1] */
336 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, sizeof(int)),
337 /* enum E { */ /* [2] */
338 BTF_TYPE_ENC(NAME_TBD, BTF_INFO_ENC(BTF_KIND_ENUM, 0, 2), sizeof(int)),
339 BTF_ENUM_ENC(NAME_TBD, 0),
340 BTF_ENUM_ENC(NAME_TBD, 1),
341 /* } */
342 /* struct A { */ /* [3] */
343 BTF_TYPE_ENC(NAME_TBD, BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 2), sizeof(int) * 2 - 1),
344 BTF_MEMBER_ENC(NAME_TBD, 1, 0), /* int m; */
345 BTF_MEMBER_ENC(NAME_TBD, 2, 32),/* enum E n; */
346 /* } */
347 BTF_END_RAW,
348 },
349 .str_sec = "\0E\0E0\0E1\0A\0m\0n",
350 .str_sec_size = sizeof("\0E\0E0\0E1\0A\0m\0n"),
351 .map_type = BPF_MAP_TYPE_ARRAY,
352 .map_name = "size_check4_map",
353 .key_size = sizeof(int),
354 .value_size = 1,
355 .key_id = 1,
356 .value_id = 3,
357 .max_entries = 4,
358 .btf_load_err = true,
359},
360
361/* typedef const void * const_void_ptr;
362 * struct A {
363 * const_void_ptr m;
364 * };
365 */
366{
367 .descr = "void test #1",
368 .raw_types = {
369 /* int */ /* [1] */
370 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4),
371 /* const void */ /* [2] */
372 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_CONST, 0, 0), 0),
373 /* const void* */ /* [3] */
374 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_PTR, 0, 0), 2),
375 /* typedef const void * const_void_ptr */
376 BTF_TYPE_ENC(NAME_TBD, BTF_INFO_ENC(BTF_KIND_PTR, 0, 0), 3),
377 /* struct A { */ /* [4] */
378 BTF_TYPE_ENC(NAME_TBD, BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 1), sizeof(void *)),
379 /* const_void_ptr m; */
380 BTF_MEMBER_ENC(NAME_TBD, 3, 0),
381 /* } */
382 BTF_END_RAW,
383 },
384 .str_sec = "\0const_void_ptr\0A\0m",
385 .str_sec_size = sizeof("\0const_void_ptr\0A\0m"),
386 .map_type = BPF_MAP_TYPE_ARRAY,
387 .map_name = "void_test1_map",
388 .key_size = sizeof(int),
389 .value_size = sizeof(void *),
390 .key_id = 1,
391 .value_id = 4,
392 .max_entries = 4,
393},
394
395/* struct A {
396 * const void m;
397 * };
398 */
399{
400 .descr = "void test #2",
401 .raw_types = {
402 /* int */ /* [1] */
403 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4),
404 /* const void */ /* [2] */
405 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_CONST, 0, 0), 0),
406 /* struct A { */ /* [3] */
407 BTF_TYPE_ENC(NAME_TBD, BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 1), 8),
408 /* const void m; */
409 BTF_MEMBER_ENC(NAME_TBD, 2, 0),
410 /* } */
411 BTF_END_RAW,
412 },
413 .str_sec = "\0A\0m",
414 .str_sec_size = sizeof("\0A\0m"),
415 .map_type = BPF_MAP_TYPE_ARRAY,
416 .map_name = "void_test2_map",
417 .key_size = sizeof(int),
418 .value_size = sizeof(void *),
419 .key_id = 1,
420 .value_id = 3,
421 .max_entries = 4,
422 .btf_load_err = true,
423},
424
425/* typedef const void * const_void_ptr;
426 * const_void_ptr[4]
427 */
428{
429 .descr = "void test #3",
430 .raw_types = {
431 /* int */ /* [1] */
432 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4),
433 /* const void */ /* [2] */
434 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_CONST, 0, 0), 0),
435 /* const void* */ /* [3] */
436 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_PTR, 0, 0), 2),
437 /* typedef const void * const_void_ptr */
438 BTF_TYPE_ENC(NAME_TBD, BTF_INFO_ENC(BTF_KIND_PTR, 0, 0), 3),
439 /* const_void_ptr[4] */ /* [4] */
440 BTF_TYPE_ARRAY_ENC(3, 1, 4),
441 BTF_END_RAW,
442 },
443 .str_sec = "\0const_void_ptr",
444 .str_sec_size = sizeof("\0const_void_ptr"),
445 .map_type = BPF_MAP_TYPE_ARRAY,
446 .map_name = "void_test3_map",
447 .key_size = sizeof(int),
448 .value_size = sizeof(void *) * 4,
449 .key_id = 1,
450 .value_id = 4,
451 .max_entries = 4,
452},
453
454/* const void[4] */
455{
456 .descr = "void test #4",
457 .raw_types = {
458 /* int */ /* [1] */
459 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4),
460 /* const void */ /* [2] */
461 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_CONST, 0, 0), 0),
462 /* const void[4] */ /* [3] */
463 BTF_TYPE_ARRAY_ENC(2, 1, 4),
464 BTF_END_RAW,
465 },
466 .str_sec = "\0A\0m",
467 .str_sec_size = sizeof("\0A\0m"),
468 .map_type = BPF_MAP_TYPE_ARRAY,
469 .map_name = "void_test4_map",
470 .key_size = sizeof(int),
471 .value_size = sizeof(void *) * 4,
472 .key_id = 1,
473 .value_id = 3,
474 .max_entries = 4,
475 .btf_load_err = true,
476},
477
478/* Array_A <------------------+
479 * elem_type == Array_B |
480 * | |
481 * | |
482 * Array_B <-------- + |
483 * elem_type == Array A --+
484 */
485{
486 .descr = "loop test #1",
487 .raw_types = {
488 /* int */ /* [1] */
489 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4),
490 /* Array_A */ /* [2] */
491 BTF_TYPE_ARRAY_ENC(3, 1, 8),
492 /* Array_B */ /* [3] */
493 BTF_TYPE_ARRAY_ENC(2, 1, 8),
494 BTF_END_RAW,
495 },
496 .str_sec = "",
497 .str_sec_size = sizeof(""),
498 .map_type = BPF_MAP_TYPE_ARRAY,
499 .map_name = "loop_test1_map",
500 .key_size = sizeof(int),
501 .value_size = sizeof(sizeof(int) * 8),
502 .key_id = 1,
503 .value_id = 2,
504 .max_entries = 4,
505 .btf_load_err = true,
506},
507
508/* typedef is _before_ the BTF type of Array_A and Array_B
509 *
510 * typedef Array_B int_array;
511 *
512 * Array_A <------------------+
513 * elem_type == int_array |
514 * | |
515 * | |
516 * Array_B <-------- + |
517 * elem_type == Array_A --+
518 */
519{
520 .descr = "loop test #2",
521 .raw_types = {
522 /* int */
523 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
524 /* typedef Array_B int_array */
525 BTF_TYPEDEF_ENC(1, 4), /* [2] */
526 /* Array_A */
527 BTF_TYPE_ARRAY_ENC(2, 1, 8), /* [3] */
528 /* Array_B */
529 BTF_TYPE_ARRAY_ENC(3, 1, 8), /* [4] */
530
531 BTF_END_RAW,
532 },
533 .str_sec = "\0int_array\0",
534 .str_sec_size = sizeof("\0int_array"),
535 .map_type = BPF_MAP_TYPE_ARRAY,
536 .map_name = "loop_test2_map",
537 .key_size = sizeof(int),
538 .value_size = sizeof(sizeof(int) * 8),
539 .key_id = 1,
540 .value_id = 2,
541 .max_entries = 4,
542 .btf_load_err = true,
543},
544
545/* Array_A <------------------+
546 * elem_type == Array_B |
547 * | |
548 * | |
549 * Array_B <-------- + |
550 * elem_type == Array_A --+
551 */
552{
553 .descr = "loop test #3",
554 .raw_types = {
555 /* int */ /* [1] */
556 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4),
557 /* Array_A */ /* [2] */
558 BTF_TYPE_ARRAY_ENC(3, 1, 8),
559 /* Array_B */ /* [3] */
560 BTF_TYPE_ARRAY_ENC(2, 1, 8),
561
562 BTF_END_RAW,
563 },
564 .str_sec = "",
565 .str_sec_size = sizeof(""),
566 .map_type = BPF_MAP_TYPE_ARRAY,
567 .map_name = "loop_test3_map",
568 .key_size = sizeof(int),
569 .value_size = sizeof(sizeof(int) * 8),
570 .key_id = 1,
571 .value_id = 2,
572 .max_entries = 4,
573 .btf_load_err = true,
574},
575
576/* typedef is _between_ the BTF type of Array_A and Array_B
577 *
578 * typedef Array_B int_array;
579 *
580 * Array_A <------------------+
581 * elem_type == int_array |
582 * | |
583 * | |
584 * Array_B <-------- + |
585 * elem_type == Array_A --+
586 */
587{
588 .descr = "loop test #4",
589 .raw_types = {
590 /* int */ /* [1] */
591 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4),
592 /* Array_A */ /* [2] */
593 BTF_TYPE_ARRAY_ENC(3, 1, 8),
594 /* typedef Array_B int_array */ /* [3] */
595 BTF_TYPEDEF_ENC(NAME_TBD, 4),
596 /* Array_B */ /* [4] */
597 BTF_TYPE_ARRAY_ENC(2, 1, 8),
598 BTF_END_RAW,
599 },
600 .str_sec = "\0int_array\0",
601 .str_sec_size = sizeof("\0int_array"),
602 .map_type = BPF_MAP_TYPE_ARRAY,
603 .map_name = "loop_test4_map",
604 .key_size = sizeof(int),
605 .value_size = sizeof(sizeof(int) * 8),
606 .key_id = 1,
607 .value_id = 2,
608 .max_entries = 4,
609 .btf_load_err = true,
610},
611
612/* typedef struct B Struct_B
613 *
614 * struct A {
615 * int x;
616 * Struct_B y;
617 * };
618 *
619 * struct B {
620 * int x;
621 * struct A y;
622 * };
623 */
624{
625 .descr = "loop test #5",
626 .raw_types = {
627 /* int */
628 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
629 /* struct A */ /* [2] */
630 BTF_TYPE_ENC(NAME_TBD, BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 2), 8),
631 BTF_MEMBER_ENC(NAME_TBD, 1, 0), /* int x; */
632 BTF_MEMBER_ENC(NAME_TBD, 3, 32),/* Struct_B y; */
633 /* typedef struct B Struct_B */
634 BTF_TYPEDEF_ENC(NAME_TBD, 4), /* [3] */
635 /* struct B */ /* [4] */
636 BTF_TYPE_ENC(NAME_TBD, BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 2), 8),
637 BTF_MEMBER_ENC(NAME_TBD, 1, 0), /* int x; */
638 BTF_MEMBER_ENC(NAME_TBD, 2, 32),/* struct A y; */
639 BTF_END_RAW,
640 },
641 .str_sec = "\0A\0x\0y\0Struct_B\0B\0x\0y",
642 .str_sec_size = sizeof("\0A\0x\0y\0Struct_B\0B\0x\0y"),
643 .map_type = BPF_MAP_TYPE_ARRAY,
644 .map_name = "loop_test5_map",
645 .key_size = sizeof(int),
646 .value_size = 8,
647 .key_id = 1,
648 .value_id = 2,
649 .max_entries = 4,
650 .btf_load_err = true,
651},
652
653/* struct A {
654 * int x;
655 * struct A array_a[4];
656 * };
657 */
658{
659 .descr = "loop test #6",
660 .raw_types = {
661 /* int */
662 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
663 BTF_TYPE_ARRAY_ENC(3, 1, 4), /* [2] */
664 /* struct A */ /* [3] */
665 BTF_TYPE_ENC(NAME_TBD, BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 2), 8),
666 BTF_MEMBER_ENC(NAME_TBD, 1, 0), /* int x; */
667 BTF_MEMBER_ENC(NAME_TBD, 2, 32),/* struct A array_a[4]; */
668 BTF_END_RAW,
669 },
670 .str_sec = "\0A\0x\0y",
671 .str_sec_size = sizeof("\0A\0x\0y"),
672 .map_type = BPF_MAP_TYPE_ARRAY,
673 .map_name = "loop_test6_map",
674 .key_size = sizeof(int),
675 .value_size = 8,
676 .key_id = 1,
677 .value_id = 2,
678 .max_entries = 4,
679 .btf_load_err = true,
680},
681
682{
683 .descr = "loop test #7",
684 .raw_types = {
685 /* int */ /* [1] */
686 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4),
687 /* struct A { */ /* [2] */
688 BTF_TYPE_ENC(NAME_TBD, BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 1), sizeof(void *)),
689 /* const void *m; */
690 BTF_MEMBER_ENC(NAME_TBD, 3, 0),
691 /* CONST type_id=3 */ /* [3] */
692 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_CONST, 0, 0), 4),
693 /* PTR type_id=2 */ /* [4] */
694 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_PTR, 0, 0), 3),
695 BTF_END_RAW,
696 },
697 .str_sec = "\0A\0m",
698 .str_sec_size = sizeof("\0A\0m"),
699 .map_type = BPF_MAP_TYPE_ARRAY,
700 .map_name = "loop_test7_map",
701 .key_size = sizeof(int),
702 .value_size = sizeof(void *),
703 .key_id = 1,
704 .value_id = 2,
705 .max_entries = 4,
706 .btf_load_err = true,
707},
708
709{
710 .descr = "loop test #8",
711 .raw_types = {
712 /* int */ /* [1] */
713 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4),
714 /* struct A { */ /* [2] */
715 BTF_TYPE_ENC(NAME_TBD, BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 1), sizeof(void *)),
716 /* const void *m; */
717 BTF_MEMBER_ENC(NAME_TBD, 4, 0),
718 /* struct B { */ /* [3] */
719 BTF_TYPE_ENC(NAME_TBD, BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 1), sizeof(void *)),
720 /* const void *n; */
721 BTF_MEMBER_ENC(NAME_TBD, 6, 0),
722 /* CONST type_id=5 */ /* [4] */
723 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_CONST, 0, 0), 5),
724 /* PTR type_id=6 */ /* [5] */
725 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_PTR, 0, 0), 6),
726 /* CONST type_id=7 */ /* [6] */
727 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_CONST, 0, 0), 7),
728 /* PTR type_id=4 */ /* [7] */
729 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_PTR, 0, 0), 4),
730 BTF_END_RAW,
731 },
732 .str_sec = "\0A\0m\0B\0n",
733 .str_sec_size = sizeof("\0A\0m\0B\0n"),
734 .map_type = BPF_MAP_TYPE_ARRAY,
735 .map_name = "loop_test8_map",
736 .key_size = sizeof(int),
737 .value_size = sizeof(void *),
738 .key_id = 1,
739 .value_id = 2,
740 .max_entries = 4,
741 .btf_load_err = true,
742},
743
744{
745 .descr = "type_off == str_off",
746 .raw_types = {
747 /* int */ /* [1] */
748 BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4),
749 BTF_END_RAW,
750 },
751 .str_sec = "\0int",
752 .str_sec_size = sizeof("\0int"),
753 .map_type = BPF_MAP_TYPE_ARRAY,
754 .map_name = "hdr_test_map",
755 .key_size = sizeof(int),
756 .value_size = sizeof(int),
757 .key_id = 1,
758 .value_id = 1,
759 .max_entries = 4,
760 .btf_load_err = true,
761 .type_off_delta = sizeof(struct btf_type) + sizeof(int) + sizeof("\0int"),
762},
763
764{
765 .descr = "Unaligned type_off",
766 .raw_types = {
767 /* int */ /* [1] */
768 BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4),
769 BTF_END_RAW,
770 },
771 .str_sec = "\0int",
772 .str_sec_size = sizeof("\0int"),
773 .map_type = BPF_MAP_TYPE_ARRAY,
774 .map_name = "hdr_test_map",
775 .key_size = sizeof(int),
776 .value_size = sizeof(int),
777 .key_id = 1,
778 .value_id = 1,
779 .max_entries = 4,
780 .btf_load_err = true,
781 .type_off_delta = 1,
782},
783
784{
785 .descr = "str_off beyonds btf size",
786 .raw_types = {
787 /* int */ /* [1] */
788 BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4),
789 BTF_END_RAW,
790 },
791 .str_sec = "\0int",
792 .str_sec_size = sizeof("\0int"),
793 .map_type = BPF_MAP_TYPE_ARRAY,
794 .map_name = "hdr_test_map",
795 .key_size = sizeof(int),
796 .value_size = sizeof(int),
797 .key_id = 1,
798 .value_id = 1,
799 .max_entries = 4,
800 .btf_load_err = true,
801 .str_off_delta = sizeof("\0int") + 1,
802},
803
804{
805 .descr = "str_len beyonds btf size",
806 .raw_types = {
807 /* int */ /* [1] */
808 BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4),
809 BTF_END_RAW,
810 },
811 .str_sec = "\0int",
812 .str_sec_size = sizeof("\0int"),
813 .map_type = BPF_MAP_TYPE_ARRAY,
814 .map_name = "hdr_test_map",
815 .key_size = sizeof(int),
816 .value_size = sizeof(int),
817 .key_id = 1,
818 .value_id = 1,
819 .max_entries = 4,
820 .btf_load_err = true,
821 .str_len_delta = 1,
822},
823
824{
825 .descr = "String section does not end with null",
826 .raw_types = {
827 /* int */ /* [1] */
828 BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4),
829 BTF_END_RAW,
830 },
831 .str_sec = "\0int",
832 .str_sec_size = sizeof("\0int"),
833 .map_type = BPF_MAP_TYPE_ARRAY,
834 .map_name = "hdr_test_map",
835 .key_size = sizeof(int),
836 .value_size = sizeof(int),
837 .key_id = 1,
838 .value_id = 1,
839 .max_entries = 4,
840 .btf_load_err = true,
841 .str_len_delta = -1,
842},
843
844{
845 .descr = "Empty string section",
846 .raw_types = {
847 /* int */ /* [1] */
848 BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4),
849 BTF_END_RAW,
850 },
851 .str_sec = "\0int",
852 .str_sec_size = sizeof("\0int"),
853 .map_type = BPF_MAP_TYPE_ARRAY,
854 .map_name = "hdr_test_map",
855 .key_size = sizeof(int),
856 .value_size = sizeof(int),
857 .key_id = 1,
858 .value_id = 1,
859 .max_entries = 4,
860 .btf_load_err = true,
861 .str_len_delta = 0 - (int)sizeof("\0int"),
862},
863
864}; /* struct btf_raw_test raw_tests[] */
865
866static const char *get_next_str(const char *start, const char *end)
867{
868 return start < end - 1 ? start + 1 : NULL;
869}
870
871static int get_type_sec_size(const __u32 *raw_types)
872{
873 int i;
874
875 for (i = MAX_NR_RAW_TYPES - 1;
876 i >= 0 && raw_types[i] != BTF_END_RAW;
877 i--)
878 ;
879
880 return i < 0 ? i : i * sizeof(raw_types[0]);
881}
882
883static void *btf_raw_create(const struct btf_header *hdr,
884 const __u32 *raw_types,
885 const char *str,
886 unsigned int str_sec_size,
887 unsigned int *btf_size)
888{
889 const char *next_str = str, *end_str = str + str_sec_size;
890 unsigned int size_needed, offset;
891 struct btf_header *ret_hdr;
892 int i, type_sec_size;
893 uint32_t *ret_types;
894 void *raw_btf;
895
896 type_sec_size = get_type_sec_size(raw_types);
897 if (type_sec_size < 0) {
898 fprintf(stderr, "Cannot get nr_raw_types\n");
899 return NULL;
900 }
901
902 size_needed = sizeof(*hdr) + type_sec_size + str_sec_size;
903 raw_btf = malloc(size_needed);
904 if (!raw_btf) {
905 fprintf(stderr, "Cannot allocate memory for raw_btf\n");
906 return NULL;
907 }
908
909 /* Copy header */
910 memcpy(raw_btf, hdr, sizeof(*hdr));
911 offset = sizeof(*hdr);
912
913 /* Copy type section */
914 ret_types = raw_btf + offset;
915 for (i = 0; i < type_sec_size / sizeof(raw_types[0]); i++) {
916 if (raw_types[i] == NAME_TBD) {
917 next_str = get_next_str(next_str, end_str);
918 if (!next_str) {
919 fprintf(stderr, "Error in getting next_str\n");
920 free(raw_btf);
921 return NULL;
922 }
923 ret_types[i] = next_str - str;
924 next_str += strlen(next_str);
925 } else {
926 ret_types[i] = raw_types[i];
927 }
928 }
929 offset += type_sec_size;
930
931 /* Copy string section */
932 memcpy(raw_btf + offset, str, str_sec_size);
933
934 ret_hdr = (struct btf_header *)raw_btf;
935 ret_hdr->str_off = type_sec_size;
936 ret_hdr->str_len = str_sec_size;
937
938 *btf_size = size_needed;
939
940 return raw_btf;
941}
942
943static int do_test_raw(unsigned int test_num)
944{
945 struct btf_raw_test *test = &raw_tests[test_num - 1];
946 struct bpf_create_map_attr create_attr = {};
947 int map_fd = -1, btf_fd = -1;
948 unsigned int raw_btf_size;
949 struct btf_header *hdr;
950 void *raw_btf;
951 int err;
952
953 fprintf(stderr, "BTF raw test[%u] (%s): ", test_num, test->descr);
954 raw_btf = btf_raw_create(&hdr_tmpl,
955 test->raw_types,
956 test->str_sec,
957 test->str_sec_size,
958 &raw_btf_size);
959
960 if (!raw_btf)
961 return -1;
962
963 hdr = raw_btf;
964
965 hdr->type_off = (int)hdr->type_off + test->type_off_delta;
966 hdr->str_off = (int)hdr->str_off + test->str_off_delta;
967 hdr->str_len = (int)hdr->str_len + test->str_len_delta;
968
969 *btf_log_buf = '\0';
970 btf_fd = bpf_load_btf(raw_btf, raw_btf_size,
971 btf_log_buf, BTF_LOG_BUF_SIZE,
972 args.always_log);
973 free(raw_btf);
974
975 err = ((btf_fd == -1) != test->btf_load_err);
976 if (err)
977 fprintf(stderr, "btf_load_err:%d btf_fd:%d\n",
978 test->btf_load_err, btf_fd);
979
980 if (err || btf_fd == -1)
981 goto done;
982
983 create_attr.name = test->map_name;
984 create_attr.map_type = test->map_type;
985 create_attr.key_size = test->key_size;
986 create_attr.value_size = test->value_size;
987 create_attr.max_entries = test->max_entries;
988 create_attr.btf_fd = btf_fd;
989 create_attr.btf_key_id = test->key_id;
990 create_attr.btf_value_id = test->value_id;
991
992 map_fd = bpf_create_map_xattr(&create_attr);
993
994 err = ((map_fd == -1) != test->map_create_err);
995 if (err)
996 fprintf(stderr, "map_create_err:%d map_fd:%d\n",
997 test->map_create_err, map_fd);
998
999done:
1000 if (!err)
1001 fprintf(stderr, "OK\n");
1002
1003 if (*btf_log_buf && (err || args.always_log))
1004 fprintf(stderr, "%s\n", btf_log_buf);
1005
1006 if (btf_fd != -1)
1007 close(btf_fd);
1008 if (map_fd != -1)
1009 close(map_fd);
1010
1011 return err;
1012}
1013
1014static int test_raw(void)
1015{
1016 unsigned int i;
1017 int err = 0;
1018
1019 if (args.raw_test_num)
1020 return do_test_raw(args.raw_test_num);
1021
1022 for (i = 1; i <= ARRAY_SIZE(raw_tests); i++)
1023 err |= do_test_raw(i);
1024
1025 return err;
1026}
1027
1028struct btf_get_info_test {
1029 const char *descr;
1030 const char *str_sec;
1031 __u32 raw_types[MAX_NR_RAW_TYPES];
1032 __u32 str_sec_size;
1033 int info_size_delta;
1034};
1035
1036const struct btf_get_info_test get_info_tests[] = {
1037{
1038 .descr = "== raw_btf_size+1",
1039 .raw_types = {
1040 /* int */ /* [1] */
1041 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4),
1042 BTF_END_RAW,
1043 },
1044 .str_sec = "",
1045 .str_sec_size = sizeof(""),
1046 .info_size_delta = 1,
1047},
1048{
1049 .descr = "== raw_btf_size-3",
1050 .raw_types = {
1051 /* int */ /* [1] */
1052 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4),
1053 BTF_END_RAW,
1054 },
1055 .str_sec = "",
1056 .str_sec_size = sizeof(""),
1057 .info_size_delta = -3,
1058},
1059};
1060
1061static int do_test_get_info(unsigned int test_num)
1062{
1063 const struct btf_get_info_test *test = &get_info_tests[test_num - 1];
1064 unsigned int raw_btf_size, user_btf_size, expected_nbytes;
1065 uint8_t *raw_btf = NULL, *user_btf = NULL;
1066 int btf_fd = -1, err;
1067
1068 fprintf(stderr, "BTF GET_INFO_BY_ID test[%u] (%s): ",
1069 test_num, test->descr);
1070
1071 raw_btf = btf_raw_create(&hdr_tmpl,
1072 test->raw_types,
1073 test->str_sec,
1074 test->str_sec_size,
1075 &raw_btf_size);
1076
1077 if (!raw_btf)
1078 return -1;
1079
1080 *btf_log_buf = '\0';
1081
1082 user_btf = malloc(raw_btf_size);
1083 if (!user_btf) {
1084 fprintf(stderr, "Cannot allocate memory for user_btf\n");
1085 err = -1;
1086 goto done;
1087 }
1088
1089 btf_fd = bpf_load_btf(raw_btf, raw_btf_size,
1090 btf_log_buf, BTF_LOG_BUF_SIZE,
1091 args.always_log);
1092 if (btf_fd == -1) {
1093 fprintf(stderr, "bpf_load_btf:%s(%d)\n",
1094 strerror(errno), errno);
1095 err = -1;
1096 goto done;
1097 }
1098
1099 user_btf_size = (int)raw_btf_size + test->info_size_delta;
1100 expected_nbytes = min(raw_btf_size, user_btf_size);
1101 if (raw_btf_size > expected_nbytes)
1102 memset(user_btf + expected_nbytes, 0xff,
1103 raw_btf_size - expected_nbytes);
1104
1105 err = bpf_obj_get_info_by_fd(btf_fd, user_btf, &user_btf_size);
1106 if (err || user_btf_size != raw_btf_size ||
1107 memcmp(raw_btf, user_btf, expected_nbytes)) {
1108 fprintf(stderr,
1109 "err:%d(errno:%d) raw_btf_size:%u user_btf_size:%u expected_nbytes:%u memcmp:%d\n",
1110 err, errno,
1111 raw_btf_size, user_btf_size, expected_nbytes,
1112 memcmp(raw_btf, user_btf, expected_nbytes));
1113 err = -1;
1114 goto done;
1115 }
1116
1117 while (expected_nbytes < raw_btf_size) {
1118 fprintf(stderr, "%u...", expected_nbytes);
1119 if (user_btf[expected_nbytes++] != 0xff) {
1120 fprintf(stderr, "!= 0xff\n");
1121 err = -1;
1122 goto done;
1123 }
1124 }
1125
1126 fprintf(stderr, "OK\n");
1127
1128done:
1129 if (*btf_log_buf && (err || args.always_log))
1130 fprintf(stderr, "%s\n", btf_log_buf);
1131
1132 free(raw_btf);
1133 free(user_btf);
1134
1135 if (btf_fd != -1)
1136 close(btf_fd);
1137
1138 return err;
1139}
1140
1141static int test_get_info(void)
1142{
1143 unsigned int i;
1144 int err = 0;
1145
1146 if (args.get_info_test_num)
1147 return do_test_get_info(args.get_info_test_num);
1148
1149 for (i = 1; i <= ARRAY_SIZE(get_info_tests); i++)
1150 err |= do_test_get_info(i);
1151
1152 return err;
1153}
1154
1155struct btf_file_test {
1156 const char *file;
1157 bool btf_kv_notfound;
1158};
1159
1160static struct btf_file_test file_tests[] = {
1161{
1162 .file = "test_btf_haskv.o",
1163},
1164{
1165 .file = "test_btf_nokv.o",
1166 .btf_kv_notfound = true,
1167},
1168};
1169
1170static int file_has_btf_elf(const char *fn)
1171{
1172 Elf_Scn *scn = NULL;
1173 GElf_Ehdr ehdr;
1174 int elf_fd;
1175 Elf *elf;
1176 int ret;
1177
1178 if (elf_version(EV_CURRENT) == EV_NONE) {
1179 fprintf(stderr, "Failed to init libelf\n");
1180 return -1;
1181 }
1182
1183 elf_fd = open(fn, O_RDONLY);
1184 if (elf_fd == -1) {
1185 fprintf(stderr, "Cannot open file %s: %s(%d)\n",
1186 fn, strerror(errno), errno);
1187 return -1;
1188 }
1189
1190 elf = elf_begin(elf_fd, ELF_C_READ, NULL);
1191 if (!elf) {
1192 fprintf(stderr, "Failed to read ELF from %s. %s\n", fn,
1193 elf_errmsg(elf_errno()));
1194 ret = -1;
1195 goto done;
1196 }
1197
1198 if (!gelf_getehdr(elf, &ehdr)) {
1199 fprintf(stderr, "Failed to get EHDR from %s\n", fn);
1200 ret = -1;
1201 goto done;
1202 }
1203
1204 while ((scn = elf_nextscn(elf, scn))) {
1205 const char *sh_name;
1206 GElf_Shdr sh;
1207
1208 if (gelf_getshdr(scn, &sh) != &sh) {
1209 fprintf(stderr,
1210 "Failed to get section header from %s\n", fn);
1211 ret = -1;
1212 goto done;
1213 }
1214
1215 sh_name = elf_strptr(elf, ehdr.e_shstrndx, sh.sh_name);
1216 if (!strcmp(sh_name, BTF_ELF_SEC)) {
1217 ret = 1;
1218 goto done;
1219 }
1220 }
1221
1222 ret = 0;
1223
1224done:
1225 close(elf_fd);
1226 elf_end(elf);
1227 return ret;
1228}
1229
1230static int do_test_file(unsigned int test_num)
1231{
1232 const struct btf_file_test *test = &file_tests[test_num - 1];
1233 struct bpf_object *obj = NULL;
1234 struct bpf_program *prog;
1235 struct bpf_map *map;
1236 int err;
1237
1238 fprintf(stderr, "BTF libbpf test[%u] (%s): ", test_num,
1239 test->file);
1240
1241 err = file_has_btf_elf(test->file);
1242 if (err == -1)
1243 return err;
1244
1245 if (err == 0) {
1246 fprintf(stderr, "SKIP. No ELF %s found\n", BTF_ELF_SEC);
1247 return 0;
1248 }
1249
1250 obj = bpf_object__open(test->file);
1251 if (IS_ERR(obj))
1252 return PTR_ERR(obj);
1253
1254 err = bpf_object__btf_fd(obj);
1255 if (err == -1) {
1256 fprintf(stderr, "bpf_object__btf_fd: -1\n");
1257 goto done;
1258 }
1259
1260 prog = bpf_program__next(NULL, obj);
1261 if (!prog) {
1262 fprintf(stderr, "Cannot find bpf_prog\n");
1263 err = -1;
1264 goto done;
1265 }
1266
1267 bpf_program__set_type(prog, BPF_PROG_TYPE_TRACEPOINT);
1268 err = bpf_object__load(obj);
1269 if (err < 0) {
1270 fprintf(stderr, "bpf_object__load: %d\n", err);
1271 goto done;
1272 }
1273
1274 map = bpf_object__find_map_by_name(obj, "btf_map");
1275 if (!map) {
1276 fprintf(stderr, "btf_map not found\n");
1277 err = -1;
1278 goto done;
1279 }
1280
1281 err = (bpf_map__btf_key_id(map) == 0 || bpf_map__btf_value_id(map) == 0)
1282 != test->btf_kv_notfound;
1283 if (err) {
1284 fprintf(stderr,
1285 "btf_kv_notfound:%u btf_key_id:%u btf_value_id:%u\n",
1286 test->btf_kv_notfound,
1287 bpf_map__btf_key_id(map),
1288 bpf_map__btf_value_id(map));
1289 goto done;
1290 }
1291
1292 fprintf(stderr, "OK\n");
1293
1294done:
1295 bpf_object__close(obj);
1296 return err;
1297}
1298
1299static int test_file(void)
1300{
1301 unsigned int i;
1302 int err = 0;
1303
1304 if (args.file_test_num)
1305 return do_test_file(args.file_test_num);
1306
1307 for (i = 1; i <= ARRAY_SIZE(file_tests); i++)
1308 err |= do_test_file(i);
1309
1310 return err;
1311}
1312
1313const char *pprint_enum_str[] = {
1314 "ENUM_ZERO",
1315 "ENUM_ONE",
1316 "ENUM_TWO",
1317 "ENUM_THREE",
1318};
1319
1320struct pprint_mapv {
1321 uint32_t ui32;
1322 uint16_t ui16;
1323 /* 2 bytes hole */
1324 int32_t si32;
1325 uint32_t unused_bits2a:2,
1326 bits28:28,
1327 unused_bits2b:2;
1328 union {
1329 uint64_t ui64;
1330 uint8_t ui8a[8];
1331 };
1332 enum {
1333 ENUM_ZERO,
1334 ENUM_ONE,
1335 ENUM_TWO,
1336 ENUM_THREE,
1337 } aenum;
1338};
1339
1340static struct btf_raw_test pprint_test = {
1341 .descr = "BTF pretty print test #1",
1342 .raw_types = {
1343 /* unsighed char */ /* [1] */
1344 BTF_TYPE_INT_ENC(NAME_TBD, 0, 0, 8, 1),
1345 /* unsigned short */ /* [2] */
1346 BTF_TYPE_INT_ENC(NAME_TBD, 0, 0, 16, 2),
1347 /* unsigned int */ /* [3] */
1348 BTF_TYPE_INT_ENC(NAME_TBD, 0, 0, 32, 4),
1349 /* int */ /* [4] */
1350 BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4),
1351 /* unsigned long long */ /* [5] */
1352 BTF_TYPE_INT_ENC(NAME_TBD, 0, 0, 64, 8),
1353 /* 2 bits */ /* [6] */
1354 BTF_TYPE_INT_ENC(0, 0, 0, 2, 2),
1355 /* 28 bits */ /* [7] */
1356 BTF_TYPE_INT_ENC(0, 0, 0, 28, 4),
1357 /* uint8_t[8] */ /* [8] */
1358 BTF_TYPE_ARRAY_ENC(9, 3, 8),
1359 /* typedef unsigned char uint8_t */ /* [9] */
1360 BTF_TYPEDEF_ENC(NAME_TBD, 1),
1361 /* typedef unsigned short uint16_t */ /* [10] */
1362 BTF_TYPEDEF_ENC(NAME_TBD, 2),
1363 /* typedef unsigned int uint32_t */ /* [11] */
1364 BTF_TYPEDEF_ENC(NAME_TBD, 3),
1365 /* typedef int int32_t */ /* [12] */
1366 BTF_TYPEDEF_ENC(NAME_TBD, 4),
1367 /* typedef unsigned long long uint64_t *//* [13] */
1368 BTF_TYPEDEF_ENC(NAME_TBD, 5),
1369 /* union (anon) */ /* [14] */
1370 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_UNION, 0, 2), 8),
1371 BTF_MEMBER_ENC(NAME_TBD, 13, 0),/* uint64_t ui64; */
1372 BTF_MEMBER_ENC(NAME_TBD, 8, 0), /* uint8_t ui8a[8]; */
1373 /* enum (anon) */ /* [15] */
1374 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_ENUM, 0, 4), 4),
1375 BTF_ENUM_ENC(NAME_TBD, 0),
1376 BTF_ENUM_ENC(NAME_TBD, 1),
1377 BTF_ENUM_ENC(NAME_TBD, 2),
1378 BTF_ENUM_ENC(NAME_TBD, 3),
1379 /* struct pprint_mapv */ /* [16] */
1380 BTF_TYPE_ENC(NAME_TBD, BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 8), 28),
1381 BTF_MEMBER_ENC(NAME_TBD, 11, 0), /* uint32_t ui32 */
1382 BTF_MEMBER_ENC(NAME_TBD, 10, 32), /* uint16_t ui16 */
1383 BTF_MEMBER_ENC(NAME_TBD, 12, 64), /* int32_t si32 */
1384 BTF_MEMBER_ENC(NAME_TBD, 6, 96), /* unused_bits2a */
1385 BTF_MEMBER_ENC(NAME_TBD, 7, 98), /* bits28 */
1386 BTF_MEMBER_ENC(NAME_TBD, 6, 126), /* unused_bits2b */
1387 BTF_MEMBER_ENC(0, 14, 128), /* union (anon) */
1388 BTF_MEMBER_ENC(NAME_TBD, 15, 192), /* aenum */
1389 BTF_END_RAW,
1390 },
1391 .str_sec = "\0unsigned char\0unsigned short\0unsigned int\0int\0unsigned long long\0uint8_t\0uint16_t\0uint32_t\0int32_t\0uint64_t\0ui64\0ui8a\0ENUM_ZERO\0ENUM_ONE\0ENUM_TWO\0ENUM_THREE\0pprint_mapv\0ui32\0ui16\0si32\0unused_bits2a\0bits28\0unused_bits2b\0aenum",
1392 .str_sec_size = sizeof("\0unsigned char\0unsigned short\0unsigned int\0int\0unsigned long long\0uint8_t\0uint16_t\0uint32_t\0int32_t\0uint64_t\0ui64\0ui8a\0ENUM_ZERO\0ENUM_ONE\0ENUM_TWO\0ENUM_THREE\0pprint_mapv\0ui32\0ui16\0si32\0unused_bits2a\0bits28\0unused_bits2b\0aenum"),
1393 .map_type = BPF_MAP_TYPE_ARRAY,
1394 .map_name = "pprint_test",
1395 .key_size = sizeof(unsigned int),
1396 .value_size = sizeof(struct pprint_mapv),
1397 .key_id = 3, /* unsigned int */
1398 .value_id = 16, /* struct pprint_mapv */
1399 .max_entries = 128 * 1024,
1400};
1401
1402static void set_pprint_mapv(struct pprint_mapv *v, uint32_t i)
1403{
1404 v->ui32 = i;
1405 v->si32 = -i;
1406 v->unused_bits2a = 3;
1407 v->bits28 = i;
1408 v->unused_bits2b = 3;
1409 v->ui64 = i;
1410 v->aenum = i & 0x03;
1411}
1412
1413static int test_pprint(void)
1414{
1415 const struct btf_raw_test *test = &pprint_test;
1416 struct bpf_create_map_attr create_attr = {};
1417 int map_fd = -1, btf_fd = -1;
1418 struct pprint_mapv mapv = {};
1419 unsigned int raw_btf_size;
1420 char expected_line[255];
1421 FILE *pin_file = NULL;
1422 char pin_path[255];
1423 size_t line_len = 0;
1424 char *line = NULL;
1425 unsigned int key;
1426 uint8_t *raw_btf;
1427 ssize_t nread;
1428 int err;
1429
1430 fprintf(stderr, "%s......", test->descr);
1431 raw_btf = btf_raw_create(&hdr_tmpl, test->raw_types,
1432 test->str_sec, test->str_sec_size,
1433 &raw_btf_size);
1434
1435 if (!raw_btf)
1436 return -1;
1437
1438 *btf_log_buf = '\0';
1439 btf_fd = bpf_load_btf(raw_btf, raw_btf_size,
1440 btf_log_buf, BTF_LOG_BUF_SIZE,
1441 args.always_log);
1442 free(raw_btf);
1443
1444 if (btf_fd == -1) {
1445 err = -1;
1446 fprintf(stderr, "bpf_load_btf: %s(%d)\n",
1447 strerror(errno), errno);
1448 goto done;
1449 }
1450
1451 create_attr.name = test->map_name;
1452 create_attr.map_type = test->map_type;
1453 create_attr.key_size = test->key_size;
1454 create_attr.value_size = test->value_size;
1455 create_attr.max_entries = test->max_entries;
1456 create_attr.btf_fd = btf_fd;
1457 create_attr.btf_key_id = test->key_id;
1458 create_attr.btf_value_id = test->value_id;
1459
1460 map_fd = bpf_create_map_xattr(&create_attr);
1461 if (map_fd == -1) {
1462 err = -1;
1463 fprintf(stderr, "bpf_creat_map_btf: %s(%d)\n",
1464 strerror(errno), errno);
1465 goto done;
1466 }
1467
1468 if (snprintf(pin_path, sizeof(pin_path), "%s/%s",
1469 "/sys/fs/bpf", test->map_name) == sizeof(pin_path)) {
1470 err = -1;
1471 fprintf(stderr, "pin_path is too long\n");
1472 goto done;
1473 }
1474
1475 err = bpf_obj_pin(map_fd, pin_path);
1476 if (err) {
1477 fprintf(stderr, "Cannot pin to %s. %s(%d).\n", pin_path,
1478 strerror(errno), errno);
1479 goto done;
1480 }
1481
1482 for (key = 0; key < test->max_entries; key++) {
1483 set_pprint_mapv(&mapv, key);
1484 bpf_map_update_elem(map_fd, &key, &mapv, 0);
1485 }
1486
1487 pin_file = fopen(pin_path, "r");
1488 if (!pin_file) {
1489 err = -1;
1490 fprintf(stderr, "fopen(%s): %s(%d)\n", pin_path,
1491 strerror(errno), errno);
1492 goto done;
1493 }
1494
1495 /* Skip lines start with '#' */
1496 while ((nread = getline(&line, &line_len, pin_file)) > 0 &&
1497 *line == '#')
1498 ;
1499
1500 if (nread <= 0) {
1501 err = -1;
1502 fprintf(stderr, "Unexpected EOF\n");
1503 goto done;
1504 }
1505
1506 key = 0;
1507 do {
1508 ssize_t nexpected_line;
1509
1510 set_pprint_mapv(&mapv, key);
1511 nexpected_line = snprintf(expected_line, sizeof(expected_line),
1512 "%u: {%u,0,%d,0x%x,0x%x,0x%x,{%lu|[%u,%u,%u,%u,%u,%u,%u,%u]},%s}\n",
1513 key,
1514 mapv.ui32, mapv.si32,
1515 mapv.unused_bits2a, mapv.bits28, mapv.unused_bits2b,
1516 mapv.ui64,
1517 mapv.ui8a[0], mapv.ui8a[1], mapv.ui8a[2], mapv.ui8a[3],
1518 mapv.ui8a[4], mapv.ui8a[5], mapv.ui8a[6], mapv.ui8a[7],
1519 pprint_enum_str[mapv.aenum]);
1520
1521 if (nexpected_line == sizeof(expected_line)) {
1522 err = -1;
1523 fprintf(stderr, "expected_line is too long\n");
1524 goto done;
1525 }
1526
1527 if (strcmp(expected_line, line)) {
1528 err = -1;
1529 fprintf(stderr, "unexpected pprint output\n");
1530 fprintf(stderr, "expected: %s", expected_line);
1531 fprintf(stderr, " read: %s", line);
1532 goto done;
1533 }
1534
1535 nread = getline(&line, &line_len, pin_file);
1536 } while (++key < test->max_entries && nread > 0);
1537
1538 if (key < test->max_entries) {
1539 err = -1;
1540 fprintf(stderr, "Unexpected EOF\n");
1541 goto done;
1542 }
1543
1544 if (nread > 0) {
1545 err = -1;
1546 fprintf(stderr, "Unexpected extra pprint output: %s\n", line);
1547 goto done;
1548 }
1549
1550 err = 0;
1551
1552done:
1553 if (!err)
1554 fprintf(stderr, "OK\n");
1555 if (*btf_log_buf && (err || args.always_log))
1556 fprintf(stderr, "%s\n", btf_log_buf);
1557 if (btf_fd != -1)
1558 close(btf_fd);
1559 if (map_fd != -1)
1560 close(map_fd);
1561 if (pin_file)
1562 fclose(pin_file);
1563 unlink(pin_path);
1564 free(line);
1565
1566 return err;
1567}
1568
1569static void usage(const char *cmd)
1570{
1571 fprintf(stderr, "Usage: %s [-l] [[-r test_num (1 - %zu)] | [-g test_num (1 - %zu)] | [-f test_num (1 - %zu)] | [-p]]\n",
1572 cmd, ARRAY_SIZE(raw_tests), ARRAY_SIZE(get_info_tests),
1573 ARRAY_SIZE(file_tests));
1574}
1575
1576static int parse_args(int argc, char **argv)
1577{
1578 const char *optstr = "lpf:r:g:";
1579 int opt;
1580
1581 while ((opt = getopt(argc, argv, optstr)) != -1) {
1582 switch (opt) {
1583 case 'l':
1584 args.always_log = true;
1585 break;
1586 case 'f':
1587 args.file_test_num = atoi(optarg);
1588 args.file_test = true;
1589 break;
1590 case 'r':
1591 args.raw_test_num = atoi(optarg);
1592 args.raw_test = true;
1593 break;
1594 case 'g':
1595 args.get_info_test_num = atoi(optarg);
1596 args.get_info_test = true;
1597 break;
1598 case 'p':
1599 args.pprint_test = true;
1600 break;
1601 case 'h':
1602 usage(argv[0]);
1603 exit(0);
1604 default:
1605 usage(argv[0]);
1606 return -1;
1607 }
1608 }
1609
1610 if (args.raw_test_num &&
1611 (args.raw_test_num < 1 ||
1612 args.raw_test_num > ARRAY_SIZE(raw_tests))) {
1613 fprintf(stderr, "BTF raw test number must be [1 - %zu]\n",
1614 ARRAY_SIZE(raw_tests));
1615 return -1;
1616 }
1617
1618 if (args.file_test_num &&
1619 (args.file_test_num < 1 ||
1620 args.file_test_num > ARRAY_SIZE(file_tests))) {
1621 fprintf(stderr, "BTF file test number must be [1 - %zu]\n",
1622 ARRAY_SIZE(file_tests));
1623 return -1;
1624 }
1625
1626 if (args.get_info_test_num &&
1627 (args.get_info_test_num < 1 ||
1628 args.get_info_test_num > ARRAY_SIZE(get_info_tests))) {
1629 fprintf(stderr, "BTF get info test number must be [1 - %zu]\n",
1630 ARRAY_SIZE(get_info_tests));
1631 return -1;
1632 }
1633
1634 return 0;
1635}
1636
1637int main(int argc, char **argv)
1638{
1639 int err = 0;
1640
1641 err = parse_args(argc, argv);
1642 if (err)
1643 return err;
1644
1645 if (args.always_log)
1646 libbpf_set_print(__base_pr, __base_pr, __base_pr);
1647
1648 if (args.raw_test)
1649 err |= test_raw();
1650
1651 if (args.get_info_test)
1652 err |= test_get_info();
1653
1654 if (args.file_test)
1655 err |= test_file();
1656
1657 if (args.pprint_test)
1658 err |= test_pprint();
1659
1660 if (args.raw_test || args.get_info_test || args.file_test ||
1661 args.pprint_test)
1662 return err;
1663
1664 err |= test_raw();
1665 err |= test_get_info();
1666 err |= test_file();
1667
1668 return err;
1669}
diff --git a/tools/testing/selftests/bpf/test_btf_haskv.c b/tools/testing/selftests/bpf/test_btf_haskv.c
new file mode 100644
index 000000000000..8c7ca096ecf2
--- /dev/null
+++ b/tools/testing/selftests/bpf/test_btf_haskv.c
@@ -0,0 +1,48 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/* Copyright (c) 2018 Facebook */
3#include <linux/bpf.h>
4#include "bpf_helpers.h"
5
6int _version SEC("version") = 1;
7
8struct ipv_counts {
9 unsigned int v4;
10 unsigned int v6;
11};
12
13typedef int btf_map_key;
14typedef struct ipv_counts btf_map_value;
15btf_map_key dumm_key;
16btf_map_value dummy_value;
17
18struct bpf_map_def SEC("maps") btf_map = {
19 .type = BPF_MAP_TYPE_ARRAY,
20 .key_size = sizeof(int),
21 .value_size = sizeof(struct ipv_counts),
22 .max_entries = 4,
23};
24
25struct dummy_tracepoint_args {
26 unsigned long long pad;
27 struct sock *sock;
28};
29
30SEC("dummy_tracepoint")
31int _dummy_tracepoint(struct dummy_tracepoint_args *arg)
32{
33 struct ipv_counts *counts;
34 int key = 0;
35
36 if (!arg->sock)
37 return 0;
38
39 counts = bpf_map_lookup_elem(&btf_map, &key);
40 if (!counts)
41 return 0;
42
43 counts->v6++;
44
45 return 0;
46}
47
48char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/test_btf_nokv.c b/tools/testing/selftests/bpf/test_btf_nokv.c
new file mode 100644
index 000000000000..0ed8e088eebf
--- /dev/null
+++ b/tools/testing/selftests/bpf/test_btf_nokv.c
@@ -0,0 +1,43 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/* Copyright (c) 2018 Facebook */
3#include <linux/bpf.h>
4#include "bpf_helpers.h"
5
6int _version SEC("version") = 1;
7
8struct ipv_counts {
9 unsigned int v4;
10 unsigned int v6;
11};
12
13struct bpf_map_def SEC("maps") btf_map = {
14 .type = BPF_MAP_TYPE_ARRAY,
15 .key_size = sizeof(int),
16 .value_size = sizeof(struct ipv_counts),
17 .max_entries = 4,
18};
19
20struct dummy_tracepoint_args {
21 unsigned long long pad;
22 struct sock *sock;
23};
24
25SEC("dummy_tracepoint")
26int _dummy_tracepoint(struct dummy_tracepoint_args *arg)
27{
28 struct ipv_counts *counts;
29 int key = 0;
30
31 if (!arg->sock)
32 return 0;
33
34 counts = bpf_map_lookup_elem(&btf_map, &key);
35 if (!counts)
36 return 0;
37
38 counts->v6++;
39
40 return 0;
41}
42
43char _license[] SEC("license") = "GPL";
generated by cgit v1.2.2 (git 2.25.0) at 2025-08-30 18:07:52 -0400