aboutsummaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/bpf/Makefile1
-rw-r--r--kernel/bpf/arraymap.c50
-rw-r--r--kernel/bpf/btf.c2064
-rw-r--r--kernel/bpf/inode.c156
-rw-r--r--kernel/bpf/syscall.c51
5 files changed, 2318 insertions, 4 deletions
diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile
index a713fd23ec88..35c485fa9ea3 100644
--- a/kernel/bpf/Makefile
+++ b/kernel/bpf/Makefile
@@ -4,6 +4,7 @@ obj-y := core.o
4obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o 4obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o
5obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o 5obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o
6obj-$(CONFIG_BPF_SYSCALL) += disasm.o 6obj-$(CONFIG_BPF_SYSCALL) += disasm.o
7obj-$(CONFIG_BPF_SYSCALL) += btf.o
7ifeq ($(CONFIG_NET),y) 8ifeq ($(CONFIG_NET),y)
8obj-$(CONFIG_BPF_SYSCALL) += devmap.o 9obj-$(CONFIG_BPF_SYSCALL) += devmap.o
9obj-$(CONFIG_BPF_SYSCALL) += cpumap.o 10obj-$(CONFIG_BPF_SYSCALL) += cpumap.o
diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c
index 14750e7c5ee4..02a189339381 100644
--- a/kernel/bpf/arraymap.c
+++ b/kernel/bpf/arraymap.c
@@ -11,11 +11,13 @@
11 * General Public License for more details. 11 * General Public License for more details.
12 */ 12 */
13#include <linux/bpf.h> 13#include <linux/bpf.h>
14#include <linux/btf.h>
14#include <linux/err.h> 15#include <linux/err.h>
15#include <linux/slab.h> 16#include <linux/slab.h>
16#include <linux/mm.h> 17#include <linux/mm.h>
17#include <linux/filter.h> 18#include <linux/filter.h>
18#include <linux/perf_event.h> 19#include <linux/perf_event.h>
20#include <uapi/linux/btf.h>
19 21
20#include "map_in_map.h" 22#include "map_in_map.h"
21 23
@@ -336,6 +338,52 @@ static void array_map_free(struct bpf_map *map)
336 bpf_map_area_free(array); 338 bpf_map_area_free(array);
337} 339}
338 340
341static void array_map_seq_show_elem(struct bpf_map *map, void *key,
342 struct seq_file *m)
343{
344 void *value;
345
346 rcu_read_lock();
347
348 value = array_map_lookup_elem(map, key);
349 if (!value) {
350 rcu_read_unlock();
351 return;
352 }
353
354 seq_printf(m, "%u: ", *(u32 *)key);
355 btf_type_seq_show(map->btf, map->btf_value_id, value, m);
356 seq_puts(m, "\n");
357
358 rcu_read_unlock();
359}
360
361static int array_map_check_btf(const struct bpf_map *map, const struct btf *btf,
362 u32 btf_key_id, u32 btf_value_id)
363{
364 const struct btf_type *key_type, *value_type;
365 u32 key_size, value_size;
366 u32 int_data;
367
368 key_type = btf_type_id_size(btf, &btf_key_id, &key_size);
369 if (!key_type || BTF_INFO_KIND(key_type->info) != BTF_KIND_INT)
370 return -EINVAL;
371
372 int_data = *(u32 *)(key_type + 1);
373 /* bpf array can only take a u32 key. This check makes
374 * sure that the btf matches the attr used during map_create.
375 */
376 if (BTF_INT_BITS(int_data) != 32 || key_size != 4 ||
377 BTF_INT_OFFSET(int_data))
378 return -EINVAL;
379
380 value_type = btf_type_id_size(btf, &btf_value_id, &value_size);
381 if (!value_type || value_size > map->value_size)
382 return -EINVAL;
383
384 return 0;
385}
386
339const struct bpf_map_ops array_map_ops = { 387const struct bpf_map_ops array_map_ops = {
340 .map_alloc_check = array_map_alloc_check, 388 .map_alloc_check = array_map_alloc_check,
341 .map_alloc = array_map_alloc, 389 .map_alloc = array_map_alloc,
@@ -345,6 +393,8 @@ const struct bpf_map_ops array_map_ops = {
345 .map_update_elem = array_map_update_elem, 393 .map_update_elem = array_map_update_elem,
346 .map_delete_elem = array_map_delete_elem, 394 .map_delete_elem = array_map_delete_elem,
347 .map_gen_lookup = array_map_gen_lookup, 395 .map_gen_lookup = array_map_gen_lookup,
396 .map_seq_show_elem = array_map_seq_show_elem,
397 .map_check_btf = array_map_check_btf,
348}; 398};
349 399
350const struct bpf_map_ops percpu_array_map_ops = { 400const struct bpf_map_ops percpu_array_map_ops = {
diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c
new file mode 100644
index 000000000000..eb56ac760547
--- /dev/null
+++ b/kernel/bpf/btf.c
@@ -0,0 +1,2064 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/* Copyright (c) 2018 Facebook */
3
4#include <uapi/linux/btf.h>
5#include <uapi/linux/types.h>
6#include <linux/seq_file.h>
7#include <linux/compiler.h>
8#include <linux/errno.h>
9#include <linux/slab.h>
10#include <linux/anon_inodes.h>
11#include <linux/file.h>
12#include <linux/uaccess.h>
13#include <linux/kernel.h>
14#include <linux/bpf_verifier.h>
15#include <linux/btf.h>
16
17/* BTF (BPF Type Format) is the meta data format which describes
18 * the data types of BPF program/map. Hence, it basically focus
19 * on the C programming language which the modern BPF is primary
20 * using.
21 *
22 * ELF Section:
23 * ~~~~~~~~~~~
24 * The BTF data is stored under the ".BTF" ELF section
25 *
26 * struct btf_type:
27 * ~~~~~~~~~~~~~~~
28 * Each 'struct btf_type' object describes a C data type.
29 * Depending on the type it is describing, a 'struct btf_type'
30 * object may be followed by more data. F.e.
31 * To describe an array, 'struct btf_type' is followed by
32 * 'struct btf_array'.
33 *
34 * 'struct btf_type' and any extra data following it are
35 * 4 bytes aligned.
36 *
37 * Type section:
38 * ~~~~~~~~~~~~~
39 * The BTF type section contains a list of 'struct btf_type' objects.
40 * Each one describes a C type. Recall from the above section
41 * that a 'struct btf_type' object could be immediately followed by extra
42 * data in order to desribe some particular C types.
43 *
44 * type_id:
45 * ~~~~~~~
46 * Each btf_type object is identified by a type_id. The type_id
47 * is implicitly implied by the location of the btf_type object in
48 * the BTF type section. The first one has type_id 1. The second
49 * one has type_id 2...etc. Hence, an earlier btf_type has
50 * a smaller type_id.
51 *
52 * A btf_type object may refer to another btf_type object by using
53 * type_id (i.e. the "type" in the "struct btf_type").
54 *
55 * NOTE that we cannot assume any reference-order.
56 * A btf_type object can refer to an earlier btf_type object
57 * but it can also refer to a later btf_type object.
58 *
59 * For example, to describe "const void *". A btf_type
60 * object describing "const" may refer to another btf_type
61 * object describing "void *". This type-reference is done
62 * by specifying type_id:
63 *
64 * [1] CONST (anon) type_id=2
65 * [2] PTR (anon) type_id=0
66 *
67 * The above is the btf_verifier debug log:
68 * - Each line started with "[?]" is a btf_type object
69 * - [?] is the type_id of the btf_type object.
70 * - CONST/PTR is the BTF_KIND_XXX
71 * - "(anon)" is the name of the type. It just
72 * happens that CONST and PTR has no name.
73 * - type_id=XXX is the 'u32 type' in btf_type
74 *
75 * NOTE: "void" has type_id 0
76 *
77 * String section:
78 * ~~~~~~~~~~~~~~
79 * The BTF string section contains the names used by the type section.
80 * Each string is referred by an "offset" from the beginning of the
81 * string section.
82 *
83 * Each string is '\0' terminated.
84 *
85 * The first character in the string section must be '\0'
86 * which is used to mean 'anonymous'. Some btf_type may not
87 * have a name.
88 */
89
90/* BTF verification:
91 *
92 * To verify BTF data, two passes are needed.
93 *
94 * Pass #1
95 * ~~~~~~~
96 * The first pass is to collect all btf_type objects to
97 * an array: "btf->types".
98 *
99 * Depending on the C type that a btf_type is describing,
100 * a btf_type may be followed by extra data. We don't know
101 * how many btf_type is there, and more importantly we don't
102 * know where each btf_type is located in the type section.
103 *
104 * Without knowing the location of each type_id, most verifications
105 * cannot be done. e.g. an earlier btf_type may refer to a later
106 * btf_type (recall the "const void *" above), so we cannot
107 * check this type-reference in the first pass.
108 *
109 * In the first pass, it still does some verifications (e.g.
110 * checking the name is a valid offset to the string section).
111 *
112 * Pass #2
113 * ~~~~~~~
114 * The main focus is to resolve a btf_type that is referring
115 * to another type.
116 *
117 * We have to ensure the referring type:
118 * 1) does exist in the BTF (i.e. in btf->types[])
119 * 2) does not cause a loop:
120 * struct A {
121 * struct B b;
122 * };
123 *
124 * struct B {
125 * struct A a;
126 * };
127 *
128 * btf_type_needs_resolve() decides if a btf_type needs
129 * to be resolved.
130 *
131 * The needs_resolve type implements the "resolve()" ops which
132 * essentially does a DFS and detects backedge.
133 *
134 * During resolve (or DFS), different C types have different
135 * "RESOLVED" conditions.
136 *
137 * When resolving a BTF_KIND_STRUCT, we need to resolve all its
138 * members because a member is always referring to another
139 * type. A struct's member can be treated as "RESOLVED" if
140 * it is referring to a BTF_KIND_PTR. Otherwise, the
141 * following valid C struct would be rejected:
142 *
143 * struct A {
144 * int m;
145 * struct A *a;
146 * };
147 *
148 * When resolving a BTF_KIND_PTR, it needs to keep resolving if
149 * it is referring to another BTF_KIND_PTR. Otherwise, we cannot
150 * detect a pointer loop, e.g.:
151 * BTF_KIND_CONST -> BTF_KIND_PTR -> BTF_KIND_CONST -> BTF_KIND_PTR +
152 * ^ |
153 * +-----------------------------------------+
154 *
155 */
156
157#define BITS_PER_U64 (sizeof(u64) * BITS_PER_BYTE)
158#define BITS_PER_BYTE_MASK (BITS_PER_BYTE - 1)
159#define BITS_PER_BYTE_MASKED(bits) ((bits) & BITS_PER_BYTE_MASK)
160#define BITS_ROUNDDOWN_BYTES(bits) ((bits) >> 3)
161#define BITS_ROUNDUP_BYTES(bits) \
162 (BITS_ROUNDDOWN_BYTES(bits) + !!BITS_PER_BYTE_MASKED(bits))
163
164/* 16MB for 64k structs and each has 16 members and
165 * a few MB spaces for the string section.
166 * The hard limit is S32_MAX.
167 */
168#define BTF_MAX_SIZE (16 * 1024 * 1024)
169/* 64k. We can raise it later. The hard limit is S32_MAX. */
170#define BTF_MAX_NR_TYPES 65535
171
172#define for_each_member(i, struct_type, member) \
173 for (i = 0, member = btf_type_member(struct_type); \
174 i < btf_type_vlen(struct_type); \
175 i++, member++)
176
177#define for_each_member_from(i, from, struct_type, member) \
178 for (i = from, member = btf_type_member(struct_type) + from; \
179 i < btf_type_vlen(struct_type); \
180 i++, member++)
181
182struct btf {
183 union {
184 struct btf_header *hdr;
185 void *data;
186 };
187 struct btf_type **types;
188 u32 *resolved_ids;
189 u32 *resolved_sizes;
190 const char *strings;
191 void *nohdr_data;
192 u32 nr_types;
193 u32 types_size;
194 u32 data_size;
195 refcount_t refcnt;
196};
197
198enum verifier_phase {
199 CHECK_META,
200 CHECK_TYPE,
201};
202
203struct resolve_vertex {
204 const struct btf_type *t;
205 u32 type_id;
206 u16 next_member;
207};
208
209enum visit_state {
210 NOT_VISITED,
211 VISITED,
212 RESOLVED,
213};
214
215enum resolve_mode {
216 RESOLVE_TBD, /* To Be Determined */
217 RESOLVE_PTR, /* Resolving for Pointer */
218 RESOLVE_STRUCT_OR_ARRAY, /* Resolving for struct/union
219 * or array
220 */
221};
222
223#define MAX_RESOLVE_DEPTH 32
224
225struct btf_verifier_env {
226 struct btf *btf;
227 u8 *visit_states;
228 struct resolve_vertex stack[MAX_RESOLVE_DEPTH];
229 struct bpf_verifier_log log;
230 u32 log_type_id;
231 u32 top_stack;
232 enum verifier_phase phase;
233 enum resolve_mode resolve_mode;
234};
235
236static const char * const btf_kind_str[NR_BTF_KINDS] = {
237 [BTF_KIND_UNKN] = "UNKNOWN",
238 [BTF_KIND_INT] = "INT",
239 [BTF_KIND_PTR] = "PTR",
240 [BTF_KIND_ARRAY] = "ARRAY",
241 [BTF_KIND_STRUCT] = "STRUCT",
242 [BTF_KIND_UNION] = "UNION",
243 [BTF_KIND_ENUM] = "ENUM",
244 [BTF_KIND_FWD] = "FWD",
245 [BTF_KIND_TYPEDEF] = "TYPEDEF",
246 [BTF_KIND_VOLATILE] = "VOLATILE",
247 [BTF_KIND_CONST] = "CONST",
248 [BTF_KIND_RESTRICT] = "RESTRICT",
249};
250
251struct btf_kind_operations {
252 s32 (*check_meta)(struct btf_verifier_env *env,
253 const struct btf_type *t,
254 u32 meta_left);
255 int (*resolve)(struct btf_verifier_env *env,
256 const struct resolve_vertex *v);
257 int (*check_member)(struct btf_verifier_env *env,
258 const struct btf_type *struct_type,
259 const struct btf_member *member,
260 const struct btf_type *member_type);
261 void (*log_details)(struct btf_verifier_env *env,
262 const struct btf_type *t);
263 void (*seq_show)(const struct btf *btf, const struct btf_type *t,
264 u32 type_id, void *data, u8 bits_offsets,
265 struct seq_file *m);
266};
267
268static const struct btf_kind_operations * const kind_ops[NR_BTF_KINDS];
269static struct btf_type btf_void;
270
271static bool btf_type_is_modifier(const struct btf_type *t)
272{
273 /* Some of them is not strictly a C modifier
274 * but they are grouped into the same bucket
275 * for BTF concern:
276 * A type (t) that refers to another
277 * type through t->type AND its size cannot
278 * be determined without following the t->type.
279 *
280 * ptr does not fall into this bucket
281 * because its size is always sizeof(void *).
282 */
283 switch (BTF_INFO_KIND(t->info)) {
284 case BTF_KIND_TYPEDEF:
285 case BTF_KIND_VOLATILE:
286 case BTF_KIND_CONST:
287 case BTF_KIND_RESTRICT:
288 return true;
289 }
290
291 return false;
292}
293
294static bool btf_type_is_void(const struct btf_type *t)
295{
296 /* void => no type and size info.
297 * Hence, FWD is also treated as void.
298 */
299 return t == &btf_void || BTF_INFO_KIND(t->info) == BTF_KIND_FWD;
300}
301
302static bool btf_type_is_void_or_null(const struct btf_type *t)
303{
304 return !t || btf_type_is_void(t);
305}
306
307/* union is only a special case of struct:
308 * all its offsetof(member) == 0
309 */
310static bool btf_type_is_struct(const struct btf_type *t)
311{
312 u8 kind = BTF_INFO_KIND(t->info);
313
314 return kind == BTF_KIND_STRUCT || kind == BTF_KIND_UNION;
315}
316
317static bool btf_type_is_array(const struct btf_type *t)
318{
319 return BTF_INFO_KIND(t->info) == BTF_KIND_ARRAY;
320}
321
322static bool btf_type_is_ptr(const struct btf_type *t)
323{
324 return BTF_INFO_KIND(t->info) == BTF_KIND_PTR;
325}
326
327static bool btf_type_is_int(const struct btf_type *t)
328{
329 return BTF_INFO_KIND(t->info) == BTF_KIND_INT;
330}
331
332/* What types need to be resolved?
333 *
334 * btf_type_is_modifier() is an obvious one.
335 *
336 * btf_type_is_struct() because its member refers to
337 * another type (through member->type).
338
339 * btf_type_is_array() because its element (array->type)
340 * refers to another type. Array can be thought of a
341 * special case of struct while array just has the same
342 * member-type repeated by array->nelems of times.
343 */
344static bool btf_type_needs_resolve(const struct btf_type *t)
345{
346 return btf_type_is_modifier(t) ||
347 btf_type_is_ptr(t) ||
348 btf_type_is_struct(t) ||
349 btf_type_is_array(t);
350}
351
352/* t->size can be used */
353static bool btf_type_has_size(const struct btf_type *t)
354{
355 switch (BTF_INFO_KIND(t->info)) {
356 case BTF_KIND_INT:
357 case BTF_KIND_STRUCT:
358 case BTF_KIND_UNION:
359 case BTF_KIND_ENUM:
360 return true;
361 }
362
363 return false;
364}
365
366static const char *btf_int_encoding_str(u8 encoding)
367{
368 if (encoding == 0)
369 return "(none)";
370 else if (encoding == BTF_INT_SIGNED)
371 return "SIGNED";
372 else if (encoding == BTF_INT_CHAR)
373 return "CHAR";
374 else if (encoding == BTF_INT_BOOL)
375 return "BOOL";
376 else if (encoding == BTF_INT_VARARGS)
377 return "VARARGS";
378 else
379 return "UNKN";
380}
381
382static u16 btf_type_vlen(const struct btf_type *t)
383{
384 return BTF_INFO_VLEN(t->info);
385}
386
387static u32 btf_type_int(const struct btf_type *t)
388{
389 return *(u32 *)(t + 1);
390}
391
392static const struct btf_array *btf_type_array(const struct btf_type *t)
393{
394 return (const struct btf_array *)(t + 1);
395}
396
397static const struct btf_member *btf_type_member(const struct btf_type *t)
398{
399 return (const struct btf_member *)(t + 1);
400}
401
402static const struct btf_enum *btf_type_enum(const struct btf_type *t)
403{
404 return (const struct btf_enum *)(t + 1);
405}
406
407static const struct btf_kind_operations *btf_type_ops(const struct btf_type *t)
408{
409 return kind_ops[BTF_INFO_KIND(t->info)];
410}
411
412static bool btf_name_offset_valid(const struct btf *btf, u32 offset)
413{
414 return !BTF_STR_TBL_ELF_ID(offset) &&
415 BTF_STR_OFFSET(offset) < btf->hdr->str_len;
416}
417
418static const char *btf_name_by_offset(const struct btf *btf, u32 offset)
419{
420 if (!BTF_STR_OFFSET(offset))
421 return "(anon)";
422 else if (BTF_STR_OFFSET(offset) < btf->hdr->str_len)
423 return &btf->strings[BTF_STR_OFFSET(offset)];
424 else
425 return "(invalid-name-offset)";
426}
427
428static const struct btf_type *btf_type_by_id(const struct btf *btf, u32 type_id)
429{
430 if (type_id > btf->nr_types)
431 return NULL;
432
433 return btf->types[type_id];
434}
435
436__printf(2, 3) static void __btf_verifier_log(struct bpf_verifier_log *log,
437 const char *fmt, ...)
438{
439 va_list args;
440
441 va_start(args, fmt);
442 bpf_verifier_vlog(log, fmt, args);
443 va_end(args);
444}
445
446__printf(2, 3) static void btf_verifier_log(struct btf_verifier_env *env,
447 const char *fmt, ...)
448{
449 struct bpf_verifier_log *log = &env->log;
450 va_list args;
451
452 if (!bpf_verifier_log_needed(log))
453 return;
454
455 va_start(args, fmt);
456 bpf_verifier_vlog(log, fmt, args);
457 va_end(args);
458}
459
460__printf(4, 5) static void __btf_verifier_log_type(struct btf_verifier_env *env,
461 const struct btf_type *t,
462 bool log_details,
463 const char *fmt, ...)
464{
465 struct bpf_verifier_log *log = &env->log;
466 u8 kind = BTF_INFO_KIND(t->info);
467 struct btf *btf = env->btf;
468 va_list args;
469
470 if (!bpf_verifier_log_needed(log))
471 return;
472
473 __btf_verifier_log(log, "[%u] %s %s%s",
474 env->log_type_id,
475 btf_kind_str[kind],
476 btf_name_by_offset(btf, t->name),
477 log_details ? " " : "");
478
479 if (log_details)
480 btf_type_ops(t)->log_details(env, t);
481
482 if (fmt && *fmt) {
483 __btf_verifier_log(log, " ");
484 va_start(args, fmt);
485 bpf_verifier_vlog(log, fmt, args);
486 va_end(args);
487 }
488
489 __btf_verifier_log(log, "\n");
490}
491
492#define btf_verifier_log_type(env, t, ...) \
493 __btf_verifier_log_type((env), (t), true, __VA_ARGS__)
494#define btf_verifier_log_basic(env, t, ...) \
495 __btf_verifier_log_type((env), (t), false, __VA_ARGS__)
496
497__printf(4, 5)
498static void btf_verifier_log_member(struct btf_verifier_env *env,
499 const struct btf_type *struct_type,
500 const struct btf_member *member,
501 const char *fmt, ...)
502{
503 struct bpf_verifier_log *log = &env->log;
504 struct btf *btf = env->btf;
505 va_list args;
506
507 if (!bpf_verifier_log_needed(log))
508 return;
509
510 /* The CHECK_META phase already did a btf dump.
511 *
512 * If member is logged again, it must hit an error in
513 * parsing this member. It is useful to print out which
514 * struct this member belongs to.
515 */
516 if (env->phase != CHECK_META)
517 btf_verifier_log_type(env, struct_type, NULL);
518
519 __btf_verifier_log(log, "\t%s type_id=%u bits_offset=%u",
520 btf_name_by_offset(btf, member->name),
521 member->type, member->offset);
522
523 if (fmt && *fmt) {
524 __btf_verifier_log(log, " ");
525 va_start(args, fmt);
526 bpf_verifier_vlog(log, fmt, args);
527 va_end(args);
528 }
529
530 __btf_verifier_log(log, "\n");
531}
532
533static void btf_verifier_log_hdr(struct btf_verifier_env *env)
534{
535 struct bpf_verifier_log *log = &env->log;
536 const struct btf *btf = env->btf;
537 const struct btf_header *hdr;
538
539 if (!bpf_verifier_log_needed(log))
540 return;
541
542 hdr = btf->hdr;
543 __btf_verifier_log(log, "magic: 0x%x\n", hdr->magic);
544 __btf_verifier_log(log, "version: %u\n", hdr->version);
545 __btf_verifier_log(log, "flags: 0x%x\n", hdr->flags);
546 __btf_verifier_log(log, "parent_label: %u\n", hdr->parent_label);
547 __btf_verifier_log(log, "parent_name: %u\n", hdr->parent_name);
548 __btf_verifier_log(log, "label_off: %u\n", hdr->label_off);
549 __btf_verifier_log(log, "object_off: %u\n", hdr->object_off);
550 __btf_verifier_log(log, "func_off: %u\n", hdr->func_off);
551 __btf_verifier_log(log, "type_off: %u\n", hdr->type_off);
552 __btf_verifier_log(log, "str_off: %u\n", hdr->str_off);
553 __btf_verifier_log(log, "str_len: %u\n", hdr->str_len);
554 __btf_verifier_log(log, "btf_total_size: %u\n", btf->data_size);
555}
556
557static int btf_add_type(struct btf_verifier_env *env, struct btf_type *t)
558{
559 struct btf *btf = env->btf;
560
561 /* < 2 because +1 for btf_void which is always in btf->types[0].
562 * btf_void is not accounted in btf->nr_types because btf_void
563 * does not come from the BTF file.
564 */
565 if (btf->types_size - btf->nr_types < 2) {
566 /* Expand 'types' array */
567
568 struct btf_type **new_types;
569 u32 expand_by, new_size;
570
571 if (btf->types_size == BTF_MAX_NR_TYPES) {
572 btf_verifier_log(env, "Exceeded max num of types");
573 return -E2BIG;
574 }
575
576 expand_by = max_t(u32, btf->types_size >> 2, 16);
577 new_size = min_t(u32, BTF_MAX_NR_TYPES,
578 btf->types_size + expand_by);
579
580 new_types = kvzalloc(new_size * sizeof(*new_types),
581 GFP_KERNEL | __GFP_NOWARN);
582 if (!new_types)
583 return -ENOMEM;
584
585 if (btf->nr_types == 0)
586 new_types[0] = &btf_void;
587 else
588 memcpy(new_types, btf->types,
589 sizeof(*btf->types) * (btf->nr_types + 1));
590
591 kvfree(btf->types);
592 btf->types = new_types;
593 btf->types_size = new_size;
594 }
595
596 btf->types[++(btf->nr_types)] = t;
597
598 return 0;
599}
600
601static void btf_free(struct btf *btf)
602{
603 kvfree(btf->types);
604 kvfree(btf->resolved_sizes);
605 kvfree(btf->resolved_ids);
606 kvfree(btf->data);
607 kfree(btf);
608}
609
610static void btf_get(struct btf *btf)
611{
612 refcount_inc(&btf->refcnt);
613}
614
615void btf_put(struct btf *btf)
616{
617 if (btf && refcount_dec_and_test(&btf->refcnt))
618 btf_free(btf);
619}
620
621static int env_resolve_init(struct btf_verifier_env *env)
622{
623 struct btf *btf = env->btf;
624 u32 nr_types = btf->nr_types;
625 u32 *resolved_sizes = NULL;
626 u32 *resolved_ids = NULL;
627 u8 *visit_states = NULL;
628
629 /* +1 for btf_void */
630 resolved_sizes = kvzalloc((nr_types + 1) * sizeof(*resolved_sizes),
631 GFP_KERNEL | __GFP_NOWARN);
632 if (!resolved_sizes)
633 goto nomem;
634
635 resolved_ids = kvzalloc((nr_types + 1) * sizeof(*resolved_ids),
636 GFP_KERNEL | __GFP_NOWARN);
637 if (!resolved_ids)
638 goto nomem;
639
640 visit_states = kvzalloc((nr_types + 1) * sizeof(*visit_states),
641 GFP_KERNEL | __GFP_NOWARN);
642 if (!visit_states)
643 goto nomem;
644
645 btf->resolved_sizes = resolved_sizes;
646 btf->resolved_ids = resolved_ids;
647 env->visit_states = visit_states;
648
649 return 0;
650
651nomem:
652 kvfree(resolved_sizes);
653 kvfree(resolved_ids);
654 kvfree(visit_states);
655 return -ENOMEM;
656}
657
658static void btf_verifier_env_free(struct btf_verifier_env *env)
659{
660 kvfree(env->visit_states);
661 kfree(env);
662}
663
664static bool env_type_is_resolve_sink(const struct btf_verifier_env *env,
665 const struct btf_type *next_type)
666{
667 switch (env->resolve_mode) {
668 case RESOLVE_TBD:
669 /* int, enum or void is a sink */
670 return !btf_type_needs_resolve(next_type);
671 case RESOLVE_PTR:
672 /* int, enum, void, struct or array is a sink for ptr */
673 return !btf_type_is_modifier(next_type) &&
674 !btf_type_is_ptr(next_type);
675 case RESOLVE_STRUCT_OR_ARRAY:
676 /* int, enum, void or ptr is a sink for struct and array */
677 return !btf_type_is_modifier(next_type) &&
678 !btf_type_is_array(next_type) &&
679 !btf_type_is_struct(next_type);
680 default:
681 BUG_ON(1);
682 }
683}
684
685static bool env_type_is_resolved(const struct btf_verifier_env *env,
686 u32 type_id)
687{
688 return env->visit_states[type_id] == RESOLVED;
689}
690
691static int env_stack_push(struct btf_verifier_env *env,
692 const struct btf_type *t, u32 type_id)
693{
694 struct resolve_vertex *v;
695
696 if (env->top_stack == MAX_RESOLVE_DEPTH)
697 return -E2BIG;
698
699 if (env->visit_states[type_id] != NOT_VISITED)
700 return -EEXIST;
701
702 env->visit_states[type_id] = VISITED;
703
704 v = &env->stack[env->top_stack++];
705 v->t = t;
706 v->type_id = type_id;
707 v->next_member = 0;
708
709 if (env->resolve_mode == RESOLVE_TBD) {
710 if (btf_type_is_ptr(t))
711 env->resolve_mode = RESOLVE_PTR;
712 else if (btf_type_is_struct(t) || btf_type_is_array(t))
713 env->resolve_mode = RESOLVE_STRUCT_OR_ARRAY;
714 }
715
716 return 0;
717}
718
719static void env_stack_set_next_member(struct btf_verifier_env *env,
720 u16 next_member)
721{
722 env->stack[env->top_stack - 1].next_member = next_member;
723}
724
725static void env_stack_pop_resolved(struct btf_verifier_env *env,
726 u32 resolved_type_id,
727 u32 resolved_size)
728{
729 u32 type_id = env->stack[--(env->top_stack)].type_id;
730 struct btf *btf = env->btf;
731
732 btf->resolved_sizes[type_id] = resolved_size;
733 btf->resolved_ids[type_id] = resolved_type_id;
734 env->visit_states[type_id] = RESOLVED;
735}
736
737static const struct resolve_vertex *env_stack_peak(struct btf_verifier_env *env)
738{
739 return env->top_stack ? &env->stack[env->top_stack - 1] : NULL;
740}
741
742/* The input param "type_id" must point to a needs_resolve type */
743static const struct btf_type *btf_type_id_resolve(const struct btf *btf,
744 u32 *type_id)
745{
746 *type_id = btf->resolved_ids[*type_id];
747 return btf_type_by_id(btf, *type_id);
748}
749
750const struct btf_type *btf_type_id_size(const struct btf *btf,
751 u32 *type_id, u32 *ret_size)
752{
753 const struct btf_type *size_type;
754 u32 size_type_id = *type_id;
755 u32 size = 0;
756
757 size_type = btf_type_by_id(btf, size_type_id);
758 if (btf_type_is_void_or_null(size_type))
759 return NULL;
760
761 if (btf_type_has_size(size_type)) {
762 size = size_type->size;
763 } else if (btf_type_is_array(size_type)) {
764 size = btf->resolved_sizes[size_type_id];
765 } else if (btf_type_is_ptr(size_type)) {
766 size = sizeof(void *);
767 } else {
768 if (WARN_ON_ONCE(!btf_type_is_modifier(size_type)))
769 return NULL;
770
771 size = btf->resolved_sizes[size_type_id];
772 size_type_id = btf->resolved_ids[size_type_id];
773 size_type = btf_type_by_id(btf, size_type_id);
774 if (btf_type_is_void(size_type))
775 return NULL;
776 }
777
778 *type_id = size_type_id;
779 if (ret_size)
780 *ret_size = size;
781
782 return size_type;
783}
784
785static int btf_df_check_member(struct btf_verifier_env *env,
786 const struct btf_type *struct_type,
787 const struct btf_member *member,
788 const struct btf_type *member_type)
789{
790 btf_verifier_log_basic(env, struct_type,
791 "Unsupported check_member");
792 return -EINVAL;
793}
794
795static int btf_df_resolve(struct btf_verifier_env *env,
796 const struct resolve_vertex *v)
797{
798 btf_verifier_log_basic(env, v->t, "Unsupported resolve");
799 return -EINVAL;
800}
801
802static void btf_df_seq_show(const struct btf *btf, const struct btf_type *t,
803 u32 type_id, void *data, u8 bits_offsets,
804 struct seq_file *m)
805{
806 seq_printf(m, "<unsupported kind:%u>", BTF_INFO_KIND(t->info));
807}
808
809static int btf_int_check_member(struct btf_verifier_env *env,
810 const struct btf_type *struct_type,
811 const struct btf_member *member,
812 const struct btf_type *member_type)
813{
814 u32 int_data = btf_type_int(member_type);
815 u32 struct_bits_off = member->offset;
816 u32 struct_size = struct_type->size;
817 u32 nr_copy_bits;
818 u32 bytes_offset;
819
820 if (U32_MAX - struct_bits_off < BTF_INT_OFFSET(int_data)) {
821 btf_verifier_log_member(env, struct_type, member,
822 "bits_offset exceeds U32_MAX");
823 return -EINVAL;
824 }
825
826 struct_bits_off += BTF_INT_OFFSET(int_data);
827 bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off);
828 nr_copy_bits = BTF_INT_BITS(int_data) +
829 BITS_PER_BYTE_MASKED(struct_bits_off);
830
831 if (nr_copy_bits > BITS_PER_U64) {
832 btf_verifier_log_member(env, struct_type, member,
833 "nr_copy_bits exceeds 64");
834 return -EINVAL;
835 }
836
837 if (struct_size < bytes_offset ||
838 struct_size - bytes_offset < BITS_ROUNDUP_BYTES(nr_copy_bits)) {
839 btf_verifier_log_member(env, struct_type, member,
840 "Member exceeds struct_size");
841 return -EINVAL;
842 }
843
844 return 0;
845}
846
847static s32 btf_int_check_meta(struct btf_verifier_env *env,
848 const struct btf_type *t,
849 u32 meta_left)
850{
851 u32 int_data, nr_bits, meta_needed = sizeof(int_data);
852 u16 encoding;
853
854 if (meta_left < meta_needed) {
855 btf_verifier_log_basic(env, t,
856 "meta_left:%u meta_needed:%u",
857 meta_left, meta_needed);
858 return -EINVAL;
859 }
860
861 if (btf_type_vlen(t)) {
862 btf_verifier_log_type(env, t, "vlen != 0");
863 return -EINVAL;
864 }
865
866 int_data = btf_type_int(t);
867 nr_bits = BTF_INT_BITS(int_data) + BTF_INT_OFFSET(int_data);
868
869 if (nr_bits > BITS_PER_U64) {
870 btf_verifier_log_type(env, t, "nr_bits exceeds %zu",
871 BITS_PER_U64);
872 return -EINVAL;
873 }
874
875 if (BITS_ROUNDUP_BYTES(nr_bits) > t->size) {
876 btf_verifier_log_type(env, t, "nr_bits exceeds type_size");
877 return -EINVAL;
878 }
879
880 encoding = BTF_INT_ENCODING(int_data);
881 if (encoding &&
882 encoding != BTF_INT_SIGNED &&
883 encoding != BTF_INT_CHAR &&
884 encoding != BTF_INT_BOOL &&
885 encoding != BTF_INT_VARARGS) {
886 btf_verifier_log_type(env, t, "Unsupported encoding");
887 return -ENOTSUPP;
888 }
889
890 btf_verifier_log_type(env, t, NULL);
891
892 return meta_needed;
893}
894
895static void btf_int_log(struct btf_verifier_env *env,
896 const struct btf_type *t)
897{
898 int int_data = btf_type_int(t);
899
900 btf_verifier_log(env,
901 "size=%u bits_offset=%u nr_bits=%u encoding=%s",
902 t->size, BTF_INT_OFFSET(int_data),
903 BTF_INT_BITS(int_data),
904 btf_int_encoding_str(BTF_INT_ENCODING(int_data)));
905}
906
907static void btf_int_bits_seq_show(const struct btf *btf,
908 const struct btf_type *t,
909 void *data, u8 bits_offset,
910 struct seq_file *m)
911{
912 u32 int_data = btf_type_int(t);
913 u16 nr_bits = BTF_INT_BITS(int_data);
914 u16 total_bits_offset;
915 u16 nr_copy_bytes;
916 u16 nr_copy_bits;
917 u8 nr_upper_bits;
918 union {
919 u64 u64_num;
920 u8 u8_nums[8];
921 } print_num;
922
923 total_bits_offset = bits_offset + BTF_INT_OFFSET(int_data);
924 data += BITS_ROUNDDOWN_BYTES(total_bits_offset);
925 bits_offset = BITS_PER_BYTE_MASKED(total_bits_offset);
926 nr_copy_bits = nr_bits + bits_offset;
927 nr_copy_bytes = BITS_ROUNDUP_BYTES(nr_copy_bits);
928
929 print_num.u64_num = 0;
930 memcpy(&print_num.u64_num, data, nr_copy_bytes);
931
932 /* Ditch the higher order bits */
933 nr_upper_bits = BITS_PER_BYTE_MASKED(nr_copy_bits);
934 if (nr_upper_bits) {
935 /* We need to mask out some bits of the upper byte. */
936 u8 mask = (1 << nr_upper_bits) - 1;
937
938 print_num.u8_nums[nr_copy_bytes - 1] &= mask;
939 }
940
941 print_num.u64_num >>= bits_offset;
942
943 seq_printf(m, "0x%llx", print_num.u64_num);
944}
945
946static void btf_int_seq_show(const struct btf *btf, const struct btf_type *t,
947 u32 type_id, void *data, u8 bits_offset,
948 struct seq_file *m)
949{
950 u32 int_data = btf_type_int(t);
951 u8 encoding = BTF_INT_ENCODING(int_data);
952 bool sign = encoding & BTF_INT_SIGNED;
953 u32 nr_bits = BTF_INT_BITS(int_data);
954
955 if (bits_offset || BTF_INT_OFFSET(int_data) ||
956 BITS_PER_BYTE_MASKED(nr_bits)) {
957 btf_int_bits_seq_show(btf, t, data, bits_offset, m);
958 return;
959 }
960
961 switch (nr_bits) {
962 case 64:
963 if (sign)
964 seq_printf(m, "%lld", *(s64 *)data);
965 else
966 seq_printf(m, "%llu", *(u64 *)data);
967 break;
968 case 32:
969 if (sign)
970 seq_printf(m, "%d", *(s32 *)data);
971 else
972 seq_printf(m, "%u", *(u32 *)data);
973 break;
974 case 16:
975 if (sign)
976 seq_printf(m, "%d", *(s16 *)data);
977 else
978 seq_printf(m, "%u", *(u16 *)data);
979 break;
980 case 8:
981 if (sign)
982 seq_printf(m, "%d", *(s8 *)data);
983 else
984 seq_printf(m, "%u", *(u8 *)data);
985 break;
986 default:
987 btf_int_bits_seq_show(btf, t, data, bits_offset, m);
988 }
989}
990
991static const struct btf_kind_operations int_ops = {
992 .check_meta = btf_int_check_meta,
993 .resolve = btf_df_resolve,
994 .check_member = btf_int_check_member,
995 .log_details = btf_int_log,
996 .seq_show = btf_int_seq_show,
997};
998
999static int btf_modifier_check_member(struct btf_verifier_env *env,
1000 const struct btf_type *struct_type,
1001 const struct btf_member *member,
1002 const struct btf_type *member_type)
1003{
1004 const struct btf_type *resolved_type;
1005 u32 resolved_type_id = member->type;
1006 struct btf_member resolved_member;
1007 struct btf *btf = env->btf;
1008
1009 resolved_type = btf_type_id_size(btf, &resolved_type_id, NULL);
1010 if (!resolved_type) {
1011 btf_verifier_log_member(env, struct_type, member,
1012 "Invalid member");
1013 return -EINVAL;
1014 }
1015
1016 resolved_member = *member;
1017 resolved_member.type = resolved_type_id;
1018
1019 return btf_type_ops(resolved_type)->check_member(env, struct_type,
1020 &resolved_member,
1021 resolved_type);
1022}
1023
1024static int btf_ptr_check_member(struct btf_verifier_env *env,
1025 const struct btf_type *struct_type,
1026 const struct btf_member *member,
1027 const struct btf_type *member_type)
1028{
1029 u32 struct_size, struct_bits_off, bytes_offset;
1030
1031 struct_size = struct_type->size;
1032 struct_bits_off = member->offset;
1033 bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off);
1034
1035 if (BITS_PER_BYTE_MASKED(struct_bits_off)) {
1036 btf_verifier_log_member(env, struct_type, member,
1037 "Member is not byte aligned");
1038 return -EINVAL;
1039 }
1040
1041 if (struct_size - bytes_offset < sizeof(void *)) {
1042 btf_verifier_log_member(env, struct_type, member,
1043 "Member exceeds struct_size");
1044 return -EINVAL;
1045 }
1046
1047 return 0;
1048}
1049
1050static int btf_ref_type_check_meta(struct btf_verifier_env *env,
1051 const struct btf_type *t,
1052 u32 meta_left)
1053{
1054 if (btf_type_vlen(t)) {
1055 btf_verifier_log_type(env, t, "vlen != 0");
1056 return -EINVAL;
1057 }
1058
1059 if (BTF_TYPE_PARENT(t->type)) {
1060 btf_verifier_log_type(env, t, "Invalid type_id");
1061 return -EINVAL;
1062 }
1063
1064 btf_verifier_log_type(env, t, NULL);
1065
1066 return 0;
1067}
1068
1069static int btf_modifier_resolve(struct btf_verifier_env *env,
1070 const struct resolve_vertex *v)
1071{
1072 const struct btf_type *t = v->t;
1073 const struct btf_type *next_type;
1074 u32 next_type_id = t->type;
1075 struct btf *btf = env->btf;
1076 u32 next_type_size = 0;
1077
1078 next_type = btf_type_by_id(btf, next_type_id);
1079 if (!next_type) {
1080 btf_verifier_log_type(env, v->t, "Invalid type_id");
1081 return -EINVAL;
1082 }
1083
1084 /* "typedef void new_void", "const void"...etc */
1085 if (btf_type_is_void(next_type))
1086 goto resolved;
1087
1088 if (!env_type_is_resolve_sink(env, next_type) &&
1089 !env_type_is_resolved(env, next_type_id))
1090 return env_stack_push(env, next_type, next_type_id);
1091
1092 /* Figure out the resolved next_type_id with size.
1093 * They will be stored in the current modifier's
1094 * resolved_ids and resolved_sizes such that it can
1095 * save us a few type-following when we use it later (e.g. in
1096 * pretty print).
1097 */
1098 if (!btf_type_id_size(btf, &next_type_id, &next_type_size) &&
1099 !btf_type_is_void(btf_type_id_resolve(btf, &next_type_id))) {
1100 btf_verifier_log_type(env, v->t, "Invalid type_id");
1101 return -EINVAL;
1102 }
1103
1104resolved:
1105 env_stack_pop_resolved(env, next_type_id, next_type_size);
1106
1107 return 0;
1108}
1109
1110static int btf_ptr_resolve(struct btf_verifier_env *env,
1111 const struct resolve_vertex *v)
1112{
1113 const struct btf_type *next_type;
1114 const struct btf_type *t = v->t;
1115 u32 next_type_id = t->type;
1116 struct btf *btf = env->btf;
1117 u32 next_type_size = 0;
1118
1119 next_type = btf_type_by_id(btf, next_type_id);
1120 if (!next_type) {
1121 btf_verifier_log_type(env, v->t, "Invalid type_id");
1122 return -EINVAL;
1123 }
1124
1125 /* "void *" */
1126 if (btf_type_is_void(next_type))
1127 goto resolved;
1128
1129 if (!env_type_is_resolve_sink(env, next_type) &&
1130 !env_type_is_resolved(env, next_type_id))
1131 return env_stack_push(env, next_type, next_type_id);
1132
1133 /* If the modifier was RESOLVED during RESOLVE_STRUCT_OR_ARRAY,
1134 * the modifier may have stopped resolving when it was resolved
1135 * to a ptr (last-resolved-ptr).
1136 *
1137 * We now need to continue from the last-resolved-ptr to
1138 * ensure the last-resolved-ptr will not referring back to
1139 * the currenct ptr (t).
1140 */
1141 if (btf_type_is_modifier(next_type)) {
1142 const struct btf_type *resolved_type;
1143 u32 resolved_type_id;
1144
1145 resolved_type_id = next_type_id;
1146 resolved_type = btf_type_id_resolve(btf, &resolved_type_id);
1147
1148 if (btf_type_is_ptr(resolved_type) &&
1149 !env_type_is_resolve_sink(env, resolved_type) &&
1150 !env_type_is_resolved(env, resolved_type_id))
1151 return env_stack_push(env, resolved_type,
1152 resolved_type_id);
1153 }
1154
1155 if (!btf_type_id_size(btf, &next_type_id, &next_type_size) &&
1156 !btf_type_is_void(btf_type_id_resolve(btf, &next_type_id))) {
1157 btf_verifier_log_type(env, v->t, "Invalid type_id");
1158 return -EINVAL;
1159 }
1160
1161resolved:
1162 env_stack_pop_resolved(env, next_type_id, 0);
1163
1164 return 0;
1165}
1166
1167static void btf_modifier_seq_show(const struct btf *btf,
1168 const struct btf_type *t,
1169 u32 type_id, void *data,
1170 u8 bits_offset, struct seq_file *m)
1171{
1172 t = btf_type_id_resolve(btf, &type_id);
1173
1174 btf_type_ops(t)->seq_show(btf, t, type_id, data, bits_offset, m);
1175}
1176
1177static void btf_ptr_seq_show(const struct btf *btf, const struct btf_type *t,
1178 u32 type_id, void *data, u8 bits_offset,
1179 struct seq_file *m)
1180{
1181 /* It is a hashed value */
1182 seq_printf(m, "%p", *(void **)data);
1183}
1184
1185static void btf_ref_type_log(struct btf_verifier_env *env,
1186 const struct btf_type *t)
1187{
1188 btf_verifier_log(env, "type_id=%u", t->type);
1189}
1190
1191static struct btf_kind_operations modifier_ops = {
1192 .check_meta = btf_ref_type_check_meta,
1193 .resolve = btf_modifier_resolve,
1194 .check_member = btf_modifier_check_member,
1195 .log_details = btf_ref_type_log,
1196 .seq_show = btf_modifier_seq_show,
1197};
1198
1199static struct btf_kind_operations ptr_ops = {
1200 .check_meta = btf_ref_type_check_meta,
1201 .resolve = btf_ptr_resolve,
1202 .check_member = btf_ptr_check_member,
1203 .log_details = btf_ref_type_log,
1204 .seq_show = btf_ptr_seq_show,
1205};
1206
1207static struct btf_kind_operations fwd_ops = {
1208 .check_meta = btf_ref_type_check_meta,
1209 .resolve = btf_df_resolve,
1210 .check_member = btf_df_check_member,
1211 .log_details = btf_ref_type_log,
1212 .seq_show = btf_df_seq_show,
1213};
1214
1215static int btf_array_check_member(struct btf_verifier_env *env,
1216 const struct btf_type *struct_type,
1217 const struct btf_member *member,
1218 const struct btf_type *member_type)
1219{
1220 u32 struct_bits_off = member->offset;
1221 u32 struct_size, bytes_offset;
1222 u32 array_type_id, array_size;
1223 struct btf *btf = env->btf;
1224
1225 if (BITS_PER_BYTE_MASKED(struct_bits_off)) {
1226 btf_verifier_log_member(env, struct_type, member,
1227 "Member is not byte aligned");
1228 return -EINVAL;
1229 }
1230
1231 array_type_id = member->type;
1232 btf_type_id_size(btf, &array_type_id, &array_size);
1233 struct_size = struct_type->size;
1234 bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off);
1235 if (struct_size - bytes_offset < array_size) {
1236 btf_verifier_log_member(env, struct_type, member,
1237 "Member exceeds struct_size");
1238 return -EINVAL;
1239 }
1240
1241 return 0;
1242}
1243
1244static s32 btf_array_check_meta(struct btf_verifier_env *env,
1245 const struct btf_type *t,
1246 u32 meta_left)
1247{
1248 const struct btf_array *array = btf_type_array(t);
1249 u32 meta_needed = sizeof(*array);
1250
1251 if (meta_left < meta_needed) {
1252 btf_verifier_log_basic(env, t,
1253 "meta_left:%u meta_needed:%u",
1254 meta_left, meta_needed);
1255 return -EINVAL;
1256 }
1257
1258 if (btf_type_vlen(t)) {
1259 btf_verifier_log_type(env, t, "vlen != 0");
1260 return -EINVAL;
1261 }
1262
1263 /* We are a little forgiving on array->index_type since
1264 * the kernel is not using it.
1265 */
1266 /* Array elem cannot be in type void,
1267 * so !array->type is not allowed.
1268 */
1269 if (!array->type || BTF_TYPE_PARENT(array->type)) {
1270 btf_verifier_log_type(env, t, "Invalid type_id");
1271 return -EINVAL;
1272 }
1273
1274 btf_verifier_log_type(env, t, NULL);
1275
1276 return meta_needed;
1277}
1278
1279static int btf_array_resolve(struct btf_verifier_env *env,
1280 const struct resolve_vertex *v)
1281{
1282 const struct btf_array *array = btf_type_array(v->t);
1283 const struct btf_type *elem_type;
1284 u32 elem_type_id = array->type;
1285 struct btf *btf = env->btf;
1286 u32 elem_size;
1287
1288 elem_type = btf_type_by_id(btf, elem_type_id);
1289 if (btf_type_is_void_or_null(elem_type)) {
1290 btf_verifier_log_type(env, v->t,
1291 "Invalid elem");
1292 return -EINVAL;
1293 }
1294
1295 if (!env_type_is_resolve_sink(env, elem_type) &&
1296 !env_type_is_resolved(env, elem_type_id))
1297 return env_stack_push(env, elem_type, elem_type_id);
1298
1299 elem_type = btf_type_id_size(btf, &elem_type_id, &elem_size);
1300 if (!elem_type) {
1301 btf_verifier_log_type(env, v->t, "Invalid elem");
1302 return -EINVAL;
1303 }
1304
1305 if (btf_type_is_int(elem_type)) {
1306 int int_type_data = btf_type_int(elem_type);
1307 u16 nr_bits = BTF_INT_BITS(int_type_data);
1308 u16 nr_bytes = BITS_ROUNDUP_BYTES(nr_bits);
1309
1310 /* Put more restriction on array of int. The int cannot
1311 * be a bit field and it must be either u8/u16/u32/u64.
1312 */
1313 if (BITS_PER_BYTE_MASKED(nr_bits) ||
1314 BTF_INT_OFFSET(int_type_data) ||
1315 (nr_bytes != sizeof(u8) && nr_bytes != sizeof(u16) &&
1316 nr_bytes != sizeof(u32) && nr_bytes != sizeof(u64))) {
1317 btf_verifier_log_type(env, v->t,
1318 "Invalid array of int");
1319 return -EINVAL;
1320 }
1321 }
1322
1323 if (array->nelems && elem_size > U32_MAX / array->nelems) {
1324 btf_verifier_log_type(env, v->t,
1325 "Array size overflows U32_MAX");
1326 return -EINVAL;
1327 }
1328
1329 env_stack_pop_resolved(env, elem_type_id, elem_size * array->nelems);
1330
1331 return 0;
1332}
1333
1334static void btf_array_log(struct btf_verifier_env *env,
1335 const struct btf_type *t)
1336{
1337 const struct btf_array *array = btf_type_array(t);
1338
1339 btf_verifier_log(env, "type_id=%u index_type_id=%u nr_elems=%u",
1340 array->type, array->index_type, array->nelems);
1341}
1342
1343static void btf_array_seq_show(const struct btf *btf, const struct btf_type *t,
1344 u32 type_id, void *data, u8 bits_offset,
1345 struct seq_file *m)
1346{
1347 const struct btf_array *array = btf_type_array(t);
1348 const struct btf_kind_operations *elem_ops;
1349 const struct btf_type *elem_type;
1350 u32 i, elem_size, elem_type_id;
1351
1352 elem_type_id = array->type;
1353 elem_type = btf_type_id_size(btf, &elem_type_id, &elem_size);
1354 elem_ops = btf_type_ops(elem_type);
1355 seq_puts(m, "[");
1356 for (i = 0; i < array->nelems; i++) {
1357 if (i)
1358 seq_puts(m, ",");
1359
1360 elem_ops->seq_show(btf, elem_type, elem_type_id, data,
1361 bits_offset, m);
1362 data += elem_size;
1363 }
1364 seq_puts(m, "]");
1365}
1366
1367static struct btf_kind_operations array_ops = {
1368 .check_meta = btf_array_check_meta,
1369 .resolve = btf_array_resolve,
1370 .check_member = btf_array_check_member,
1371 .log_details = btf_array_log,
1372 .seq_show = btf_array_seq_show,
1373};
1374
1375static int btf_struct_check_member(struct btf_verifier_env *env,
1376 const struct btf_type *struct_type,
1377 const struct btf_member *member,
1378 const struct btf_type *member_type)
1379{
1380 u32 struct_bits_off = member->offset;
1381 u32 struct_size, bytes_offset;
1382
1383 if (BITS_PER_BYTE_MASKED(struct_bits_off)) {
1384 btf_verifier_log_member(env, struct_type, member,
1385 "Member is not byte aligned");
1386 return -EINVAL;
1387 }
1388
1389 struct_size = struct_type->size;
1390 bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off);
1391 if (struct_size - bytes_offset < member_type->size) {
1392 btf_verifier_log_member(env, struct_type, member,
1393 "Member exceeds struct_size");
1394 return -EINVAL;
1395 }
1396
1397 return 0;
1398}
1399
1400static s32 btf_struct_check_meta(struct btf_verifier_env *env,
1401 const struct btf_type *t,
1402 u32 meta_left)
1403{
1404 bool is_union = BTF_INFO_KIND(t->info) == BTF_KIND_UNION;
1405 const struct btf_member *member;
1406 struct btf *btf = env->btf;
1407 u32 struct_size = t->size;
1408 u32 meta_needed;
1409 u16 i;
1410
1411 meta_needed = btf_type_vlen(t) * sizeof(*member);
1412 if (meta_left < meta_needed) {
1413 btf_verifier_log_basic(env, t,
1414 "meta_left:%u meta_needed:%u",
1415 meta_left, meta_needed);
1416 return -EINVAL;
1417 }
1418
1419 btf_verifier_log_type(env, t, NULL);
1420
1421 for_each_member(i, t, member) {
1422 if (!btf_name_offset_valid(btf, member->name)) {
1423 btf_verifier_log_member(env, t, member,
1424 "Invalid member name_offset:%u",
1425 member->name);
1426 return -EINVAL;
1427 }
1428
1429 /* A member cannot be in type void */
1430 if (!member->type || BTF_TYPE_PARENT(member->type)) {
1431 btf_verifier_log_member(env, t, member,
1432 "Invalid type_id");
1433 return -EINVAL;
1434 }
1435
1436 if (is_union && member->offset) {
1437 btf_verifier_log_member(env, t, member,
1438 "Invalid member bits_offset");
1439 return -EINVAL;
1440 }
1441
1442 if (BITS_ROUNDUP_BYTES(member->offset) > struct_size) {
1443 btf_verifier_log_member(env, t, member,
1444 "Memmber bits_offset exceeds its struct size");
1445 return -EINVAL;
1446 }
1447
1448 btf_verifier_log_member(env, t, member, NULL);
1449 }
1450
1451 return meta_needed;
1452}
1453
1454static int btf_struct_resolve(struct btf_verifier_env *env,
1455 const struct resolve_vertex *v)
1456{
1457 const struct btf_member *member;
1458 int err;
1459 u16 i;
1460
1461 /* Before continue resolving the next_member,
1462 * ensure the last member is indeed resolved to a
1463 * type with size info.
1464 */
1465 if (v->next_member) {
1466 const struct btf_type *last_member_type;
1467 const struct btf_member *last_member;
1468 u16 last_member_type_id;
1469
1470 last_member = btf_type_member(v->t) + v->next_member - 1;
1471 last_member_type_id = last_member->type;
1472 if (WARN_ON_ONCE(!env_type_is_resolved(env,
1473 last_member_type_id)))
1474 return -EINVAL;
1475
1476 last_member_type = btf_type_by_id(env->btf,
1477 last_member_type_id);
1478 err = btf_type_ops(last_member_type)->check_member(env, v->t,
1479 last_member,
1480 last_member_type);
1481 if (err)
1482 return err;
1483 }
1484
1485 for_each_member_from(i, v->next_member, v->t, member) {
1486 u32 member_type_id = member->type;
1487 const struct btf_type *member_type = btf_type_by_id(env->btf,
1488 member_type_id);
1489
1490 if (btf_type_is_void_or_null(member_type)) {
1491 btf_verifier_log_member(env, v->t, member,
1492 "Invalid member");
1493 return -EINVAL;
1494 }
1495
1496 if (!env_type_is_resolve_sink(env, member_type) &&
1497 !env_type_is_resolved(env, member_type_id)) {
1498 env_stack_set_next_member(env, i + 1);
1499 return env_stack_push(env, member_type, member_type_id);
1500 }
1501
1502 err = btf_type_ops(member_type)->check_member(env, v->t,
1503 member,
1504 member_type);
1505 if (err)
1506 return err;
1507 }
1508
1509 env_stack_pop_resolved(env, 0, 0);
1510
1511 return 0;
1512}
1513
1514static void btf_struct_log(struct btf_verifier_env *env,
1515 const struct btf_type *t)
1516{
1517 btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t));
1518}
1519
1520static void btf_struct_seq_show(const struct btf *btf, const struct btf_type *t,
1521 u32 type_id, void *data, u8 bits_offset,
1522 struct seq_file *m)
1523{
1524 const char *seq = BTF_INFO_KIND(t->info) == BTF_KIND_UNION ? "|" : ",";
1525 const struct btf_member *member;
1526 u32 i;
1527
1528 seq_puts(m, "{");
1529 for_each_member(i, t, member) {
1530 const struct btf_type *member_type = btf_type_by_id(btf,
1531 member->type);
1532 u32 member_offset = member->offset;
1533 u32 bytes_offset = BITS_ROUNDDOWN_BYTES(member_offset);
1534 u8 bits8_offset = BITS_PER_BYTE_MASKED(member_offset);
1535 const struct btf_kind_operations *ops;
1536
1537 if (i)
1538 seq_puts(m, seq);
1539
1540 ops = btf_type_ops(member_type);
1541 ops->seq_show(btf, member_type, member->type,
1542 data + bytes_offset, bits8_offset, m);
1543 }
1544 seq_puts(m, "}");
1545}
1546
1547static struct btf_kind_operations struct_ops = {
1548 .check_meta = btf_struct_check_meta,
1549 .resolve = btf_struct_resolve,
1550 .check_member = btf_struct_check_member,
1551 .log_details = btf_struct_log,
1552 .seq_show = btf_struct_seq_show,
1553};
1554
1555static int btf_enum_check_member(struct btf_verifier_env *env,
1556 const struct btf_type *struct_type,
1557 const struct btf_member *member,
1558 const struct btf_type *member_type)
1559{
1560 u32 struct_bits_off = member->offset;
1561 u32 struct_size, bytes_offset;
1562
1563 if (BITS_PER_BYTE_MASKED(struct_bits_off)) {
1564 btf_verifier_log_member(env, struct_type, member,
1565 "Member is not byte aligned");
1566 return -EINVAL;
1567 }
1568
1569 struct_size = struct_type->size;
1570 bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off);
1571 if (struct_size - bytes_offset < sizeof(int)) {
1572 btf_verifier_log_member(env, struct_type, member,
1573 "Member exceeds struct_size");
1574 return -EINVAL;
1575 }
1576
1577 return 0;
1578}
1579
1580static s32 btf_enum_check_meta(struct btf_verifier_env *env,
1581 const struct btf_type *t,
1582 u32 meta_left)
1583{
1584 const struct btf_enum *enums = btf_type_enum(t);
1585 struct btf *btf = env->btf;
1586 u16 i, nr_enums;
1587 u32 meta_needed;
1588
1589 nr_enums = btf_type_vlen(t);
1590 meta_needed = nr_enums * sizeof(*enums);
1591
1592 if (meta_left < meta_needed) {
1593 btf_verifier_log_basic(env, t,
1594 "meta_left:%u meta_needed:%u",
1595 meta_left, meta_needed);
1596 return -EINVAL;
1597 }
1598
1599 if (t->size != sizeof(int)) {
1600 btf_verifier_log_type(env, t, "Expected size:%zu",
1601 sizeof(int));
1602 return -EINVAL;
1603 }
1604
1605 btf_verifier_log_type(env, t, NULL);
1606
1607 for (i = 0; i < nr_enums; i++) {
1608 if (!btf_name_offset_valid(btf, enums[i].name)) {
1609 btf_verifier_log(env, "\tInvalid name_offset:%u",
1610 enums[i].name);
1611 return -EINVAL;
1612 }
1613
1614 btf_verifier_log(env, "\t%s val=%d\n",
1615 btf_name_by_offset(btf, enums[i].name),
1616 enums[i].val);
1617 }
1618
1619 return meta_needed;
1620}
1621
1622static void btf_enum_log(struct btf_verifier_env *env,
1623 const struct btf_type *t)
1624{
1625 btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t));
1626}
1627
1628static void btf_enum_seq_show(const struct btf *btf, const struct btf_type *t,
1629 u32 type_id, void *data, u8 bits_offset,
1630 struct seq_file *m)
1631{
1632 const struct btf_enum *enums = btf_type_enum(t);
1633 u32 i, nr_enums = btf_type_vlen(t);
1634 int v = *(int *)data;
1635
1636 for (i = 0; i < nr_enums; i++) {
1637 if (v == enums[i].val) {
1638 seq_printf(m, "%s",
1639 btf_name_by_offset(btf, enums[i].name));
1640 return;
1641 }
1642 }
1643
1644 seq_printf(m, "%d", v);
1645}
1646
1647static struct btf_kind_operations enum_ops = {
1648 .check_meta = btf_enum_check_meta,
1649 .resolve = btf_df_resolve,
1650 .check_member = btf_enum_check_member,
1651 .log_details = btf_enum_log,
1652 .seq_show = btf_enum_seq_show,
1653};
1654
1655static const struct btf_kind_operations * const kind_ops[NR_BTF_KINDS] = {
1656 [BTF_KIND_INT] = &int_ops,
1657 [BTF_KIND_PTR] = &ptr_ops,
1658 [BTF_KIND_ARRAY] = &array_ops,
1659 [BTF_KIND_STRUCT] = &struct_ops,
1660 [BTF_KIND_UNION] = &struct_ops,
1661 [BTF_KIND_ENUM] = &enum_ops,
1662 [BTF_KIND_FWD] = &fwd_ops,
1663 [BTF_KIND_TYPEDEF] = &modifier_ops,
1664 [BTF_KIND_VOLATILE] = &modifier_ops,
1665 [BTF_KIND_CONST] = &modifier_ops,
1666 [BTF_KIND_RESTRICT] = &modifier_ops,
1667};
1668
1669static s32 btf_check_meta(struct btf_verifier_env *env,
1670 const struct btf_type *t,
1671 u32 meta_left)
1672{
1673 u32 saved_meta_left = meta_left;
1674 s32 var_meta_size;
1675
1676 if (meta_left < sizeof(*t)) {
1677 btf_verifier_log(env, "[%u] meta_left:%u meta_needed:%zu",
1678 env->log_type_id, meta_left, sizeof(*t));
1679 return -EINVAL;
1680 }
1681 meta_left -= sizeof(*t);
1682
1683 if (BTF_INFO_KIND(t->info) > BTF_KIND_MAX ||
1684 BTF_INFO_KIND(t->info) == BTF_KIND_UNKN) {
1685 btf_verifier_log(env, "[%u] Invalid kind:%u",
1686 env->log_type_id, BTF_INFO_KIND(t->info));
1687 return -EINVAL;
1688 }
1689
1690 if (!btf_name_offset_valid(env->btf, t->name)) {
1691 btf_verifier_log(env, "[%u] Invalid name_offset:%u",
1692 env->log_type_id, t->name);
1693 return -EINVAL;
1694 }
1695
1696 var_meta_size = btf_type_ops(t)->check_meta(env, t, meta_left);
1697 if (var_meta_size < 0)
1698 return var_meta_size;
1699
1700 meta_left -= var_meta_size;
1701
1702 return saved_meta_left - meta_left;
1703}
1704
1705static int btf_check_all_metas(struct btf_verifier_env *env)
1706{
1707 struct btf *btf = env->btf;
1708 struct btf_header *hdr;
1709 void *cur, *end;
1710
1711 hdr = btf->hdr;
1712 cur = btf->nohdr_data + hdr->type_off;
1713 end = btf->nohdr_data + hdr->str_off;
1714
1715 env->log_type_id = 1;
1716 while (cur < end) {
1717 struct btf_type *t = cur;
1718 s32 meta_size;
1719
1720 meta_size = btf_check_meta(env, t, end - cur);
1721 if (meta_size < 0)
1722 return meta_size;
1723
1724 btf_add_type(env, t);
1725 cur += meta_size;
1726 env->log_type_id++;
1727 }
1728
1729 return 0;
1730}
1731
1732static int btf_resolve(struct btf_verifier_env *env,
1733 const struct btf_type *t, u32 type_id)
1734{
1735 const struct resolve_vertex *v;
1736 int err = 0;
1737
1738 env->resolve_mode = RESOLVE_TBD;
1739 env_stack_push(env, t, type_id);
1740 while (!err && (v = env_stack_peak(env))) {
1741 env->log_type_id = v->type_id;
1742 err = btf_type_ops(v->t)->resolve(env, v);
1743 }
1744
1745 env->log_type_id = type_id;
1746 if (err == -E2BIG)
1747 btf_verifier_log_type(env, t,
1748 "Exceeded max resolving depth:%u",
1749 MAX_RESOLVE_DEPTH);
1750 else if (err == -EEXIST)
1751 btf_verifier_log_type(env, t, "Loop detected");
1752
1753 return err;
1754}
1755
1756static bool btf_resolve_valid(struct btf_verifier_env *env,
1757 const struct btf_type *t,
1758 u32 type_id)
1759{
1760 struct btf *btf = env->btf;
1761
1762 if (!env_type_is_resolved(env, type_id))
1763 return false;
1764
1765 if (btf_type_is_struct(t))
1766 return !btf->resolved_ids[type_id] &&
1767 !btf->resolved_sizes[type_id];
1768
1769 if (btf_type_is_modifier(t) || btf_type_is_ptr(t)) {
1770 t = btf_type_id_resolve(btf, &type_id);
1771 return t && !btf_type_is_modifier(t);
1772 }
1773
1774 if (btf_type_is_array(t)) {
1775 const struct btf_array *array = btf_type_array(t);
1776 const struct btf_type *elem_type;
1777 u32 elem_type_id = array->type;
1778 u32 elem_size;
1779
1780 elem_type = btf_type_id_size(btf, &elem_type_id, &elem_size);
1781 return elem_type && !btf_type_is_modifier(elem_type) &&
1782 (array->nelems * elem_size ==
1783 btf->resolved_sizes[type_id]);
1784 }
1785
1786 return false;
1787}
1788
1789static int btf_check_all_types(struct btf_verifier_env *env)
1790{
1791 struct btf *btf = env->btf;
1792 u32 type_id;
1793 int err;
1794
1795 err = env_resolve_init(env);
1796 if (err)
1797 return err;
1798
1799 env->phase++;
1800 for (type_id = 1; type_id <= btf->nr_types; type_id++) {
1801 const struct btf_type *t = btf_type_by_id(btf, type_id);
1802
1803 env->log_type_id = type_id;
1804 if (btf_type_needs_resolve(t) &&
1805 !env_type_is_resolved(env, type_id)) {
1806 err = btf_resolve(env, t, type_id);
1807 if (err)
1808 return err;
1809 }
1810
1811 if (btf_type_needs_resolve(t) &&
1812 !btf_resolve_valid(env, t, type_id)) {
1813 btf_verifier_log_type(env, t, "Invalid resolve state");
1814 return -EINVAL;
1815 }
1816 }
1817
1818 return 0;
1819}
1820
1821static int btf_parse_type_sec(struct btf_verifier_env *env)
1822{
1823 int err;
1824
1825 err = btf_check_all_metas(env);
1826 if (err)
1827 return err;
1828
1829 return btf_check_all_types(env);
1830}
1831
1832static int btf_parse_str_sec(struct btf_verifier_env *env)
1833{
1834 const struct btf_header *hdr;
1835 struct btf *btf = env->btf;
1836 const char *start, *end;
1837
1838 hdr = btf->hdr;
1839 start = btf->nohdr_data + hdr->str_off;
1840 end = start + hdr->str_len;
1841
1842 if (!hdr->str_len || hdr->str_len - 1 > BTF_MAX_NAME_OFFSET ||
1843 start[0] || end[-1]) {
1844 btf_verifier_log(env, "Invalid string section");
1845 return -EINVAL;
1846 }
1847
1848 btf->strings = start;
1849
1850 return 0;
1851}
1852
1853static int btf_parse_hdr(struct btf_verifier_env *env)
1854{
1855 const struct btf_header *hdr;
1856 struct btf *btf = env->btf;
1857 u32 meta_left;
1858
1859 if (btf->data_size < sizeof(*hdr)) {
1860 btf_verifier_log(env, "btf_header not found");
1861 return -EINVAL;
1862 }
1863
1864 btf_verifier_log_hdr(env);
1865
1866 hdr = btf->hdr;
1867 if (hdr->magic != BTF_MAGIC) {
1868 btf_verifier_log(env, "Invalid magic");
1869 return -EINVAL;
1870 }
1871
1872 if (hdr->version != BTF_VERSION) {
1873 btf_verifier_log(env, "Unsupported version");
1874 return -ENOTSUPP;
1875 }
1876
1877 if (hdr->flags) {
1878 btf_verifier_log(env, "Unsupported flags");
1879 return -ENOTSUPP;
1880 }
1881
1882 meta_left = btf->data_size - sizeof(*hdr);
1883 if (!meta_left) {
1884 btf_verifier_log(env, "No data");
1885 return -EINVAL;
1886 }
1887
1888 if (meta_left < hdr->type_off || hdr->str_off <= hdr->type_off ||
1889 /* Type section must align to 4 bytes */
1890 hdr->type_off & (sizeof(u32) - 1)) {
1891 btf_verifier_log(env, "Invalid type_off");
1892 return -EINVAL;
1893 }
1894
1895 if (meta_left < hdr->str_off ||
1896 meta_left - hdr->str_off < hdr->str_len) {
1897 btf_verifier_log(env, "Invalid str_off or str_len");
1898 return -EINVAL;
1899 }
1900
1901 btf->nohdr_data = btf->hdr + 1;
1902
1903 return 0;
1904}
1905
1906static struct btf *btf_parse(void __user *btf_data, u32 btf_data_size,
1907 u32 log_level, char __user *log_ubuf, u32 log_size)
1908{
1909 struct btf_verifier_env *env = NULL;
1910 struct bpf_verifier_log *log;
1911 struct btf *btf = NULL;
1912 u8 *data;
1913 int err;
1914
1915 if (btf_data_size > BTF_MAX_SIZE)
1916 return ERR_PTR(-E2BIG);
1917
1918 env = kzalloc(sizeof(*env), GFP_KERNEL | __GFP_NOWARN);
1919 if (!env)
1920 return ERR_PTR(-ENOMEM);
1921
1922 log = &env->log;
1923 if (log_level || log_ubuf || log_size) {
1924 /* user requested verbose verifier output
1925 * and supplied buffer to store the verification trace
1926 */
1927 log->level = log_level;
1928 log->ubuf = log_ubuf;
1929 log->len_total = log_size;
1930
1931 /* log attributes have to be sane */
1932 if (log->len_total < 128 || log->len_total > UINT_MAX >> 8 ||
1933 !log->level || !log->ubuf) {
1934 err = -EINVAL;
1935 goto errout;
1936 }
1937 }
1938
1939 btf = kzalloc(sizeof(*btf), GFP_KERNEL | __GFP_NOWARN);
1940 if (!btf) {
1941 err = -ENOMEM;
1942 goto errout;
1943 }
1944
1945 data = kvmalloc(btf_data_size, GFP_KERNEL | __GFP_NOWARN);
1946 if (!data) {
1947 err = -ENOMEM;
1948 goto errout;
1949 }
1950
1951 btf->data = data;
1952 btf->data_size = btf_data_size;
1953
1954 if (copy_from_user(data, btf_data, btf_data_size)) {
1955 err = -EFAULT;
1956 goto errout;
1957 }
1958
1959 env->btf = btf;
1960
1961 err = btf_parse_hdr(env);
1962 if (err)
1963 goto errout;
1964
1965 err = btf_parse_str_sec(env);
1966 if (err)
1967 goto errout;
1968
1969 err = btf_parse_type_sec(env);
1970 if (err)
1971 goto errout;
1972
1973 if (!err && log->level && bpf_verifier_log_full(log)) {
1974 err = -ENOSPC;
1975 goto errout;
1976 }
1977
1978 if (!err) {
1979 btf_verifier_env_free(env);
1980 btf_get(btf);
1981 return btf;
1982 }
1983
1984errout:
1985 btf_verifier_env_free(env);
1986 if (btf)
1987 btf_free(btf);
1988 return ERR_PTR(err);
1989}
1990
1991void btf_type_seq_show(const struct btf *btf, u32 type_id, void *obj,
1992 struct seq_file *m)
1993{
1994 const struct btf_type *t = btf_type_by_id(btf, type_id);
1995
1996 btf_type_ops(t)->seq_show(btf, t, type_id, obj, 0, m);
1997}
1998
1999static int btf_release(struct inode *inode, struct file *filp)
2000{
2001 btf_put(filp->private_data);
2002 return 0;
2003}
2004
2005const struct file_operations btf_fops = {
2006 .release = btf_release,
2007};
2008
2009int btf_new_fd(const union bpf_attr *attr)
2010{
2011 struct btf *btf;
2012 int fd;
2013
2014 btf = btf_parse(u64_to_user_ptr(attr->btf),
2015 attr->btf_size, attr->btf_log_level,
2016 u64_to_user_ptr(attr->btf_log_buf),
2017 attr->btf_log_size);
2018 if (IS_ERR(btf))
2019 return PTR_ERR(btf);
2020
2021 fd = anon_inode_getfd("btf", &btf_fops, btf,
2022 O_RDONLY | O_CLOEXEC);
2023 if (fd < 0)
2024 btf_put(btf);
2025
2026 return fd;
2027}
2028
2029struct btf *btf_get_by_fd(int fd)
2030{
2031 struct btf *btf;
2032 struct fd f;
2033
2034 f = fdget(fd);
2035
2036 if (!f.file)
2037 return ERR_PTR(-EBADF);
2038
2039 if (f.file->f_op != &btf_fops) {
2040 fdput(f);
2041 return ERR_PTR(-EINVAL);
2042 }
2043
2044 btf = f.file->private_data;
2045 btf_get(btf);
2046 fdput(f);
2047
2048 return btf;
2049}
2050
2051int btf_get_info_by_fd(const struct btf *btf,
2052 const union bpf_attr *attr,
2053 union bpf_attr __user *uattr)
2054{
2055 void __user *udata = u64_to_user_ptr(attr->info.info);
2056 u32 copy_len = min_t(u32, btf->data_size,
2057 attr->info.info_len);
2058
2059 if (copy_to_user(udata, btf->data, copy_len) ||
2060 put_user(btf->data_size, &uattr->info.info_len))
2061 return -EFAULT;
2062
2063 return 0;
2064}
diff --git a/kernel/bpf/inode.c b/kernel/bpf/inode.c
index bf6da59ae0d0..a41343009ccc 100644
--- a/kernel/bpf/inode.c
+++ b/kernel/bpf/inode.c
@@ -150,8 +150,154 @@ static int bpf_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
150 return 0; 150 return 0;
151} 151}
152 152
153struct map_iter {
154 void *key;
155 bool done;
156};
157
158static struct map_iter *map_iter(struct seq_file *m)
159{
160 return m->private;
161}
162
163static struct bpf_map *seq_file_to_map(struct seq_file *m)
164{
165 return file_inode(m->file)->i_private;
166}
167
168static void map_iter_free(struct map_iter *iter)
169{
170 if (iter) {
171 kfree(iter->key);
172 kfree(iter);
173 }
174}
175
176static struct map_iter *map_iter_alloc(struct bpf_map *map)
177{
178 struct map_iter *iter;
179
180 iter = kzalloc(sizeof(*iter), GFP_KERNEL | __GFP_NOWARN);
181 if (!iter)
182 goto error;
183
184 iter->key = kzalloc(map->key_size, GFP_KERNEL | __GFP_NOWARN);
185 if (!iter->key)
186 goto error;
187
188 return iter;
189
190error:
191 map_iter_free(iter);
192 return NULL;
193}
194
195static void *map_seq_next(struct seq_file *m, void *v, loff_t *pos)
196{
197 struct bpf_map *map = seq_file_to_map(m);
198 void *key = map_iter(m)->key;
199
200 if (map_iter(m)->done)
201 return NULL;
202
203 if (unlikely(v == SEQ_START_TOKEN))
204 goto done;
205
206 if (map->ops->map_get_next_key(map, key, key)) {
207 map_iter(m)->done = true;
208 return NULL;
209 }
210
211done:
212 ++(*pos);
213 return key;
214}
215
216static void *map_seq_start(struct seq_file *m, loff_t *pos)
217{
218 if (map_iter(m)->done)
219 return NULL;
220
221 return *pos ? map_iter(m)->key : SEQ_START_TOKEN;
222}
223
224static void map_seq_stop(struct seq_file *m, void *v)
225{
226}
227
228static int map_seq_show(struct seq_file *m, void *v)
229{
230 struct bpf_map *map = seq_file_to_map(m);
231 void *key = map_iter(m)->key;
232
233 if (unlikely(v == SEQ_START_TOKEN)) {
234 seq_puts(m, "# WARNING!! The output is for debug purpose only\n");
235 seq_puts(m, "# WARNING!! The output format will change\n");
236 } else {
237 map->ops->map_seq_show_elem(map, key, m);
238 }
239
240 return 0;
241}
242
243static const struct seq_operations bpffs_map_seq_ops = {
244 .start = map_seq_start,
245 .next = map_seq_next,
246 .show = map_seq_show,
247 .stop = map_seq_stop,
248};
249
250static int bpffs_map_open(struct inode *inode, struct file *file)
251{
252 struct bpf_map *map = inode->i_private;
253 struct map_iter *iter;
254 struct seq_file *m;
255 int err;
256
257 iter = map_iter_alloc(map);
258 if (!iter)
259 return -ENOMEM;
260
261 err = seq_open(file, &bpffs_map_seq_ops);
262 if (err) {
263 map_iter_free(iter);
264 return err;
265 }
266
267 m = file->private_data;
268 m->private = iter;
269
270 return 0;
271}
272
273static int bpffs_map_release(struct inode *inode, struct file *file)
274{
275 struct seq_file *m = file->private_data;
276
277 map_iter_free(map_iter(m));
278
279 return seq_release(inode, file);
280}
281
282/* bpffs_map_fops should only implement the basic
283 * read operation for a BPF map. The purpose is to
284 * provide a simple user intuitive way to do
285 * "cat bpffs/pathto/a-pinned-map".
286 *
287 * Other operations (e.g. write, lookup...) should be realized by
288 * the userspace tools (e.g. bpftool) through the
289 * BPF_OBJ_GET_INFO_BY_FD and the map's lookup/update
290 * interface.
291 */
292static const struct file_operations bpffs_map_fops = {
293 .open = bpffs_map_open,
294 .read = seq_read,
295 .release = bpffs_map_release,
296};
297
153static int bpf_mkobj_ops(struct dentry *dentry, umode_t mode, void *raw, 298static int bpf_mkobj_ops(struct dentry *dentry, umode_t mode, void *raw,
154 const struct inode_operations *iops) 299 const struct inode_operations *iops,
300 const struct file_operations *fops)
155{ 301{
156 struct inode *dir = dentry->d_parent->d_inode; 302 struct inode *dir = dentry->d_parent->d_inode;
157 struct inode *inode = bpf_get_inode(dir->i_sb, dir, mode); 303 struct inode *inode = bpf_get_inode(dir->i_sb, dir, mode);
@@ -159,6 +305,7 @@ static int bpf_mkobj_ops(struct dentry *dentry, umode_t mode, void *raw,
159 return PTR_ERR(inode); 305 return PTR_ERR(inode);
160 306
161 inode->i_op = iops; 307 inode->i_op = iops;
308 inode->i_fop = fops;
162 inode->i_private = raw; 309 inode->i_private = raw;
163 310
164 bpf_dentry_finalize(dentry, inode, dir); 311 bpf_dentry_finalize(dentry, inode, dir);
@@ -167,12 +314,15 @@ static int bpf_mkobj_ops(struct dentry *dentry, umode_t mode, void *raw,
167 314
168static int bpf_mkprog(struct dentry *dentry, umode_t mode, void *arg) 315static int bpf_mkprog(struct dentry *dentry, umode_t mode, void *arg)
169{ 316{
170 return bpf_mkobj_ops(dentry, mode, arg, &bpf_prog_iops); 317 return bpf_mkobj_ops(dentry, mode, arg, &bpf_prog_iops, NULL);
171} 318}
172 319
173static int bpf_mkmap(struct dentry *dentry, umode_t mode, void *arg) 320static int bpf_mkmap(struct dentry *dentry, umode_t mode, void *arg)
174{ 321{
175 return bpf_mkobj_ops(dentry, mode, arg, &bpf_map_iops); 322 struct bpf_map *map = arg;
323
324 return bpf_mkobj_ops(dentry, mode, arg, &bpf_map_iops,
325 map->btf ? &bpffs_map_fops : NULL);
176} 326}
177 327
178static struct dentry * 328static struct dentry *
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 4ca46df19c9a..fe23dc5a3ec4 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -11,6 +11,7 @@
11 */ 11 */
12#include <linux/bpf.h> 12#include <linux/bpf.h>
13#include <linux/bpf_trace.h> 13#include <linux/bpf_trace.h>
14#include <linux/btf.h>
14#include <linux/syscalls.h> 15#include <linux/syscalls.h>
15#include <linux/slab.h> 16#include <linux/slab.h>
16#include <linux/sched/signal.h> 17#include <linux/sched/signal.h>
@@ -26,6 +27,7 @@
26#include <linux/cred.h> 27#include <linux/cred.h>
27#include <linux/timekeeping.h> 28#include <linux/timekeeping.h>
28#include <linux/ctype.h> 29#include <linux/ctype.h>
30#include <linux/btf.h>
29 31
30#define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY || \ 32#define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY || \
31 (map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \ 33 (map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \
@@ -250,6 +252,7 @@ static void bpf_map_free_deferred(struct work_struct *work)
250 252
251 bpf_map_uncharge_memlock(map); 253 bpf_map_uncharge_memlock(map);
252 security_bpf_map_free(map); 254 security_bpf_map_free(map);
255 btf_put(map->btf);
253 /* implementation dependent freeing */ 256 /* implementation dependent freeing */
254 map->ops->map_free(map); 257 map->ops->map_free(map);
255} 258}
@@ -415,7 +418,7 @@ static int bpf_obj_name_cpy(char *dst, const char *src)
415 return 0; 418 return 0;
416} 419}
417 420
418#define BPF_MAP_CREATE_LAST_FIELD map_ifindex 421#define BPF_MAP_CREATE_LAST_FIELD btf_value_id
419/* called via syscall */ 422/* called via syscall */
420static int map_create(union bpf_attr *attr) 423static int map_create(union bpf_attr *attr)
421{ 424{
@@ -449,6 +452,33 @@ static int map_create(union bpf_attr *attr)
449 atomic_set(&map->refcnt, 1); 452 atomic_set(&map->refcnt, 1);
450 atomic_set(&map->usercnt, 1); 453 atomic_set(&map->usercnt, 1);
451 454
455 if (bpf_map_support_seq_show(map) &&
456 (attr->btf_key_id || attr->btf_value_id)) {
457 struct btf *btf;
458
459 if (!attr->btf_key_id || !attr->btf_value_id) {
460 err = -EINVAL;
461 goto free_map_nouncharge;
462 }
463
464 btf = btf_get_by_fd(attr->btf_fd);
465 if (IS_ERR(btf)) {
466 err = PTR_ERR(btf);
467 goto free_map_nouncharge;
468 }
469
470 err = map->ops->map_check_btf(map, btf, attr->btf_key_id,
471 attr->btf_value_id);
472 if (err) {
473 btf_put(btf);
474 goto free_map_nouncharge;
475 }
476
477 map->btf = btf;
478 map->btf_key_id = attr->btf_key_id;
479 map->btf_value_id = attr->btf_value_id;
480 }
481
452 err = security_bpf_map_alloc(map); 482 err = security_bpf_map_alloc(map);
453 if (err) 483 if (err)
454 goto free_map_nouncharge; 484 goto free_map_nouncharge;
@@ -481,6 +511,7 @@ free_map:
481free_map_sec: 511free_map_sec:
482 security_bpf_map_free(map); 512 security_bpf_map_free(map);
483free_map_nouncharge: 513free_map_nouncharge:
514 btf_put(map->btf);
484 map->ops->map_free(map); 515 map->ops->map_free(map);
485 return err; 516 return err;
486} 517}
@@ -2016,6 +2047,8 @@ static int bpf_obj_get_info_by_fd(const union bpf_attr *attr,
2016 else if (f.file->f_op == &bpf_map_fops) 2047 else if (f.file->f_op == &bpf_map_fops)
2017 err = bpf_map_get_info_by_fd(f.file->private_data, attr, 2048 err = bpf_map_get_info_by_fd(f.file->private_data, attr,
2018 uattr); 2049 uattr);
2050 else if (f.file->f_op == &btf_fops)
2051 err = btf_get_info_by_fd(f.file->private_data, attr, uattr);
2019 else 2052 else
2020 err = -EINVAL; 2053 err = -EINVAL;
2021 2054
@@ -2023,6 +2056,19 @@ static int bpf_obj_get_info_by_fd(const union bpf_attr *attr,
2023 return err; 2056 return err;
2024} 2057}
2025 2058
2059#define BPF_BTF_LOAD_LAST_FIELD btf_log_level
2060
2061static int bpf_btf_load(const union bpf_attr *attr)
2062{
2063 if (CHECK_ATTR(BPF_BTF_LOAD))
2064 return -EINVAL;
2065
2066 if (!capable(CAP_SYS_ADMIN))
2067 return -EPERM;
2068
2069 return btf_new_fd(attr);
2070}
2071
2026SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size) 2072SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size)
2027{ 2073{
2028 union bpf_attr attr = {}; 2074 union bpf_attr attr = {};
@@ -2103,6 +2149,9 @@ SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, siz
2103 case BPF_RAW_TRACEPOINT_OPEN: 2149 case BPF_RAW_TRACEPOINT_OPEN:
2104 err = bpf_raw_tracepoint_open(&attr); 2150 err = bpf_raw_tracepoint_open(&attr);
2105 break; 2151 break;
2152 case BPF_BTF_LOAD:
2153 err = bpf_btf_load(&attr);
2154 break;
2106 default: 2155 default:
2107 err = -EINVAL; 2156 err = -EINVAL;
2108 break; 2157 break;
generated by cgit v1.2.2 (git 2.25.0) at 2025-09-02 09:15:29 -0400