diff options
Diffstat (limited to 'tools/lib/bpf/libbpf.c')
-rw-r--r-- | tools/lib/bpf/libbpf.c | 342 |
1 files changed, 295 insertions, 47 deletions
diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index 6dba0f01673b..f7b245fbb960 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c | |||
@@ -7,6 +7,7 @@ | |||
7 | * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> | 7 | * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> |
8 | * Copyright (C) 2015 Huawei Inc. | 8 | * Copyright (C) 2015 Huawei Inc. |
9 | * Copyright (C) 2017 Nicira, Inc. | 9 | * Copyright (C) 2017 Nicira, Inc. |
10 | * Copyright (C) 2019 Isovalent, Inc. | ||
10 | */ | 11 | */ |
11 | 12 | ||
12 | #ifndef _GNU_SOURCE | 13 | #ifndef _GNU_SOURCE |
@@ -149,6 +150,7 @@ struct bpf_program { | |||
149 | enum { | 150 | enum { |
150 | RELO_LD64, | 151 | RELO_LD64, |
151 | RELO_CALL, | 152 | RELO_CALL, |
153 | RELO_DATA, | ||
152 | } type; | 154 | } type; |
153 | int insn_idx; | 155 | int insn_idx; |
154 | union { | 156 | union { |
@@ -182,6 +184,19 @@ struct bpf_program { | |||
182 | __u32 line_info_cnt; | 184 | __u32 line_info_cnt; |
183 | }; | 185 | }; |
184 | 186 | ||
187 | enum libbpf_map_type { | ||
188 | LIBBPF_MAP_UNSPEC, | ||
189 | LIBBPF_MAP_DATA, | ||
190 | LIBBPF_MAP_BSS, | ||
191 | LIBBPF_MAP_RODATA, | ||
192 | }; | ||
193 | |||
194 | static const char * const libbpf_type_to_btf_name[] = { | ||
195 | [LIBBPF_MAP_DATA] = ".data", | ||
196 | [LIBBPF_MAP_BSS] = ".bss", | ||
197 | [LIBBPF_MAP_RODATA] = ".rodata", | ||
198 | }; | ||
199 | |||
185 | struct bpf_map { | 200 | struct bpf_map { |
186 | int fd; | 201 | int fd; |
187 | char *name; | 202 | char *name; |
@@ -193,11 +208,18 @@ struct bpf_map { | |||
193 | __u32 btf_value_type_id; | 208 | __u32 btf_value_type_id; |
194 | void *priv; | 209 | void *priv; |
195 | bpf_map_clear_priv_t clear_priv; | 210 | bpf_map_clear_priv_t clear_priv; |
211 | enum libbpf_map_type libbpf_type; | ||
212 | }; | ||
213 | |||
214 | struct bpf_secdata { | ||
215 | void *rodata; | ||
216 | void *data; | ||
196 | }; | 217 | }; |
197 | 218 | ||
198 | static LIST_HEAD(bpf_objects_list); | 219 | static LIST_HEAD(bpf_objects_list); |
199 | 220 | ||
200 | struct bpf_object { | 221 | struct bpf_object { |
222 | char name[BPF_OBJ_NAME_LEN]; | ||
201 | char license[64]; | 223 | char license[64]; |
202 | __u32 kern_version; | 224 | __u32 kern_version; |
203 | 225 | ||
@@ -205,6 +227,7 @@ struct bpf_object { | |||
205 | size_t nr_programs; | 227 | size_t nr_programs; |
206 | struct bpf_map *maps; | 228 | struct bpf_map *maps; |
207 | size_t nr_maps; | 229 | size_t nr_maps; |
230 | struct bpf_secdata sections; | ||
208 | 231 | ||
209 | bool loaded; | 232 | bool loaded; |
210 | bool has_pseudo_calls; | 233 | bool has_pseudo_calls; |
@@ -220,6 +243,9 @@ struct bpf_object { | |||
220 | Elf *elf; | 243 | Elf *elf; |
221 | GElf_Ehdr ehdr; | 244 | GElf_Ehdr ehdr; |
222 | Elf_Data *symbols; | 245 | Elf_Data *symbols; |
246 | Elf_Data *data; | ||
247 | Elf_Data *rodata; | ||
248 | Elf_Data *bss; | ||
223 | size_t strtabidx; | 249 | size_t strtabidx; |
224 | struct { | 250 | struct { |
225 | GElf_Shdr shdr; | 251 | GElf_Shdr shdr; |
@@ -228,6 +254,9 @@ struct bpf_object { | |||
228 | int nr_reloc; | 254 | int nr_reloc; |
229 | int maps_shndx; | 255 | int maps_shndx; |
230 | int text_shndx; | 256 | int text_shndx; |
257 | int data_shndx; | ||
258 | int rodata_shndx; | ||
259 | int bss_shndx; | ||
231 | } efile; | 260 | } efile; |
232 | /* | 261 | /* |
233 | * All loaded bpf_object is linked in a list, which is | 262 | * All loaded bpf_object is linked in a list, which is |
@@ -449,6 +478,7 @@ static struct bpf_object *bpf_object__new(const char *path, | |||
449 | size_t obj_buf_sz) | 478 | size_t obj_buf_sz) |
450 | { | 479 | { |
451 | struct bpf_object *obj; | 480 | struct bpf_object *obj; |
481 | char *end; | ||
452 | 482 | ||
453 | obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1); | 483 | obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1); |
454 | if (!obj) { | 484 | if (!obj) { |
@@ -457,8 +487,14 @@ static struct bpf_object *bpf_object__new(const char *path, | |||
457 | } | 487 | } |
458 | 488 | ||
459 | strcpy(obj->path, path); | 489 | strcpy(obj->path, path); |
460 | obj->efile.fd = -1; | 490 | /* Using basename() GNU version which doesn't modify arg. */ |
491 | strncpy(obj->name, basename((void *)path), | ||
492 | sizeof(obj->name) - 1); | ||
493 | end = strchr(obj->name, '.'); | ||
494 | if (end) | ||
495 | *end = 0; | ||
461 | 496 | ||
497 | obj->efile.fd = -1; | ||
462 | /* | 498 | /* |
463 | * Caller of this function should also calls | 499 | * Caller of this function should also calls |
464 | * bpf_object__elf_finish() after data collection to return | 500 | * bpf_object__elf_finish() after data collection to return |
@@ -468,6 +504,9 @@ static struct bpf_object *bpf_object__new(const char *path, | |||
468 | obj->efile.obj_buf = obj_buf; | 504 | obj->efile.obj_buf = obj_buf; |
469 | obj->efile.obj_buf_sz = obj_buf_sz; | 505 | obj->efile.obj_buf_sz = obj_buf_sz; |
470 | obj->efile.maps_shndx = -1; | 506 | obj->efile.maps_shndx = -1; |
507 | obj->efile.data_shndx = -1; | ||
508 | obj->efile.rodata_shndx = -1; | ||
509 | obj->efile.bss_shndx = -1; | ||
471 | 510 | ||
472 | obj->loaded = false; | 511 | obj->loaded = false; |
473 | 512 | ||
@@ -486,6 +525,9 @@ static void bpf_object__elf_finish(struct bpf_object *obj) | |||
486 | obj->efile.elf = NULL; | 525 | obj->efile.elf = NULL; |
487 | } | 526 | } |
488 | obj->efile.symbols = NULL; | 527 | obj->efile.symbols = NULL; |
528 | obj->efile.data = NULL; | ||
529 | obj->efile.rodata = NULL; | ||
530 | obj->efile.bss = NULL; | ||
489 | 531 | ||
490 | zfree(&obj->efile.reloc); | 532 | zfree(&obj->efile.reloc); |
491 | obj->efile.nr_reloc = 0; | 533 | obj->efile.nr_reloc = 0; |
@@ -627,27 +669,76 @@ static bool bpf_map_type__is_map_in_map(enum bpf_map_type type) | |||
627 | return false; | 669 | return false; |
628 | } | 670 | } |
629 | 671 | ||
672 | static bool bpf_object__has_maps(const struct bpf_object *obj) | ||
673 | { | ||
674 | return obj->efile.maps_shndx >= 0 || | ||
675 | obj->efile.data_shndx >= 0 || | ||
676 | obj->efile.rodata_shndx >= 0 || | ||
677 | obj->efile.bss_shndx >= 0; | ||
678 | } | ||
679 | |||
680 | static int | ||
681 | bpf_object__init_internal_map(struct bpf_object *obj, struct bpf_map *map, | ||
682 | enum libbpf_map_type type, Elf_Data *data, | ||
683 | void **data_buff) | ||
684 | { | ||
685 | struct bpf_map_def *def = &map->def; | ||
686 | char map_name[BPF_OBJ_NAME_LEN]; | ||
687 | |||
688 | map->libbpf_type = type; | ||
689 | map->offset = ~(typeof(map->offset))0; | ||
690 | snprintf(map_name, sizeof(map_name), "%.8s%.7s", obj->name, | ||
691 | libbpf_type_to_btf_name[type]); | ||
692 | map->name = strdup(map_name); | ||
693 | if (!map->name) { | ||
694 | pr_warning("failed to alloc map name\n"); | ||
695 | return -ENOMEM; | ||
696 | } | ||
697 | |||
698 | def->type = BPF_MAP_TYPE_ARRAY; | ||
699 | def->key_size = sizeof(int); | ||
700 | def->value_size = data->d_size; | ||
701 | def->max_entries = 1; | ||
702 | def->map_flags = type == LIBBPF_MAP_RODATA ? | ||
703 | BPF_F_RDONLY_PROG : 0; | ||
704 | if (data_buff) { | ||
705 | *data_buff = malloc(data->d_size); | ||
706 | if (!*data_buff) { | ||
707 | zfree(&map->name); | ||
708 | pr_warning("failed to alloc map content buffer\n"); | ||
709 | return -ENOMEM; | ||
710 | } | ||
711 | memcpy(*data_buff, data->d_buf, data->d_size); | ||
712 | } | ||
713 | |||
714 | pr_debug("map %ld is \"%s\"\n", map - obj->maps, map->name); | ||
715 | return 0; | ||
716 | } | ||
717 | |||
630 | static int | 718 | static int |
631 | bpf_object__init_maps(struct bpf_object *obj, int flags) | 719 | bpf_object__init_maps(struct bpf_object *obj, int flags) |
632 | { | 720 | { |
721 | int i, map_idx, map_def_sz, nr_syms, nr_maps = 0, nr_maps_glob = 0; | ||
633 | bool strict = !(flags & MAPS_RELAX_COMPAT); | 722 | bool strict = !(flags & MAPS_RELAX_COMPAT); |
634 | int i, map_idx, map_def_sz, nr_maps = 0; | ||
635 | Elf_Scn *scn; | ||
636 | Elf_Data *data = NULL; | ||
637 | Elf_Data *symbols = obj->efile.symbols; | 723 | Elf_Data *symbols = obj->efile.symbols; |
724 | Elf_Data *data = NULL; | ||
725 | int ret = 0; | ||
638 | 726 | ||
639 | if (obj->efile.maps_shndx < 0) | ||
640 | return -EINVAL; | ||
641 | if (!symbols) | 727 | if (!symbols) |
642 | return -EINVAL; | 728 | return -EINVAL; |
729 | nr_syms = symbols->d_size / sizeof(GElf_Sym); | ||
643 | 730 | ||
644 | scn = elf_getscn(obj->efile.elf, obj->efile.maps_shndx); | 731 | if (obj->efile.maps_shndx >= 0) { |
645 | if (scn) | 732 | Elf_Scn *scn = elf_getscn(obj->efile.elf, |
646 | data = elf_getdata(scn, NULL); | 733 | obj->efile.maps_shndx); |
647 | if (!scn || !data) { | 734 | |
648 | pr_warning("failed to get Elf_Data from map section %d\n", | 735 | if (scn) |
649 | obj->efile.maps_shndx); | 736 | data = elf_getdata(scn, NULL); |
650 | return -EINVAL; | 737 | if (!scn || !data) { |
738 | pr_warning("failed to get Elf_Data from map section %d\n", | ||
739 | obj->efile.maps_shndx); | ||
740 | return -EINVAL; | ||
741 | } | ||
651 | } | 742 | } |
652 | 743 | ||
653 | /* | 744 | /* |
@@ -657,7 +748,13 @@ bpf_object__init_maps(struct bpf_object *obj, int flags) | |||
657 | * | 748 | * |
658 | * TODO: Detect array of map and report error. | 749 | * TODO: Detect array of map and report error. |
659 | */ | 750 | */ |
660 | for (i = 0; i < symbols->d_size / sizeof(GElf_Sym); i++) { | 751 | if (obj->efile.data_shndx >= 0) |
752 | nr_maps_glob++; | ||
753 | if (obj->efile.rodata_shndx >= 0) | ||
754 | nr_maps_glob++; | ||
755 | if (obj->efile.bss_shndx >= 0) | ||
756 | nr_maps_glob++; | ||
757 | for (i = 0; data && i < nr_syms; i++) { | ||
661 | GElf_Sym sym; | 758 | GElf_Sym sym; |
662 | 759 | ||
663 | if (!gelf_getsym(symbols, i, &sym)) | 760 | if (!gelf_getsym(symbols, i, &sym)) |
@@ -670,19 +767,21 @@ bpf_object__init_maps(struct bpf_object *obj, int flags) | |||
670 | /* Alloc obj->maps and fill nr_maps. */ | 767 | /* Alloc obj->maps and fill nr_maps. */ |
671 | pr_debug("maps in %s: %d maps in %zd bytes\n", obj->path, | 768 | pr_debug("maps in %s: %d maps in %zd bytes\n", obj->path, |
672 | nr_maps, data->d_size); | 769 | nr_maps, data->d_size); |
673 | 770 | if (!nr_maps && !nr_maps_glob) | |
674 | if (!nr_maps) | ||
675 | return 0; | 771 | return 0; |
676 | 772 | ||
677 | /* Assume equally sized map definitions */ | 773 | /* Assume equally sized map definitions */ |
678 | map_def_sz = data->d_size / nr_maps; | 774 | if (data) { |
679 | if (!data->d_size || (data->d_size % nr_maps) != 0) { | 775 | map_def_sz = data->d_size / nr_maps; |
680 | pr_warning("unable to determine map definition size " | 776 | if (!data->d_size || (data->d_size % nr_maps) != 0) { |
681 | "section %s, %d maps in %zd bytes\n", | 777 | pr_warning("unable to determine map definition size " |
682 | obj->path, nr_maps, data->d_size); | 778 | "section %s, %d maps in %zd bytes\n", |
683 | return -EINVAL; | 779 | obj->path, nr_maps, data->d_size); |
780 | return -EINVAL; | ||
781 | } | ||
684 | } | 782 | } |
685 | 783 | ||
784 | nr_maps += nr_maps_glob; | ||
686 | obj->maps = calloc(nr_maps, sizeof(obj->maps[0])); | 785 | obj->maps = calloc(nr_maps, sizeof(obj->maps[0])); |
687 | if (!obj->maps) { | 786 | if (!obj->maps) { |
688 | pr_warning("alloc maps for object failed\n"); | 787 | pr_warning("alloc maps for object failed\n"); |
@@ -703,7 +802,7 @@ bpf_object__init_maps(struct bpf_object *obj, int flags) | |||
703 | /* | 802 | /* |
704 | * Fill obj->maps using data in "maps" section. | 803 | * Fill obj->maps using data in "maps" section. |
705 | */ | 804 | */ |
706 | for (i = 0, map_idx = 0; i < symbols->d_size / sizeof(GElf_Sym); i++) { | 805 | for (i = 0, map_idx = 0; data && i < nr_syms; i++) { |
707 | GElf_Sym sym; | 806 | GElf_Sym sym; |
708 | const char *map_name; | 807 | const char *map_name; |
709 | struct bpf_map_def *def; | 808 | struct bpf_map_def *def; |
@@ -716,6 +815,8 @@ bpf_object__init_maps(struct bpf_object *obj, int flags) | |||
716 | map_name = elf_strptr(obj->efile.elf, | 815 | map_name = elf_strptr(obj->efile.elf, |
717 | obj->efile.strtabidx, | 816 | obj->efile.strtabidx, |
718 | sym.st_name); | 817 | sym.st_name); |
818 | |||
819 | obj->maps[map_idx].libbpf_type = LIBBPF_MAP_UNSPEC; | ||
719 | obj->maps[map_idx].offset = sym.st_value; | 820 | obj->maps[map_idx].offset = sym.st_value; |
720 | if (sym.st_value + map_def_sz > data->d_size) { | 821 | if (sym.st_value + map_def_sz > data->d_size) { |
721 | pr_warning("corrupted maps section in %s: last map \"%s\" too small\n", | 822 | pr_warning("corrupted maps section in %s: last map \"%s\" too small\n", |
@@ -764,8 +865,27 @@ bpf_object__init_maps(struct bpf_object *obj, int flags) | |||
764 | map_idx++; | 865 | map_idx++; |
765 | } | 866 | } |
766 | 867 | ||
767 | qsort(obj->maps, obj->nr_maps, sizeof(obj->maps[0]), compare_bpf_map); | 868 | /* |
768 | return 0; | 869 | * Populate rest of obj->maps with libbpf internal maps. |
870 | */ | ||
871 | if (obj->efile.data_shndx >= 0) | ||
872 | ret = bpf_object__init_internal_map(obj, &obj->maps[map_idx++], | ||
873 | LIBBPF_MAP_DATA, | ||
874 | obj->efile.data, | ||
875 | &obj->sections.data); | ||
876 | if (!ret && obj->efile.rodata_shndx >= 0) | ||
877 | ret = bpf_object__init_internal_map(obj, &obj->maps[map_idx++], | ||
878 | LIBBPF_MAP_RODATA, | ||
879 | obj->efile.rodata, | ||
880 | &obj->sections.rodata); | ||
881 | if (!ret && obj->efile.bss_shndx >= 0) | ||
882 | ret = bpf_object__init_internal_map(obj, &obj->maps[map_idx++], | ||
883 | LIBBPF_MAP_BSS, | ||
884 | obj->efile.bss, NULL); | ||
885 | if (!ret) | ||
886 | qsort(obj->maps, obj->nr_maps, sizeof(obj->maps[0]), | ||
887 | compare_bpf_map); | ||
888 | return ret; | ||
769 | } | 889 | } |
770 | 890 | ||
771 | static bool section_have_execinstr(struct bpf_object *obj, int idx) | 891 | static bool section_have_execinstr(struct bpf_object *obj, int idx) |
@@ -885,6 +1005,14 @@ static int bpf_object__elf_collect(struct bpf_object *obj, int flags) | |||
885 | pr_warning("failed to alloc program %s (%s): %s", | 1005 | pr_warning("failed to alloc program %s (%s): %s", |
886 | name, obj->path, cp); | 1006 | name, obj->path, cp); |
887 | } | 1007 | } |
1008 | } else if (strcmp(name, ".data") == 0) { | ||
1009 | obj->efile.data = data; | ||
1010 | obj->efile.data_shndx = idx; | ||
1011 | } else if (strcmp(name, ".rodata") == 0) { | ||
1012 | obj->efile.rodata = data; | ||
1013 | obj->efile.rodata_shndx = idx; | ||
1014 | } else { | ||
1015 | pr_debug("skip section(%d) %s\n", idx, name); | ||
888 | } | 1016 | } |
889 | } else if (sh.sh_type == SHT_REL) { | 1017 | } else if (sh.sh_type == SHT_REL) { |
890 | void *reloc = obj->efile.reloc; | 1018 | void *reloc = obj->efile.reloc; |
@@ -912,6 +1040,9 @@ static int bpf_object__elf_collect(struct bpf_object *obj, int flags) | |||
912 | obj->efile.reloc[n].shdr = sh; | 1040 | obj->efile.reloc[n].shdr = sh; |
913 | obj->efile.reloc[n].data = data; | 1041 | obj->efile.reloc[n].data = data; |
914 | } | 1042 | } |
1043 | } else if (sh.sh_type == SHT_NOBITS && strcmp(name, ".bss") == 0) { | ||
1044 | obj->efile.bss = data; | ||
1045 | obj->efile.bss_shndx = idx; | ||
915 | } else { | 1046 | } else { |
916 | pr_debug("skip section(%d) %s\n", idx, name); | 1047 | pr_debug("skip section(%d) %s\n", idx, name); |
917 | } | 1048 | } |
@@ -938,7 +1069,7 @@ static int bpf_object__elf_collect(struct bpf_object *obj, int flags) | |||
938 | } | 1069 | } |
939 | } | 1070 | } |
940 | } | 1071 | } |
941 | if (obj->efile.maps_shndx >= 0) { | 1072 | if (bpf_object__has_maps(obj)) { |
942 | err = bpf_object__init_maps(obj, flags); | 1073 | err = bpf_object__init_maps(obj, flags); |
943 | if (err) | 1074 | if (err) |
944 | goto out; | 1075 | goto out; |
@@ -974,13 +1105,46 @@ bpf_object__find_program_by_title(struct bpf_object *obj, const char *title) | |||
974 | return NULL; | 1105 | return NULL; |
975 | } | 1106 | } |
976 | 1107 | ||
1108 | static bool bpf_object__shndx_is_data(const struct bpf_object *obj, | ||
1109 | int shndx) | ||
1110 | { | ||
1111 | return shndx == obj->efile.data_shndx || | ||
1112 | shndx == obj->efile.bss_shndx || | ||
1113 | shndx == obj->efile.rodata_shndx; | ||
1114 | } | ||
1115 | |||
1116 | static bool bpf_object__shndx_is_maps(const struct bpf_object *obj, | ||
1117 | int shndx) | ||
1118 | { | ||
1119 | return shndx == obj->efile.maps_shndx; | ||
1120 | } | ||
1121 | |||
1122 | static bool bpf_object__relo_in_known_section(const struct bpf_object *obj, | ||
1123 | int shndx) | ||
1124 | { | ||
1125 | return shndx == obj->efile.text_shndx || | ||
1126 | bpf_object__shndx_is_maps(obj, shndx) || | ||
1127 | bpf_object__shndx_is_data(obj, shndx); | ||
1128 | } | ||
1129 | |||
1130 | static enum libbpf_map_type | ||
1131 | bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx) | ||
1132 | { | ||
1133 | if (shndx == obj->efile.data_shndx) | ||
1134 | return LIBBPF_MAP_DATA; | ||
1135 | else if (shndx == obj->efile.bss_shndx) | ||
1136 | return LIBBPF_MAP_BSS; | ||
1137 | else if (shndx == obj->efile.rodata_shndx) | ||
1138 | return LIBBPF_MAP_RODATA; | ||
1139 | else | ||
1140 | return LIBBPF_MAP_UNSPEC; | ||
1141 | } | ||
1142 | |||
977 | static int | 1143 | static int |
978 | bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr, | 1144 | bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr, |
979 | Elf_Data *data, struct bpf_object *obj) | 1145 | Elf_Data *data, struct bpf_object *obj) |
980 | { | 1146 | { |
981 | Elf_Data *symbols = obj->efile.symbols; | 1147 | Elf_Data *symbols = obj->efile.symbols; |
982 | int text_shndx = obj->efile.text_shndx; | ||
983 | int maps_shndx = obj->efile.maps_shndx; | ||
984 | struct bpf_map *maps = obj->maps; | 1148 | struct bpf_map *maps = obj->maps; |
985 | size_t nr_maps = obj->nr_maps; | 1149 | size_t nr_maps = obj->nr_maps; |
986 | int i, nrels; | 1150 | int i, nrels; |
@@ -1000,7 +1164,10 @@ bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr, | |||
1000 | GElf_Sym sym; | 1164 | GElf_Sym sym; |
1001 | GElf_Rel rel; | 1165 | GElf_Rel rel; |
1002 | unsigned int insn_idx; | 1166 | unsigned int insn_idx; |
1167 | unsigned int shdr_idx; | ||
1003 | struct bpf_insn *insns = prog->insns; | 1168 | struct bpf_insn *insns = prog->insns; |
1169 | enum libbpf_map_type type; | ||
1170 | const char *name; | ||
1004 | size_t map_idx; | 1171 | size_t map_idx; |
1005 | 1172 | ||
1006 | if (!gelf_getrel(data, i, &rel)) { | 1173 | if (!gelf_getrel(data, i, &rel)) { |
@@ -1015,13 +1182,18 @@ bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr, | |||
1015 | GELF_R_SYM(rel.r_info)); | 1182 | GELF_R_SYM(rel.r_info)); |
1016 | return -LIBBPF_ERRNO__FORMAT; | 1183 | return -LIBBPF_ERRNO__FORMAT; |
1017 | } | 1184 | } |
1018 | pr_debug("relo for %lld value %lld name %d\n", | 1185 | |
1186 | name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, | ||
1187 | sym.st_name) ? : "<?>"; | ||
1188 | |||
1189 | pr_debug("relo for %lld value %lld name %d (\'%s\')\n", | ||
1019 | (long long) (rel.r_info >> 32), | 1190 | (long long) (rel.r_info >> 32), |
1020 | (long long) sym.st_value, sym.st_name); | 1191 | (long long) sym.st_value, sym.st_name, name); |
1021 | 1192 | ||
1022 | if (sym.st_shndx != maps_shndx && sym.st_shndx != text_shndx) { | 1193 | shdr_idx = sym.st_shndx; |
1023 | pr_warning("Program '%s' contains non-map related relo data pointing to section %u\n", | 1194 | if (!bpf_object__relo_in_known_section(obj, shdr_idx)) { |
1024 | prog->section_name, sym.st_shndx); | 1195 | pr_warning("Program '%s' contains unrecognized relo data pointing to section %u\n", |
1196 | prog->section_name, shdr_idx); | ||
1025 | return -LIBBPF_ERRNO__RELOC; | 1197 | return -LIBBPF_ERRNO__RELOC; |
1026 | } | 1198 | } |
1027 | 1199 | ||
@@ -1046,10 +1218,22 @@ bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr, | |||
1046 | return -LIBBPF_ERRNO__RELOC; | 1218 | return -LIBBPF_ERRNO__RELOC; |
1047 | } | 1219 | } |
1048 | 1220 | ||
1049 | if (sym.st_shndx == maps_shndx) { | 1221 | if (bpf_object__shndx_is_maps(obj, shdr_idx) || |
1050 | /* TODO: 'maps' is sorted. We can use bsearch to make it faster. */ | 1222 | bpf_object__shndx_is_data(obj, shdr_idx)) { |
1223 | type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx); | ||
1224 | if (type != LIBBPF_MAP_UNSPEC && | ||
1225 | GELF_ST_BIND(sym.st_info) == STB_GLOBAL) { | ||
1226 | pr_warning("bpf: relocation: not yet supported relo for non-static global \'%s\' variable found in insns[%d].code 0x%x\n", | ||
1227 | name, insn_idx, insns[insn_idx].code); | ||
1228 | return -LIBBPF_ERRNO__RELOC; | ||
1229 | } | ||
1230 | |||
1051 | for (map_idx = 0; map_idx < nr_maps; map_idx++) { | 1231 | for (map_idx = 0; map_idx < nr_maps; map_idx++) { |
1052 | if (maps[map_idx].offset == sym.st_value) { | 1232 | if (maps[map_idx].libbpf_type != type) |
1233 | continue; | ||
1234 | if (type != LIBBPF_MAP_UNSPEC || | ||
1235 | (type == LIBBPF_MAP_UNSPEC && | ||
1236 | maps[map_idx].offset == sym.st_value)) { | ||
1053 | pr_debug("relocation: find map %zd (%s) for insn %u\n", | 1237 | pr_debug("relocation: find map %zd (%s) for insn %u\n", |
1054 | map_idx, maps[map_idx].name, insn_idx); | 1238 | map_idx, maps[map_idx].name, insn_idx); |
1055 | break; | 1239 | break; |
@@ -1062,7 +1246,8 @@ bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr, | |||
1062 | return -LIBBPF_ERRNO__RELOC; | 1246 | return -LIBBPF_ERRNO__RELOC; |
1063 | } | 1247 | } |
1064 | 1248 | ||
1065 | prog->reloc_desc[i].type = RELO_LD64; | 1249 | prog->reloc_desc[i].type = type != LIBBPF_MAP_UNSPEC ? |
1250 | RELO_DATA : RELO_LD64; | ||
1066 | prog->reloc_desc[i].insn_idx = insn_idx; | 1251 | prog->reloc_desc[i].insn_idx = insn_idx; |
1067 | prog->reloc_desc[i].map_idx = map_idx; | 1252 | prog->reloc_desc[i].map_idx = map_idx; |
1068 | } | 1253 | } |
@@ -1073,18 +1258,27 @@ bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr, | |||
1073 | static int bpf_map_find_btf_info(struct bpf_map *map, const struct btf *btf) | 1258 | static int bpf_map_find_btf_info(struct bpf_map *map, const struct btf *btf) |
1074 | { | 1259 | { |
1075 | struct bpf_map_def *def = &map->def; | 1260 | struct bpf_map_def *def = &map->def; |
1076 | __u32 key_type_id, value_type_id; | 1261 | __u32 key_type_id = 0, value_type_id = 0; |
1077 | int ret; | 1262 | int ret; |
1078 | 1263 | ||
1079 | ret = btf__get_map_kv_tids(btf, map->name, def->key_size, | 1264 | if (!bpf_map__is_internal(map)) { |
1080 | def->value_size, &key_type_id, | 1265 | ret = btf__get_map_kv_tids(btf, map->name, def->key_size, |
1081 | &value_type_id); | 1266 | def->value_size, &key_type_id, |
1082 | if (ret) | 1267 | &value_type_id); |
1268 | } else { | ||
1269 | /* | ||
1270 | * LLVM annotates global data differently in BTF, that is, | ||
1271 | * only as '.data', '.bss' or '.rodata'. | ||
1272 | */ | ||
1273 | ret = btf__find_by_name(btf, | ||
1274 | libbpf_type_to_btf_name[map->libbpf_type]); | ||
1275 | } | ||
1276 | if (ret < 0) | ||
1083 | return ret; | 1277 | return ret; |
1084 | 1278 | ||
1085 | map->btf_key_type_id = key_type_id; | 1279 | map->btf_key_type_id = key_type_id; |
1086 | map->btf_value_type_id = value_type_id; | 1280 | map->btf_value_type_id = bpf_map__is_internal(map) ? |
1087 | 1281 | ret : value_type_id; | |
1088 | return 0; | 1282 | return 0; |
1089 | } | 1283 | } |
1090 | 1284 | ||
@@ -1196,6 +1390,34 @@ bpf_object__probe_caps(struct bpf_object *obj) | |||
1196 | } | 1390 | } |
1197 | 1391 | ||
1198 | static int | 1392 | static int |
1393 | bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map) | ||
1394 | { | ||
1395 | char *cp, errmsg[STRERR_BUFSIZE]; | ||
1396 | int err, zero = 0; | ||
1397 | __u8 *data; | ||
1398 | |||
1399 | /* Nothing to do here since kernel already zero-initializes .bss map. */ | ||
1400 | if (map->libbpf_type == LIBBPF_MAP_BSS) | ||
1401 | return 0; | ||
1402 | |||
1403 | data = map->libbpf_type == LIBBPF_MAP_DATA ? | ||
1404 | obj->sections.data : obj->sections.rodata; | ||
1405 | |||
1406 | err = bpf_map_update_elem(map->fd, &zero, data, 0); | ||
1407 | /* Freeze .rodata map as read-only from syscall side. */ | ||
1408 | if (!err && map->libbpf_type == LIBBPF_MAP_RODATA) { | ||
1409 | err = bpf_map_freeze(map->fd); | ||
1410 | if (err) { | ||
1411 | cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); | ||
1412 | pr_warning("Error freezing map(%s) as read-only: %s\n", | ||
1413 | map->name, cp); | ||
1414 | err = 0; | ||
1415 | } | ||
1416 | } | ||
1417 | return err; | ||
1418 | } | ||
1419 | |||
1420 | static int | ||
1199 | bpf_object__create_maps(struct bpf_object *obj) | 1421 | bpf_object__create_maps(struct bpf_object *obj) |
1200 | { | 1422 | { |
1201 | struct bpf_create_map_attr create_attr = {}; | 1423 | struct bpf_create_map_attr create_attr = {}; |
@@ -1252,6 +1474,7 @@ bpf_object__create_maps(struct bpf_object *obj) | |||
1252 | size_t j; | 1474 | size_t j; |
1253 | 1475 | ||
1254 | err = *pfd; | 1476 | err = *pfd; |
1477 | err_out: | ||
1255 | cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); | 1478 | cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); |
1256 | pr_warning("failed to create map (name: '%s'): %s\n", | 1479 | pr_warning("failed to create map (name: '%s'): %s\n", |
1257 | map->name, cp); | 1480 | map->name, cp); |
@@ -1259,6 +1482,15 @@ bpf_object__create_maps(struct bpf_object *obj) | |||
1259 | zclose(obj->maps[j].fd); | 1482 | zclose(obj->maps[j].fd); |
1260 | return err; | 1483 | return err; |
1261 | } | 1484 | } |
1485 | |||
1486 | if (bpf_map__is_internal(map)) { | ||
1487 | err = bpf_object__populate_internal_map(obj, map); | ||
1488 | if (err < 0) { | ||
1489 | zclose(*pfd); | ||
1490 | goto err_out; | ||
1491 | } | ||
1492 | } | ||
1493 | |||
1262 | pr_debug("create map %s: fd=%d\n", map->name, *pfd); | 1494 | pr_debug("create map %s: fd=%d\n", map->name, *pfd); |
1263 | } | 1495 | } |
1264 | 1496 | ||
@@ -1413,19 +1645,27 @@ bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj) | |||
1413 | return 0; | 1645 | return 0; |
1414 | 1646 | ||
1415 | for (i = 0; i < prog->nr_reloc; i++) { | 1647 | for (i = 0; i < prog->nr_reloc; i++) { |
1416 | if (prog->reloc_desc[i].type == RELO_LD64) { | 1648 | if (prog->reloc_desc[i].type == RELO_LD64 || |
1649 | prog->reloc_desc[i].type == RELO_DATA) { | ||
1650 | bool relo_data = prog->reloc_desc[i].type == RELO_DATA; | ||
1417 | struct bpf_insn *insns = prog->insns; | 1651 | struct bpf_insn *insns = prog->insns; |
1418 | int insn_idx, map_idx; | 1652 | int insn_idx, map_idx; |
1419 | 1653 | ||
1420 | insn_idx = prog->reloc_desc[i].insn_idx; | 1654 | insn_idx = prog->reloc_desc[i].insn_idx; |
1421 | map_idx = prog->reloc_desc[i].map_idx; | 1655 | map_idx = prog->reloc_desc[i].map_idx; |
1422 | 1656 | ||
1423 | if (insn_idx >= (int)prog->insns_cnt) { | 1657 | if (insn_idx + 1 >= (int)prog->insns_cnt) { |
1424 | pr_warning("relocation out of range: '%s'\n", | 1658 | pr_warning("relocation out of range: '%s'\n", |
1425 | prog->section_name); | 1659 | prog->section_name); |
1426 | return -LIBBPF_ERRNO__RELOC; | 1660 | return -LIBBPF_ERRNO__RELOC; |
1427 | } | 1661 | } |
1428 | insns[insn_idx].src_reg = BPF_PSEUDO_MAP_FD; | 1662 | |
1663 | if (!relo_data) { | ||
1664 | insns[insn_idx].src_reg = BPF_PSEUDO_MAP_FD; | ||
1665 | } else { | ||
1666 | insns[insn_idx].src_reg = BPF_PSEUDO_MAP_VALUE; | ||
1667 | insns[insn_idx + 1].imm = insns[insn_idx].imm; | ||
1668 | } | ||
1429 | insns[insn_idx].imm = obj->maps[map_idx].fd; | 1669 | insns[insn_idx].imm = obj->maps[map_idx].fd; |
1430 | } else if (prog->reloc_desc[i].type == RELO_CALL) { | 1670 | } else if (prog->reloc_desc[i].type == RELO_CALL) { |
1431 | err = bpf_program__reloc_text(prog, obj, | 1671 | err = bpf_program__reloc_text(prog, obj, |
@@ -2321,6 +2561,9 @@ void bpf_object__close(struct bpf_object *obj) | |||
2321 | obj->maps[i].priv = NULL; | 2561 | obj->maps[i].priv = NULL; |
2322 | obj->maps[i].clear_priv = NULL; | 2562 | obj->maps[i].clear_priv = NULL; |
2323 | } | 2563 | } |
2564 | |||
2565 | zfree(&obj->sections.rodata); | ||
2566 | zfree(&obj->sections.data); | ||
2324 | zfree(&obj->maps); | 2567 | zfree(&obj->maps); |
2325 | obj->nr_maps = 0; | 2568 | obj->nr_maps = 0; |
2326 | 2569 | ||
@@ -2798,6 +3041,11 @@ bool bpf_map__is_offload_neutral(struct bpf_map *map) | |||
2798 | return map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY; | 3041 | return map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY; |
2799 | } | 3042 | } |
2800 | 3043 | ||
3044 | bool bpf_map__is_internal(struct bpf_map *map) | ||
3045 | { | ||
3046 | return map->libbpf_type != LIBBPF_MAP_UNSPEC; | ||
3047 | } | ||
3048 | |||
2801 | void bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex) | 3049 | void bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex) |
2802 | { | 3050 | { |
2803 | map->map_ifindex = ifindex; | 3051 | map->map_ifindex = ifindex; |