aboutsummaryrefslogtreecommitdiffstats
path: root/tools/lib/bpf/libbpf.c
diff options
context:
space:
mode:
Diffstat (limited to 'tools/lib/bpf/libbpf.c')
-rw-r--r--tools/lib/bpf/libbpf.c342
1 files changed, 295 insertions, 47 deletions
diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c
index 6dba0f01673b..f7b245fbb960 100644
--- a/tools/lib/bpf/libbpf.c
+++ b/tools/lib/bpf/libbpf.c
@@ -7,6 +7,7 @@
7 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> 7 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>
8 * Copyright (C) 2015 Huawei Inc. 8 * Copyright (C) 2015 Huawei Inc.
9 * Copyright (C) 2017 Nicira, Inc. 9 * Copyright (C) 2017 Nicira, Inc.
10 * Copyright (C) 2019 Isovalent, Inc.
10 */ 11 */
11 12
12#ifndef _GNU_SOURCE 13#ifndef _GNU_SOURCE
@@ -149,6 +150,7 @@ struct bpf_program {
149 enum { 150 enum {
150 RELO_LD64, 151 RELO_LD64,
151 RELO_CALL, 152 RELO_CALL,
153 RELO_DATA,
152 } type; 154 } type;
153 int insn_idx; 155 int insn_idx;
154 union { 156 union {
@@ -182,6 +184,19 @@ struct bpf_program {
182 __u32 line_info_cnt; 184 __u32 line_info_cnt;
183}; 185};
184 186
187enum libbpf_map_type {
188 LIBBPF_MAP_UNSPEC,
189 LIBBPF_MAP_DATA,
190 LIBBPF_MAP_BSS,
191 LIBBPF_MAP_RODATA,
192};
193
194static const char * const libbpf_type_to_btf_name[] = {
195 [LIBBPF_MAP_DATA] = ".data",
196 [LIBBPF_MAP_BSS] = ".bss",
197 [LIBBPF_MAP_RODATA] = ".rodata",
198};
199
185struct bpf_map { 200struct bpf_map {
186 int fd; 201 int fd;
187 char *name; 202 char *name;
@@ -193,11 +208,18 @@ struct bpf_map {
193 __u32 btf_value_type_id; 208 __u32 btf_value_type_id;
194 void *priv; 209 void *priv;
195 bpf_map_clear_priv_t clear_priv; 210 bpf_map_clear_priv_t clear_priv;
211 enum libbpf_map_type libbpf_type;
212};
213
214struct bpf_secdata {
215 void *rodata;
216 void *data;
196}; 217};
197 218
198static LIST_HEAD(bpf_objects_list); 219static LIST_HEAD(bpf_objects_list);
199 220
200struct bpf_object { 221struct bpf_object {
222 char name[BPF_OBJ_NAME_LEN];
201 char license[64]; 223 char license[64];
202 __u32 kern_version; 224 __u32 kern_version;
203 225
@@ -205,6 +227,7 @@ struct bpf_object {
205 size_t nr_programs; 227 size_t nr_programs;
206 struct bpf_map *maps; 228 struct bpf_map *maps;
207 size_t nr_maps; 229 size_t nr_maps;
230 struct bpf_secdata sections;
208 231
209 bool loaded; 232 bool loaded;
210 bool has_pseudo_calls; 233 bool has_pseudo_calls;
@@ -220,6 +243,9 @@ struct bpf_object {
220 Elf *elf; 243 Elf *elf;
221 GElf_Ehdr ehdr; 244 GElf_Ehdr ehdr;
222 Elf_Data *symbols; 245 Elf_Data *symbols;
246 Elf_Data *data;
247 Elf_Data *rodata;
248 Elf_Data *bss;
223 size_t strtabidx; 249 size_t strtabidx;
224 struct { 250 struct {
225 GElf_Shdr shdr; 251 GElf_Shdr shdr;
@@ -228,6 +254,9 @@ struct bpf_object {
228 int nr_reloc; 254 int nr_reloc;
229 int maps_shndx; 255 int maps_shndx;
230 int text_shndx; 256 int text_shndx;
257 int data_shndx;
258 int rodata_shndx;
259 int bss_shndx;
231 } efile; 260 } efile;
232 /* 261 /*
233 * All loaded bpf_object is linked in a list, which is 262 * All loaded bpf_object is linked in a list, which is
@@ -449,6 +478,7 @@ static struct bpf_object *bpf_object__new(const char *path,
449 size_t obj_buf_sz) 478 size_t obj_buf_sz)
450{ 479{
451 struct bpf_object *obj; 480 struct bpf_object *obj;
481 char *end;
452 482
453 obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1); 483 obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1);
454 if (!obj) { 484 if (!obj) {
@@ -457,8 +487,14 @@ static struct bpf_object *bpf_object__new(const char *path,
457 } 487 }
458 488
459 strcpy(obj->path, path); 489 strcpy(obj->path, path);
460 obj->efile.fd = -1; 490 /* Using basename() GNU version which doesn't modify arg. */
491 strncpy(obj->name, basename((void *)path),
492 sizeof(obj->name) - 1);
493 end = strchr(obj->name, '.');
494 if (end)
495 *end = 0;
461 496
497 obj->efile.fd = -1;
462 /* 498 /*
463 * Caller of this function should also calls 499 * Caller of this function should also calls
464 * bpf_object__elf_finish() after data collection to return 500 * bpf_object__elf_finish() after data collection to return
@@ -468,6 +504,9 @@ static struct bpf_object *bpf_object__new(const char *path,
468 obj->efile.obj_buf = obj_buf; 504 obj->efile.obj_buf = obj_buf;
469 obj->efile.obj_buf_sz = obj_buf_sz; 505 obj->efile.obj_buf_sz = obj_buf_sz;
470 obj->efile.maps_shndx = -1; 506 obj->efile.maps_shndx = -1;
507 obj->efile.data_shndx = -1;
508 obj->efile.rodata_shndx = -1;
509 obj->efile.bss_shndx = -1;
471 510
472 obj->loaded = false; 511 obj->loaded = false;
473 512
@@ -486,6 +525,9 @@ static void bpf_object__elf_finish(struct bpf_object *obj)
486 obj->efile.elf = NULL; 525 obj->efile.elf = NULL;
487 } 526 }
488 obj->efile.symbols = NULL; 527 obj->efile.symbols = NULL;
528 obj->efile.data = NULL;
529 obj->efile.rodata = NULL;
530 obj->efile.bss = NULL;
489 531
490 zfree(&obj->efile.reloc); 532 zfree(&obj->efile.reloc);
491 obj->efile.nr_reloc = 0; 533 obj->efile.nr_reloc = 0;
@@ -627,27 +669,76 @@ static bool bpf_map_type__is_map_in_map(enum bpf_map_type type)
627 return false; 669 return false;
628} 670}
629 671
672static bool bpf_object__has_maps(const struct bpf_object *obj)
673{
674 return obj->efile.maps_shndx >= 0 ||
675 obj->efile.data_shndx >= 0 ||
676 obj->efile.rodata_shndx >= 0 ||
677 obj->efile.bss_shndx >= 0;
678}
679
680static int
681bpf_object__init_internal_map(struct bpf_object *obj, struct bpf_map *map,
682 enum libbpf_map_type type, Elf_Data *data,
683 void **data_buff)
684{
685 struct bpf_map_def *def = &map->def;
686 char map_name[BPF_OBJ_NAME_LEN];
687
688 map->libbpf_type = type;
689 map->offset = ~(typeof(map->offset))0;
690 snprintf(map_name, sizeof(map_name), "%.8s%.7s", obj->name,
691 libbpf_type_to_btf_name[type]);
692 map->name = strdup(map_name);
693 if (!map->name) {
694 pr_warning("failed to alloc map name\n");
695 return -ENOMEM;
696 }
697
698 def->type = BPF_MAP_TYPE_ARRAY;
699 def->key_size = sizeof(int);
700 def->value_size = data->d_size;
701 def->max_entries = 1;
702 def->map_flags = type == LIBBPF_MAP_RODATA ?
703 BPF_F_RDONLY_PROG : 0;
704 if (data_buff) {
705 *data_buff = malloc(data->d_size);
706 if (!*data_buff) {
707 zfree(&map->name);
708 pr_warning("failed to alloc map content buffer\n");
709 return -ENOMEM;
710 }
711 memcpy(*data_buff, data->d_buf, data->d_size);
712 }
713
714 pr_debug("map %ld is \"%s\"\n", map - obj->maps, map->name);
715 return 0;
716}
717
630static int 718static int
631bpf_object__init_maps(struct bpf_object *obj, int flags) 719bpf_object__init_maps(struct bpf_object *obj, int flags)
632{ 720{
721 int i, map_idx, map_def_sz, nr_syms, nr_maps = 0, nr_maps_glob = 0;
633 bool strict = !(flags & MAPS_RELAX_COMPAT); 722 bool strict = !(flags & MAPS_RELAX_COMPAT);
634 int i, map_idx, map_def_sz, nr_maps = 0;
635 Elf_Scn *scn;
636 Elf_Data *data = NULL;
637 Elf_Data *symbols = obj->efile.symbols; 723 Elf_Data *symbols = obj->efile.symbols;
724 Elf_Data *data = NULL;
725 int ret = 0;
638 726
639 if (obj->efile.maps_shndx < 0)
640 return -EINVAL;
641 if (!symbols) 727 if (!symbols)
642 return -EINVAL; 728 return -EINVAL;
729 nr_syms = symbols->d_size / sizeof(GElf_Sym);
643 730
644 scn = elf_getscn(obj->efile.elf, obj->efile.maps_shndx); 731 if (obj->efile.maps_shndx >= 0) {
645 if (scn) 732 Elf_Scn *scn = elf_getscn(obj->efile.elf,
646 data = elf_getdata(scn, NULL); 733 obj->efile.maps_shndx);
647 if (!scn || !data) { 734
648 pr_warning("failed to get Elf_Data from map section %d\n", 735 if (scn)
649 obj->efile.maps_shndx); 736 data = elf_getdata(scn, NULL);
650 return -EINVAL; 737 if (!scn || !data) {
738 pr_warning("failed to get Elf_Data from map section %d\n",
739 obj->efile.maps_shndx);
740 return -EINVAL;
741 }
651 } 742 }
652 743
653 /* 744 /*
@@ -657,7 +748,13 @@ bpf_object__init_maps(struct bpf_object *obj, int flags)
657 * 748 *
658 * TODO: Detect array of map and report error. 749 * TODO: Detect array of map and report error.
659 */ 750 */
660 for (i = 0; i < symbols->d_size / sizeof(GElf_Sym); i++) { 751 if (obj->efile.data_shndx >= 0)
752 nr_maps_glob++;
753 if (obj->efile.rodata_shndx >= 0)
754 nr_maps_glob++;
755 if (obj->efile.bss_shndx >= 0)
756 nr_maps_glob++;
757 for (i = 0; data && i < nr_syms; i++) {
661 GElf_Sym sym; 758 GElf_Sym sym;
662 759
663 if (!gelf_getsym(symbols, i, &sym)) 760 if (!gelf_getsym(symbols, i, &sym))
@@ -670,19 +767,21 @@ bpf_object__init_maps(struct bpf_object *obj, int flags)
670 /* Alloc obj->maps and fill nr_maps. */ 767 /* Alloc obj->maps and fill nr_maps. */
671 pr_debug("maps in %s: %d maps in %zd bytes\n", obj->path, 768 pr_debug("maps in %s: %d maps in %zd bytes\n", obj->path,
672 nr_maps, data->d_size); 769 nr_maps, data->d_size);
673 770 if (!nr_maps && !nr_maps_glob)
674 if (!nr_maps)
675 return 0; 771 return 0;
676 772
677 /* Assume equally sized map definitions */ 773 /* Assume equally sized map definitions */
678 map_def_sz = data->d_size / nr_maps; 774 if (data) {
679 if (!data->d_size || (data->d_size % nr_maps) != 0) { 775 map_def_sz = data->d_size / nr_maps;
680 pr_warning("unable to determine map definition size " 776 if (!data->d_size || (data->d_size % nr_maps) != 0) {
681 "section %s, %d maps in %zd bytes\n", 777 pr_warning("unable to determine map definition size "
682 obj->path, nr_maps, data->d_size); 778 "section %s, %d maps in %zd bytes\n",
683 return -EINVAL; 779 obj->path, nr_maps, data->d_size);
780 return -EINVAL;
781 }
684 } 782 }
685 783
784 nr_maps += nr_maps_glob;
686 obj->maps = calloc(nr_maps, sizeof(obj->maps[0])); 785 obj->maps = calloc(nr_maps, sizeof(obj->maps[0]));
687 if (!obj->maps) { 786 if (!obj->maps) {
688 pr_warning("alloc maps for object failed\n"); 787 pr_warning("alloc maps for object failed\n");
@@ -703,7 +802,7 @@ bpf_object__init_maps(struct bpf_object *obj, int flags)
703 /* 802 /*
704 * Fill obj->maps using data in "maps" section. 803 * Fill obj->maps using data in "maps" section.
705 */ 804 */
706 for (i = 0, map_idx = 0; i < symbols->d_size / sizeof(GElf_Sym); i++) { 805 for (i = 0, map_idx = 0; data && i < nr_syms; i++) {
707 GElf_Sym sym; 806 GElf_Sym sym;
708 const char *map_name; 807 const char *map_name;
709 struct bpf_map_def *def; 808 struct bpf_map_def *def;
@@ -716,6 +815,8 @@ bpf_object__init_maps(struct bpf_object *obj, int flags)
716 map_name = elf_strptr(obj->efile.elf, 815 map_name = elf_strptr(obj->efile.elf,
717 obj->efile.strtabidx, 816 obj->efile.strtabidx,
718 sym.st_name); 817 sym.st_name);
818
819 obj->maps[map_idx].libbpf_type = LIBBPF_MAP_UNSPEC;
719 obj->maps[map_idx].offset = sym.st_value; 820 obj->maps[map_idx].offset = sym.st_value;
720 if (sym.st_value + map_def_sz > data->d_size) { 821 if (sym.st_value + map_def_sz > data->d_size) {
721 pr_warning("corrupted maps section in %s: last map \"%s\" too small\n", 822 pr_warning("corrupted maps section in %s: last map \"%s\" too small\n",
@@ -764,8 +865,27 @@ bpf_object__init_maps(struct bpf_object *obj, int flags)
764 map_idx++; 865 map_idx++;
765 } 866 }
766 867
767 qsort(obj->maps, obj->nr_maps, sizeof(obj->maps[0]), compare_bpf_map); 868 /*
768 return 0; 869 * Populate rest of obj->maps with libbpf internal maps.
870 */
871 if (obj->efile.data_shndx >= 0)
872 ret = bpf_object__init_internal_map(obj, &obj->maps[map_idx++],
873 LIBBPF_MAP_DATA,
874 obj->efile.data,
875 &obj->sections.data);
876 if (!ret && obj->efile.rodata_shndx >= 0)
877 ret = bpf_object__init_internal_map(obj, &obj->maps[map_idx++],
878 LIBBPF_MAP_RODATA,
879 obj->efile.rodata,
880 &obj->sections.rodata);
881 if (!ret && obj->efile.bss_shndx >= 0)
882 ret = bpf_object__init_internal_map(obj, &obj->maps[map_idx++],
883 LIBBPF_MAP_BSS,
884 obj->efile.bss, NULL);
885 if (!ret)
886 qsort(obj->maps, obj->nr_maps, sizeof(obj->maps[0]),
887 compare_bpf_map);
888 return ret;
769} 889}
770 890
771static bool section_have_execinstr(struct bpf_object *obj, int idx) 891static bool section_have_execinstr(struct bpf_object *obj, int idx)
@@ -885,6 +1005,14 @@ static int bpf_object__elf_collect(struct bpf_object *obj, int flags)
885 pr_warning("failed to alloc program %s (%s): %s", 1005 pr_warning("failed to alloc program %s (%s): %s",
886 name, obj->path, cp); 1006 name, obj->path, cp);
887 } 1007 }
1008 } else if (strcmp(name, ".data") == 0) {
1009 obj->efile.data = data;
1010 obj->efile.data_shndx = idx;
1011 } else if (strcmp(name, ".rodata") == 0) {
1012 obj->efile.rodata = data;
1013 obj->efile.rodata_shndx = idx;
1014 } else {
1015 pr_debug("skip section(%d) %s\n", idx, name);
888 } 1016 }
889 } else if (sh.sh_type == SHT_REL) { 1017 } else if (sh.sh_type == SHT_REL) {
890 void *reloc = obj->efile.reloc; 1018 void *reloc = obj->efile.reloc;
@@ -912,6 +1040,9 @@ static int bpf_object__elf_collect(struct bpf_object *obj, int flags)
912 obj->efile.reloc[n].shdr = sh; 1040 obj->efile.reloc[n].shdr = sh;
913 obj->efile.reloc[n].data = data; 1041 obj->efile.reloc[n].data = data;
914 } 1042 }
1043 } else if (sh.sh_type == SHT_NOBITS && strcmp(name, ".bss") == 0) {
1044 obj->efile.bss = data;
1045 obj->efile.bss_shndx = idx;
915 } else { 1046 } else {
916 pr_debug("skip section(%d) %s\n", idx, name); 1047 pr_debug("skip section(%d) %s\n", idx, name);
917 } 1048 }
@@ -938,7 +1069,7 @@ static int bpf_object__elf_collect(struct bpf_object *obj, int flags)
938 } 1069 }
939 } 1070 }
940 } 1071 }
941 if (obj->efile.maps_shndx >= 0) { 1072 if (bpf_object__has_maps(obj)) {
942 err = bpf_object__init_maps(obj, flags); 1073 err = bpf_object__init_maps(obj, flags);
943 if (err) 1074 if (err)
944 goto out; 1075 goto out;
@@ -974,13 +1105,46 @@ bpf_object__find_program_by_title(struct bpf_object *obj, const char *title)
974 return NULL; 1105 return NULL;
975} 1106}
976 1107
1108static bool bpf_object__shndx_is_data(const struct bpf_object *obj,
1109 int shndx)
1110{
1111 return shndx == obj->efile.data_shndx ||
1112 shndx == obj->efile.bss_shndx ||
1113 shndx == obj->efile.rodata_shndx;
1114}
1115
1116static bool bpf_object__shndx_is_maps(const struct bpf_object *obj,
1117 int shndx)
1118{
1119 return shndx == obj->efile.maps_shndx;
1120}
1121
1122static bool bpf_object__relo_in_known_section(const struct bpf_object *obj,
1123 int shndx)
1124{
1125 return shndx == obj->efile.text_shndx ||
1126 bpf_object__shndx_is_maps(obj, shndx) ||
1127 bpf_object__shndx_is_data(obj, shndx);
1128}
1129
1130static enum libbpf_map_type
1131bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx)
1132{
1133 if (shndx == obj->efile.data_shndx)
1134 return LIBBPF_MAP_DATA;
1135 else if (shndx == obj->efile.bss_shndx)
1136 return LIBBPF_MAP_BSS;
1137 else if (shndx == obj->efile.rodata_shndx)
1138 return LIBBPF_MAP_RODATA;
1139 else
1140 return LIBBPF_MAP_UNSPEC;
1141}
1142
977static int 1143static int
978bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr, 1144bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr,
979 Elf_Data *data, struct bpf_object *obj) 1145 Elf_Data *data, struct bpf_object *obj)
980{ 1146{
981 Elf_Data *symbols = obj->efile.symbols; 1147 Elf_Data *symbols = obj->efile.symbols;
982 int text_shndx = obj->efile.text_shndx;
983 int maps_shndx = obj->efile.maps_shndx;
984 struct bpf_map *maps = obj->maps; 1148 struct bpf_map *maps = obj->maps;
985 size_t nr_maps = obj->nr_maps; 1149 size_t nr_maps = obj->nr_maps;
986 int i, nrels; 1150 int i, nrels;
@@ -1000,7 +1164,10 @@ bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr,
1000 GElf_Sym sym; 1164 GElf_Sym sym;
1001 GElf_Rel rel; 1165 GElf_Rel rel;
1002 unsigned int insn_idx; 1166 unsigned int insn_idx;
1167 unsigned int shdr_idx;
1003 struct bpf_insn *insns = prog->insns; 1168 struct bpf_insn *insns = prog->insns;
1169 enum libbpf_map_type type;
1170 const char *name;
1004 size_t map_idx; 1171 size_t map_idx;
1005 1172
1006 if (!gelf_getrel(data, i, &rel)) { 1173 if (!gelf_getrel(data, i, &rel)) {
@@ -1015,13 +1182,18 @@ bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr,
1015 GELF_R_SYM(rel.r_info)); 1182 GELF_R_SYM(rel.r_info));
1016 return -LIBBPF_ERRNO__FORMAT; 1183 return -LIBBPF_ERRNO__FORMAT;
1017 } 1184 }
1018 pr_debug("relo for %lld value %lld name %d\n", 1185
1186 name = elf_strptr(obj->efile.elf, obj->efile.strtabidx,
1187 sym.st_name) ? : "<?>";
1188
1189 pr_debug("relo for %lld value %lld name %d (\'%s\')\n",
1019 (long long) (rel.r_info >> 32), 1190 (long long) (rel.r_info >> 32),
1020 (long long) sym.st_value, sym.st_name); 1191 (long long) sym.st_value, sym.st_name, name);
1021 1192
1022 if (sym.st_shndx != maps_shndx && sym.st_shndx != text_shndx) { 1193 shdr_idx = sym.st_shndx;
1023 pr_warning("Program '%s' contains non-map related relo data pointing to section %u\n", 1194 if (!bpf_object__relo_in_known_section(obj, shdr_idx)) {
1024 prog->section_name, sym.st_shndx); 1195 pr_warning("Program '%s' contains unrecognized relo data pointing to section %u\n",
1196 prog->section_name, shdr_idx);
1025 return -LIBBPF_ERRNO__RELOC; 1197 return -LIBBPF_ERRNO__RELOC;
1026 } 1198 }
1027 1199
@@ -1046,10 +1218,22 @@ bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr,
1046 return -LIBBPF_ERRNO__RELOC; 1218 return -LIBBPF_ERRNO__RELOC;
1047 } 1219 }
1048 1220
1049 if (sym.st_shndx == maps_shndx) { 1221 if (bpf_object__shndx_is_maps(obj, shdr_idx) ||
1050 /* TODO: 'maps' is sorted. We can use bsearch to make it faster. */ 1222 bpf_object__shndx_is_data(obj, shdr_idx)) {
1223 type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx);
1224 if (type != LIBBPF_MAP_UNSPEC &&
1225 GELF_ST_BIND(sym.st_info) == STB_GLOBAL) {
1226 pr_warning("bpf: relocation: not yet supported relo for non-static global \'%s\' variable found in insns[%d].code 0x%x\n",
1227 name, insn_idx, insns[insn_idx].code);
1228 return -LIBBPF_ERRNO__RELOC;
1229 }
1230
1051 for (map_idx = 0; map_idx < nr_maps; map_idx++) { 1231 for (map_idx = 0; map_idx < nr_maps; map_idx++) {
1052 if (maps[map_idx].offset == sym.st_value) { 1232 if (maps[map_idx].libbpf_type != type)
1233 continue;
1234 if (type != LIBBPF_MAP_UNSPEC ||
1235 (type == LIBBPF_MAP_UNSPEC &&
1236 maps[map_idx].offset == sym.st_value)) {
1053 pr_debug("relocation: find map %zd (%s) for insn %u\n", 1237 pr_debug("relocation: find map %zd (%s) for insn %u\n",
1054 map_idx, maps[map_idx].name, insn_idx); 1238 map_idx, maps[map_idx].name, insn_idx);
1055 break; 1239 break;
@@ -1062,7 +1246,8 @@ bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr,
1062 return -LIBBPF_ERRNO__RELOC; 1246 return -LIBBPF_ERRNO__RELOC;
1063 } 1247 }
1064 1248
1065 prog->reloc_desc[i].type = RELO_LD64; 1249 prog->reloc_desc[i].type = type != LIBBPF_MAP_UNSPEC ?
1250 RELO_DATA : RELO_LD64;
1066 prog->reloc_desc[i].insn_idx = insn_idx; 1251 prog->reloc_desc[i].insn_idx = insn_idx;
1067 prog->reloc_desc[i].map_idx = map_idx; 1252 prog->reloc_desc[i].map_idx = map_idx;
1068 } 1253 }
@@ -1073,18 +1258,27 @@ bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr,
1073static int bpf_map_find_btf_info(struct bpf_map *map, const struct btf *btf) 1258static int bpf_map_find_btf_info(struct bpf_map *map, const struct btf *btf)
1074{ 1259{
1075 struct bpf_map_def *def = &map->def; 1260 struct bpf_map_def *def = &map->def;
1076 __u32 key_type_id, value_type_id; 1261 __u32 key_type_id = 0, value_type_id = 0;
1077 int ret; 1262 int ret;
1078 1263
1079 ret = btf__get_map_kv_tids(btf, map->name, def->key_size, 1264 if (!bpf_map__is_internal(map)) {
1080 def->value_size, &key_type_id, 1265 ret = btf__get_map_kv_tids(btf, map->name, def->key_size,
1081 &value_type_id); 1266 def->value_size, &key_type_id,
1082 if (ret) 1267 &value_type_id);
1268 } else {
1269 /*
1270 * LLVM annotates global data differently in BTF, that is,
1271 * only as '.data', '.bss' or '.rodata'.
1272 */
1273 ret = btf__find_by_name(btf,
1274 libbpf_type_to_btf_name[map->libbpf_type]);
1275 }
1276 if (ret < 0)
1083 return ret; 1277 return ret;
1084 1278
1085 map->btf_key_type_id = key_type_id; 1279 map->btf_key_type_id = key_type_id;
1086 map->btf_value_type_id = value_type_id; 1280 map->btf_value_type_id = bpf_map__is_internal(map) ?
1087 1281 ret : value_type_id;
1088 return 0; 1282 return 0;
1089} 1283}
1090 1284
@@ -1196,6 +1390,34 @@ bpf_object__probe_caps(struct bpf_object *obj)
1196} 1390}
1197 1391
1198static int 1392static int
1393bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map)
1394{
1395 char *cp, errmsg[STRERR_BUFSIZE];
1396 int err, zero = 0;
1397 __u8 *data;
1398
1399 /* Nothing to do here since kernel already zero-initializes .bss map. */
1400 if (map->libbpf_type == LIBBPF_MAP_BSS)
1401 return 0;
1402
1403 data = map->libbpf_type == LIBBPF_MAP_DATA ?
1404 obj->sections.data : obj->sections.rodata;
1405
1406 err = bpf_map_update_elem(map->fd, &zero, data, 0);
1407 /* Freeze .rodata map as read-only from syscall side. */
1408 if (!err && map->libbpf_type == LIBBPF_MAP_RODATA) {
1409 err = bpf_map_freeze(map->fd);
1410 if (err) {
1411 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
1412 pr_warning("Error freezing map(%s) as read-only: %s\n",
1413 map->name, cp);
1414 err = 0;
1415 }
1416 }
1417 return err;
1418}
1419
1420static int
1199bpf_object__create_maps(struct bpf_object *obj) 1421bpf_object__create_maps(struct bpf_object *obj)
1200{ 1422{
1201 struct bpf_create_map_attr create_attr = {}; 1423 struct bpf_create_map_attr create_attr = {};
@@ -1252,6 +1474,7 @@ bpf_object__create_maps(struct bpf_object *obj)
1252 size_t j; 1474 size_t j;
1253 1475
1254 err = *pfd; 1476 err = *pfd;
1477err_out:
1255 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 1478 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
1256 pr_warning("failed to create map (name: '%s'): %s\n", 1479 pr_warning("failed to create map (name: '%s'): %s\n",
1257 map->name, cp); 1480 map->name, cp);
@@ -1259,6 +1482,15 @@ bpf_object__create_maps(struct bpf_object *obj)
1259 zclose(obj->maps[j].fd); 1482 zclose(obj->maps[j].fd);
1260 return err; 1483 return err;
1261 } 1484 }
1485
1486 if (bpf_map__is_internal(map)) {
1487 err = bpf_object__populate_internal_map(obj, map);
1488 if (err < 0) {
1489 zclose(*pfd);
1490 goto err_out;
1491 }
1492 }
1493
1262 pr_debug("create map %s: fd=%d\n", map->name, *pfd); 1494 pr_debug("create map %s: fd=%d\n", map->name, *pfd);
1263 } 1495 }
1264 1496
@@ -1413,19 +1645,27 @@ bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj)
1413 return 0; 1645 return 0;
1414 1646
1415 for (i = 0; i < prog->nr_reloc; i++) { 1647 for (i = 0; i < prog->nr_reloc; i++) {
1416 if (prog->reloc_desc[i].type == RELO_LD64) { 1648 if (prog->reloc_desc[i].type == RELO_LD64 ||
1649 prog->reloc_desc[i].type == RELO_DATA) {
1650 bool relo_data = prog->reloc_desc[i].type == RELO_DATA;
1417 struct bpf_insn *insns = prog->insns; 1651 struct bpf_insn *insns = prog->insns;
1418 int insn_idx, map_idx; 1652 int insn_idx, map_idx;
1419 1653
1420 insn_idx = prog->reloc_desc[i].insn_idx; 1654 insn_idx = prog->reloc_desc[i].insn_idx;
1421 map_idx = prog->reloc_desc[i].map_idx; 1655 map_idx = prog->reloc_desc[i].map_idx;
1422 1656
1423 if (insn_idx >= (int)prog->insns_cnt) { 1657 if (insn_idx + 1 >= (int)prog->insns_cnt) {
1424 pr_warning("relocation out of range: '%s'\n", 1658 pr_warning("relocation out of range: '%s'\n",
1425 prog->section_name); 1659 prog->section_name);
1426 return -LIBBPF_ERRNO__RELOC; 1660 return -LIBBPF_ERRNO__RELOC;
1427 } 1661 }
1428 insns[insn_idx].src_reg = BPF_PSEUDO_MAP_FD; 1662
1663 if (!relo_data) {
1664 insns[insn_idx].src_reg = BPF_PSEUDO_MAP_FD;
1665 } else {
1666 insns[insn_idx].src_reg = BPF_PSEUDO_MAP_VALUE;
1667 insns[insn_idx + 1].imm = insns[insn_idx].imm;
1668 }
1429 insns[insn_idx].imm = obj->maps[map_idx].fd; 1669 insns[insn_idx].imm = obj->maps[map_idx].fd;
1430 } else if (prog->reloc_desc[i].type == RELO_CALL) { 1670 } else if (prog->reloc_desc[i].type == RELO_CALL) {
1431 err = bpf_program__reloc_text(prog, obj, 1671 err = bpf_program__reloc_text(prog, obj,
@@ -2321,6 +2561,9 @@ void bpf_object__close(struct bpf_object *obj)
2321 obj->maps[i].priv = NULL; 2561 obj->maps[i].priv = NULL;
2322 obj->maps[i].clear_priv = NULL; 2562 obj->maps[i].clear_priv = NULL;
2323 } 2563 }
2564
2565 zfree(&obj->sections.rodata);
2566 zfree(&obj->sections.data);
2324 zfree(&obj->maps); 2567 zfree(&obj->maps);
2325 obj->nr_maps = 0; 2568 obj->nr_maps = 0;
2326 2569
@@ -2798,6 +3041,11 @@ bool bpf_map__is_offload_neutral(struct bpf_map *map)
2798 return map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY; 3041 return map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY;
2799} 3042}
2800 3043
3044bool bpf_map__is_internal(struct bpf_map *map)
3045{
3046 return map->libbpf_type != LIBBPF_MAP_UNSPEC;
3047}
3048
2801void bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex) 3049void bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex)
2802{ 3050{
2803 map->map_ifindex = ifindex; 3051 map->map_ifindex = ifindex;