4 files changed, 70 insertions, 24 deletions

diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 7e61c395fddf..52fb077d3c45 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -142,7 +142,7 @@ struct bpf_verifier_state_list {
 struct bpf_insn_aux_data {
         union {
                 enum bpf_reg_type ptr_type;     /* pointer type for load/store insns */
-                struct bpf_map *map_ptr;        /* pointer for call insn into lookup_elem */
+                unsigned long map_state;        /* pointer/poison value for maps */
                 s32 call_imm;                   /* saved imm field of call insn */
         };
         int ctx_field_size; /* the ctx field size for load insn, maybe 0 */
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 5dd1dcb902bf..dcebf3f7365c 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -156,7 +156,29 @@ struct bpf_verifier_stack_elem {
 #define BPF_COMPLEXITY_LIMIT_INSNS      131072
 #define BPF_COMPLEXITY_LIMIT_STACK      1024
 
-#define BPF_MAP_PTR_POISON ((void *)0xeB9F + POISON_POINTER_DELTA)
+#define BPF_MAP_PTR_UNPRIV      1UL
+#define BPF_MAP_PTR_POISON      ((void *)((0xeB9FUL << 1) +     \
+                                          POISON_POINTER_DELTA))
+#define BPF_MAP_PTR(X)          ((struct bpf_map *)((X) & ~BPF_MAP_PTR_UNPRIV))
+
+static bool bpf_map_ptr_poisoned(const struct bpf_insn_aux_data *aux)
+{
+        return BPF_MAP_PTR(aux->map_state) == BPF_MAP_PTR_POISON;
+}
+
+static bool bpf_map_ptr_unpriv(const struct bpf_insn_aux_data *aux)
+{
+        return aux->map_state & BPF_MAP_PTR_UNPRIV;
+}
+
+static void bpf_map_ptr_store(struct bpf_insn_aux_data *aux,
+                              const struct bpf_map *map, bool unpriv)
+{
+        BUILD_BUG_ON((unsigned long)BPF_MAP_PTR_POISON & BPF_MAP_PTR_UNPRIV);
+        unpriv |= bpf_map_ptr_unpriv(aux);
+        aux->map_state = (unsigned long)map |
+                         (unpriv ? BPF_MAP_PTR_UNPRIV : 0UL);
+}
 
 struct bpf_call_arg_meta {
         struct bpf_map *map_ptr;
@@ -2333,6 +2355,29 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx)
         return 0;
 }
 
+static int
+record_func_map(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
+                int func_id, int insn_idx)
+{
+        struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx];
+
+        if (func_id != BPF_FUNC_tail_call &&
+            func_id != BPF_FUNC_map_lookup_elem)
+                return 0;
+        if (meta->map_ptr == NULL) {
+                verbose(env, "kernel subsystem misconfigured verifier\n");
+                return -EINVAL;
+        }
+
+        if (!BPF_MAP_PTR(aux->map_state))
+                bpf_map_ptr_store(aux, meta->map_ptr,
+                                  meta->map_ptr->unpriv_array);
+        else if (BPF_MAP_PTR(aux->map_state) != meta->map_ptr)
+                bpf_map_ptr_store(aux, BPF_MAP_PTR_POISON,
+                                  meta->map_ptr->unpriv_array);
+        return 0;
+}
+
 static int check_helper_call(struct bpf_verifier_env *env, int func_id, int insn_idx)
 {
         const struct bpf_func_proto *fn = NULL;
@@ -2387,13 +2432,6 @@ static int check_helper_call(struct bpf_verifier_env *env, int func_id, int insn
         err = check_func_arg(env, BPF_REG_2, fn->arg2_type, &meta);
         if (err)
                 return err;
-        if (func_id == BPF_FUNC_tail_call) {
-                if (meta.map_ptr == NULL) {
-                        verbose(env, "verifier bug\n");
-                        return -EINVAL;
-                }
-                env->insn_aux_data[insn_idx].map_ptr = meta.map_ptr;
-        }
         err = check_func_arg(env, BPF_REG_3, fn->arg3_type, &meta);
         if (err)
                 return err;
@@ -2404,6 +2442,10 @@ static int check_helper_call(struct bpf_verifier_env *env, int func_id, int insn
         if (err)
                 return err;
 
+        err = record_func_map(env, &meta, func_id, insn_idx);
+        if (err)
+                return err;
+
         /* Mark slots with STACK_MISC in case of raw mode, stack offset
          * is inferred from register state.
          */
@@ -2428,8 +2470,6 @@ static int check_helper_call(struct bpf_verifier_env *env, int func_id, int insn
         } else if (fn->ret_type == RET_VOID) {
                 regs[BPF_REG_0].type = NOT_INIT;
         } else if (fn->ret_type == RET_PTR_TO_MAP_VALUE_OR_NULL) {
-                struct bpf_insn_aux_data *insn_aux;
-
                 regs[BPF_REG_0].type = PTR_TO_MAP_VALUE_OR_NULL;
                 /* There is no offset yet applied, variable or fixed */
                 mark_reg_known_zero(env, regs, BPF_REG_0);
@@ -2445,11 +2485,6 @@ static int check_helper_call(struct bpf_verifier_env *env, int func_id, int insn
                 }
                 regs[BPF_REG_0].map_ptr = meta.map_ptr;
                 regs[BPF_REG_0].id = ++env->id_gen;
-                insn_aux = &env->insn_aux_data[insn_idx];
-                if (!insn_aux->map_ptr)
-                        insn_aux->map_ptr = meta.map_ptr;
-                else if (insn_aux->map_ptr != meta.map_ptr)
-                        insn_aux->map_ptr = BPF_MAP_PTR_POISON;
         } else {
                 verbose(env, "unknown return type %d of func %s#%d\n",
                         fn->ret_type, func_id_name(func_id), func_id);
@@ -5417,6 +5452,7 @@ static int fixup_bpf_calls(struct bpf_verifier_env *env)
         struct bpf_insn *insn = prog->insnsi;
         const struct bpf_func_proto *fn;
         const int insn_cnt = prog->len;
+        struct bpf_insn_aux_data *aux;
         struct bpf_insn insn_buf[16];
         struct bpf_prog *new_prog;
         struct bpf_map *map_ptr;
@@ -5491,19 +5527,22 @@ static int fixup_bpf_calls(struct bpf_verifier_env *env)
                         insn->imm = 0;
                         insn->code = BPF_JMP | BPF_TAIL_CALL;
 
+                        aux = &env->insn_aux_data[i + delta];
+                        if (!bpf_map_ptr_unpriv(aux))
+                                continue;
+
                         /* instead of changing every JIT dealing with tail_call
                          * emit two extra insns:
                          * if (index >= max_entries) goto out;
                          * index &= array->index_mask;
                          * to avoid out-of-bounds cpu speculation
                          */
-                        map_ptr = env->insn_aux_data[i + delta].map_ptr;
-                        if (map_ptr == BPF_MAP_PTR_POISON) {
+                        if (bpf_map_ptr_poisoned(aux)) {
                                 verbose(env, "tail_call abusing map_ptr\n");
                                 return -EINVAL;
                         }
-                        if (!map_ptr->unpriv_array)
-                                continue;
+
+                        map_ptr = BPF_MAP_PTR(aux->map_state);
                         insn_buf[0] = BPF_JMP_IMM(BPF_JGE, BPF_REG_3,
                                                   map_ptr->max_entries, 2);
                         insn_buf[1] = BPF_ALU32_IMM(BPF_AND, BPF_REG_3,
@@ -5527,9 +5566,12 @@ static int fixup_bpf_calls(struct bpf_verifier_env *env)
                  */
                 if (prog->jit_requested && BITS_PER_LONG == 64 &&
                     insn->imm == BPF_FUNC_map_lookup_elem) {
-                        map_ptr = env->insn_aux_data[i + delta].map_ptr;
-                        if (map_ptr == BPF_MAP_PTR_POISON ||
-                            !map_ptr->ops->map_gen_lookup)
+                        aux = &env->insn_aux_data[i + delta];
+                        if (bpf_map_ptr_poisoned(aux))
+                                goto patch_call_imm;
+
+                        map_ptr = BPF_MAP_PTR(aux->map_state);
+                        if (!map_ptr->ops->map_gen_lookup)
                                 goto patch_call_imm;
 
                         cnt = map_ptr->ops->map_gen_lookup(map_ptr, insn_buf);
diff --git a/tools/testing/selftests/bpf/config b/tools/testing/selftests/bpf/config
index 983dd25d49f4..1eefe211a4a8 100644
--- a/tools/testing/selftests/bpf/config
+++ b/tools/testing/selftests/bpf/config
@@ -5,3 +5,5 @@ CONFIG_BPF_EVENTS=y
 CONFIG_TEST_BPF=m
 CONFIG_CGROUP_BPF=y
 CONFIG_NETDEVSIM=m
+CONFIG_NET_CLS_ACT=y
+CONFIG_NET_SCH_INGRESS=y
diff --git a/tools/testing/selftests/net/reuseport_bpf_numa.c b/tools/testing/selftests/net/reuseport_bpf_numa.c
index 365c32e84189..c9f478b40996 100644
--- a/tools/testing/selftests/net/reuseport_bpf_numa.c
+++ b/tools/testing/selftests/net/reuseport_bpf_numa.c
@@ -23,6 +23,8 @@
 #include <unistd.h>
 #include <numa.h>
 
+#include "../kselftest.h"
+
 static const int PORT = 8888;
 
 static void build_rcv_group(int *rcv_fd, size_t len, int family, int proto)
@@ -229,7 +231,7 @@ int main(void)
         int *rcv_fd, nodes;
 
         if (numa_available() < 0)
-                error(1, errno, "no numa api support");
+                ksft_exit_skip("no numa api support\n");
 
         nodes = numa_max_node() + 1;