Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Daniel Borkmann says:

====================
pull-request: bpf-next 2018-10-21

The following pull-request contains BPF updates for your *net-next* tree.

The main changes are:

1) Implement two new kind of BPF maps, that is, queue and stack
   map along with new peek, push and pop operations, from Mauricio.

2) Add support for MSG_PEEK flag when redirecting into an ingress
   psock sk_msg queue, and add a new helper bpf_msg_push_data() for
   insert data into the message, from John.

3) Allow for BPF programs of type BPF_PROG_TYPE_CGROUP_SKB to use
   direct packet access for __skb_buff, from Song.

4) Use more lightweight barriers for walking perf ring buffer for
   libbpf and perf tool as well. Also, various fixes and improvements
   from verifier side, from Daniel.

5) Add per-symbol visibility for DSO in libbpf and hide by default
   global symbols such as netlink related functions, from Andrey.

6) Two improvements to nfp's BPF offload to check vNIC capabilities
   in case prog is shared with multiple vNICs and to protect against
   mis-initializing atomic counters, from Jakub.

7) Fix for bpftool to use 4 context mode for the nfp disassembler,
   also from Jakub.

8) Fix a return value comparison in test_libbpf.sh and add several
   bpftool improvements in bash completion, documentation of bpf fs
   restrictions and batch mode summary print, from Quentin.

9) Fix a file resource leak in BPF selftest's load_kallsyms()
   helper, from Peng.

10) Fix an unused variable warning in map_lookup_and_delete_elem(),
    from Alexei.

11) Fix bpf_skb_adjust_room() signature in BPF UAPI helper doc,
    from Nicolas.

12) Add missing executables to .gitignore in BPF selftests, from Anders.
====================

Signed-off-by: David S. Miller <davem@davemloft.net>

8 files changed, 500 insertions, 20 deletions

diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile
index ff8262626b8f..4c2fa3ac56f6 100644
--- a/kernel/bpf/Makefile
+++ b/kernel/bpf/Makefile
@@ -3,7 +3,7 @@ obj-y := core.o
 
 obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o
 obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o
-obj-$(CONFIG_BPF_SYSCALL) += local_storage.o
+obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o
 obj-$(CONFIG_BPF_SYSCALL) += disasm.o
 obj-$(CONFIG_BPF_SYSCALL) += btf.o
 ifeq ($(CONFIG_NET),y)
diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c
index 00f6ed2e4f9a..9425c2fb872f 100644
--- a/kernel/bpf/cgroup.c
+++ b/kernel/bpf/cgroup.c
@@ -553,6 +553,7 @@ int __cgroup_bpf_run_filter_skb(struct sock *sk,
 {
         unsigned int offset = skb->data - skb_network_header(skb);
         struct sock *save_sk;
+        void *saved_data_end;
         struct cgroup *cgrp;
         int ret;
 
@@ -566,8 +567,13 @@ int __cgroup_bpf_run_filter_skb(struct sock *sk,
         save_sk = skb->sk;
         skb->sk = sk;
         __skb_push(skb, offset);
+
+        /* compute pointers for the bpf prog */
+        bpf_compute_and_save_data_end(skb, &saved_data_end);
+
         ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], skb,
                                  bpf_prog_run_save_cb);
+        bpf_restore_data_end(skb, saved_data_end);
         __skb_pull(skb, offset);
         skb->sk = save_sk;
         return ret == 1 ? 0 : -EPERM;
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index defcf4df6d91..7c7eeea8cffc 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -1783,6 +1783,9 @@ BPF_CALL_0(bpf_user_rnd_u32)
 const struct bpf_func_proto bpf_map_lookup_elem_proto __weak;
 const struct bpf_func_proto bpf_map_update_elem_proto __weak;
 const struct bpf_func_proto bpf_map_delete_elem_proto __weak;
+const struct bpf_func_proto bpf_map_push_elem_proto __weak;
+const struct bpf_func_proto bpf_map_pop_elem_proto __weak;
+const struct bpf_func_proto bpf_map_peek_elem_proto __weak;
 
 const struct bpf_func_proto bpf_get_prandom_u32_proto __weak;
 const struct bpf_func_proto bpf_get_smp_processor_id_proto __weak;
diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c
index 6502115e8f55..ab0d5e3f9892 100644
--- a/kernel/bpf/helpers.c
+++ b/kernel/bpf/helpers.c
@@ -76,6 +76,49 @@ const struct bpf_func_proto bpf_map_delete_elem_proto = {
         .arg2_type      = ARG_PTR_TO_MAP_KEY,
 };
 
+BPF_CALL_3(bpf_map_push_elem, struct bpf_map *, map, void *, value, u64, flags)
+{
+        return map->ops->map_push_elem(map, value, flags);
+}
+
+const struct bpf_func_proto bpf_map_push_elem_proto = {
+        .func           = bpf_map_push_elem,
+        .gpl_only       = false,
+        .pkt_access     = true,
+        .ret_type       = RET_INTEGER,
+        .arg1_type      = ARG_CONST_MAP_PTR,
+        .arg2_type      = ARG_PTR_TO_MAP_VALUE,
+        .arg3_type      = ARG_ANYTHING,
+};
+
+BPF_CALL_2(bpf_map_pop_elem, struct bpf_map *, map, void *, value)
+{
+        return map->ops->map_pop_elem(map, value);
+}
+
+const struct bpf_func_proto bpf_map_pop_elem_proto = {
+        .func           = bpf_map_pop_elem,
+        .gpl_only       = false,
+        .pkt_access     = true,
+        .ret_type       = RET_INTEGER,
+        .arg1_type      = ARG_CONST_MAP_PTR,
+        .arg2_type      = ARG_PTR_TO_UNINIT_MAP_VALUE,
+};
+
+BPF_CALL_2(bpf_map_peek_elem, struct bpf_map *, map, void *, value)
+{
+        return map->ops->map_peek_elem(map, value);
+}
+
+const struct bpf_func_proto bpf_map_peek_elem_proto = {
+        .func           = bpf_map_pop_elem,
+        .gpl_only       = false,
+        .pkt_access     = true,
+        .ret_type       = RET_INTEGER,
+        .arg1_type      = ARG_CONST_MAP_PTR,
+        .arg2_type      = ARG_PTR_TO_UNINIT_MAP_VALUE,
+};
+
 const struct bpf_func_proto bpf_get_prandom_u32_proto = {
         .func           = bpf_user_rnd_u32,
         .gpl_only       = false,
diff --git a/kernel/bpf/queue_stack_maps.c b/kernel/bpf/queue_stack_maps.c
new file mode 100644
index 000000000000..12a93fb37449
--- /dev/null
+++ b/kernel/bpf/queue_stack_maps.c
@@ -0,0 +1,288 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * queue_stack_maps.c: BPF queue and stack maps
+ *
+ * Copyright (c) 2018 Politecnico di Torino
+ */
+#include <linux/bpf.h>
+#include <linux/list.h>
+#include <linux/slab.h>
+#include "percpu_freelist.h"
+
+#define QUEUE_STACK_CREATE_FLAG_MASK \
+        (BPF_F_NUMA_NODE | BPF_F_RDONLY | BPF_F_WRONLY)
+
+
+struct bpf_queue_stack {
+        struct bpf_map map;
+        raw_spinlock_t lock;
+        u32 head, tail;
+        u32 size; /* max_entries + 1 */
+
+        char elements[0] __aligned(8);
+};
+
+static struct bpf_queue_stack *bpf_queue_stack(struct bpf_map *map)
+{
+        return container_of(map, struct bpf_queue_stack, map);
+}
+
+static bool queue_stack_map_is_empty(struct bpf_queue_stack *qs)
+{
+        return qs->head == qs->tail;
+}
+
+static bool queue_stack_map_is_full(struct bpf_queue_stack *qs)
+{
+        u32 head = qs->head + 1;
+
+        if (unlikely(head >= qs->size))
+                head = 0;
+
+        return head == qs->tail;
+}
+
+/* Called from syscall */
+static int queue_stack_map_alloc_check(union bpf_attr *attr)
+{
+        /* check sanity of attributes */
+        if (attr->max_entries == 0 || attr->key_size != 0 ||
+            attr->map_flags & ~QUEUE_STACK_CREATE_FLAG_MASK)
+                return -EINVAL;
+
+        if (attr->value_size > KMALLOC_MAX_SIZE)
+                /* if value_size is bigger, the user space won't be able to
+                 * access the elements.
+                 */
+                return -E2BIG;
+
+        return 0;
+}
+
+static struct bpf_map *queue_stack_map_alloc(union bpf_attr *attr)
+{
+        int ret, numa_node = bpf_map_attr_numa_node(attr);
+        struct bpf_queue_stack *qs;
+        u32 size, value_size;
+        u64 queue_size, cost;
+
+        size = attr->max_entries + 1;
+        value_size = attr->value_size;
+
+        queue_size = sizeof(*qs) + (u64) value_size * size;
+
+        cost = queue_size;
+        if (cost >= U32_MAX - PAGE_SIZE)
+                return ERR_PTR(-E2BIG);
+
+        cost = round_up(cost, PAGE_SIZE) >> PAGE_SHIFT;
+
+        ret = bpf_map_precharge_memlock(cost);
+        if (ret < 0)
+                return ERR_PTR(ret);
+
+        qs = bpf_map_area_alloc(queue_size, numa_node);
+        if (!qs)
+                return ERR_PTR(-ENOMEM);
+
+        memset(qs, 0, sizeof(*qs));
+
+        bpf_map_init_from_attr(&qs->map, attr);
+
+        qs->map.pages = cost;
+        qs->size = size;
+
+        raw_spin_lock_init(&qs->lock);
+
+        return &qs->map;
+}
+
+/* Called when map->refcnt goes to zero, either from workqueue or from syscall */
+static void queue_stack_map_free(struct bpf_map *map)
+{
+        struct bpf_queue_stack *qs = bpf_queue_stack(map);
+
+        /* at this point bpf_prog->aux->refcnt == 0 and this map->refcnt == 0,
+         * so the programs (can be more than one that used this map) were
+         * disconnected from events. Wait for outstanding critical sections in
+         * these programs to complete
+         */
+        synchronize_rcu();
+
+        bpf_map_area_free(qs);
+}
+
+static int __queue_map_get(struct bpf_map *map, void *value, bool delete)
+{
+        struct bpf_queue_stack *qs = bpf_queue_stack(map);
+        unsigned long flags;
+        int err = 0;
+        void *ptr;
+
+        raw_spin_lock_irqsave(&qs->lock, flags);
+
+        if (queue_stack_map_is_empty(qs)) {
+                err = -ENOENT;
+                goto out;
+        }
+
+        ptr = &qs->elements[qs->tail * qs->map.value_size];
+        memcpy(value, ptr, qs->map.value_size);
+
+        if (delete) {
+                if (unlikely(++qs->tail >= qs->size))
+                        qs->tail = 0;
+        }
+
+out:
+        raw_spin_unlock_irqrestore(&qs->lock, flags);
+        return err;
+}
+
+
+static int __stack_map_get(struct bpf_map *map, void *value, bool delete)
+{
+        struct bpf_queue_stack *qs = bpf_queue_stack(map);
+        unsigned long flags;
+        int err = 0;
+        void *ptr;
+        u32 index;
+
+        raw_spin_lock_irqsave(&qs->lock, flags);
+
+        if (queue_stack_map_is_empty(qs)) {
+                err = -ENOENT;
+                goto out;
+        }
+
+        index = qs->head - 1;
+        if (unlikely(index >= qs->size))
+                index = qs->size - 1;
+
+        ptr = &qs->elements[index * qs->map.value_size];
+        memcpy(value, ptr, qs->map.value_size);
+
+        if (delete)
+                qs->head = index;
+
+out:
+        raw_spin_unlock_irqrestore(&qs->lock, flags);
+        return err;
+}
+
+/* Called from syscall or from eBPF program */
+static int queue_map_peek_elem(struct bpf_map *map, void *value)
+{
+        return __queue_map_get(map, value, false);
+}
+
+/* Called from syscall or from eBPF program */
+static int stack_map_peek_elem(struct bpf_map *map, void *value)
+{
+        return __stack_map_get(map, value, false);
+}
+
+/* Called from syscall or from eBPF program */
+static int queue_map_pop_elem(struct bpf_map *map, void *value)
+{
+        return __queue_map_get(map, value, true);
+}
+
+/* Called from syscall or from eBPF program */
+static int stack_map_pop_elem(struct bpf_map *map, void *value)
+{
+        return __stack_map_get(map, value, true);
+}
+
+/* Called from syscall or from eBPF program */
+static int queue_stack_map_push_elem(struct bpf_map *map, void *value,
+                                     u64 flags)
+{
+        struct bpf_queue_stack *qs = bpf_queue_stack(map);
+        unsigned long irq_flags;
+        int err = 0;
+        void *dst;
+
+        /* BPF_EXIST is used to force making room for a new element in case the
+         * map is full
+         */
+        bool replace = (flags & BPF_EXIST);
+
+        /* Check supported flags for queue and stack maps */
+        if (flags & BPF_NOEXIST || flags > BPF_EXIST)
+                return -EINVAL;
+
+        raw_spin_lock_irqsave(&qs->lock, irq_flags);
+
+        if (queue_stack_map_is_full(qs)) {
+                if (!replace) {
+                        err = -E2BIG;
+                        goto out;
+                }
+                /* advance tail pointer to overwrite oldest element */
+                if (unlikely(++qs->tail >= qs->size))
+                        qs->tail = 0;
+        }
+
+        dst = &qs->elements[qs->head * qs->map.value_size];
+        memcpy(dst, value, qs->map.value_size);
+
+        if (unlikely(++qs->head >= qs->size))
+                qs->head = 0;
+
+out:
+        raw_spin_unlock_irqrestore(&qs->lock, irq_flags);
+        return err;
+}
+
+/* Called from syscall or from eBPF program */
+static void *queue_stack_map_lookup_elem(struct bpf_map *map, void *key)
+{
+        return NULL;
+}
+
+/* Called from syscall or from eBPF program */
+static int queue_stack_map_update_elem(struct bpf_map *map, void *key,
+                                       void *value, u64 flags)
+{
+        return -EINVAL;
+}
+
+/* Called from syscall or from eBPF program */
+static int queue_stack_map_delete_elem(struct bpf_map *map, void *key)
+{
+        return -EINVAL;
+}
+
+/* Called from syscall */
+static int queue_stack_map_get_next_key(struct bpf_map *map, void *key,
+                                        void *next_key)
+{
+        return -EINVAL;
+}
+
+const struct bpf_map_ops queue_map_ops = {
+        .map_alloc_check = queue_stack_map_alloc_check,
+        .map_alloc = queue_stack_map_alloc,
+        .map_free = queue_stack_map_free,
+        .map_lookup_elem = queue_stack_map_lookup_elem,
+        .map_update_elem = queue_stack_map_update_elem,
+        .map_delete_elem = queue_stack_map_delete_elem,
+        .map_push_elem = queue_stack_map_push_elem,
+        .map_pop_elem = queue_map_pop_elem,
+        .map_peek_elem = queue_map_peek_elem,
+        .map_get_next_key = queue_stack_map_get_next_key,
+};
+
+const struct bpf_map_ops stack_map_ops = {
+        .map_alloc_check = queue_stack_map_alloc_check,
+        .map_alloc = queue_stack_map_alloc,
+        .map_free = queue_stack_map_free,
+        .map_lookup_elem = queue_stack_map_lookup_elem,
+        .map_update_elem = queue_stack_map_update_elem,
+        .map_delete_elem = queue_stack_map_delete_elem,
+        .map_push_elem = queue_stack_map_push_elem,
+        .map_pop_elem = stack_map_pop_elem,
+        .map_peek_elem = stack_map_peek_elem,
+        .map_get_next_key = queue_stack_map_get_next_key,
+};
diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c
index b2ade10f7ec3..90daf285de03 100644
--- a/kernel/bpf/stackmap.c
+++ b/kernel/bpf/stackmap.c
@@ -600,7 +600,7 @@ static void stack_map_free(struct bpf_map *map)
         put_callchain_buffers();
 }
 
-const struct bpf_map_ops stack_map_ops = {
+const struct bpf_map_ops stack_trace_map_ops = {
         .map_alloc = stack_map_alloc,
         .map_free = stack_map_free,
         .map_get_next_key = stack_map_get_next_key,
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index f4ecd6ed2252..ccb93277aae2 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -651,6 +651,17 @@ int __weak bpf_stackmap_copy(struct bpf_map *map, void *key, void *value)
         return -ENOTSUPP;
 }
 
+static void *__bpf_copy_key(void __user *ukey, u64 key_size)
+{
+        if (key_size)
+                return memdup_user(ukey, key_size);
+
+        if (ukey)
+                return ERR_PTR(-EINVAL);
+
+        return NULL;
+}
+
 /* last field in 'union bpf_attr' used by this command */
 #define BPF_MAP_LOOKUP_ELEM_LAST_FIELD value
 
@@ -678,7 +689,7 @@ static int map_lookup_elem(union bpf_attr *attr)
                 goto err_put;
         }
 
-        key = memdup_user(ukey, map->key_size);
+        key = __bpf_copy_key(ukey, map->key_size);
         if (IS_ERR(key)) {
                 err = PTR_ERR(key);
                 goto err_put;
@@ -716,6 +727,9 @@ static int map_lookup_elem(union bpf_attr *attr)
                 err = bpf_fd_htab_map_lookup_elem(map, key, value);
         } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) {
                 err = bpf_fd_reuseport_array_lookup_elem(map, key, value);
+        } else if (map->map_type == BPF_MAP_TYPE_QUEUE ||
+                   map->map_type == BPF_MAP_TYPE_STACK) {
+                err = map->ops->map_peek_elem(map, value);
         } else {
                 rcu_read_lock();
                 ptr = map->ops->map_lookup_elem(map, key);
@@ -785,7 +799,7 @@ static int map_update_elem(union bpf_attr *attr)
                 goto err_put;
         }
 
-        key = memdup_user(ukey, map->key_size);
+        key = __bpf_copy_key(ukey, map->key_size);
         if (IS_ERR(key)) {
                 err = PTR_ERR(key);
                 goto err_put;
@@ -846,6 +860,9 @@ static int map_update_elem(union bpf_attr *attr)
                 /* rcu_read_lock() is not needed */
                 err = bpf_fd_reuseport_array_update_elem(map, key, value,
                                                          attr->flags);
+        } else if (map->map_type == BPF_MAP_TYPE_QUEUE ||
+                   map->map_type == BPF_MAP_TYPE_STACK) {
+                err = map->ops->map_push_elem(map, value, attr->flags);
         } else {
                 rcu_read_lock();
                 err = map->ops->map_update_elem(map, key, value, attr->flags);
@@ -888,7 +905,7 @@ static int map_delete_elem(union bpf_attr *attr)
                 goto err_put;
         }
 
-        key = memdup_user(ukey, map->key_size);
+        key = __bpf_copy_key(ukey, map->key_size);
         if (IS_ERR(key)) {
                 err = PTR_ERR(key);
                 goto err_put;
@@ -941,7 +958,7 @@ static int map_get_next_key(union bpf_attr *attr)
         }
 
         if (ukey) {
-                key = memdup_user(ukey, map->key_size);
+                key = __bpf_copy_key(ukey, map->key_size);
                 if (IS_ERR(key)) {
                         err = PTR_ERR(key);
                         goto err_put;
@@ -982,6 +999,69 @@ err_put:
         return err;
 }
 
+#define BPF_MAP_LOOKUP_AND_DELETE_ELEM_LAST_FIELD value
+
+static int map_lookup_and_delete_elem(union bpf_attr *attr)
+{
+        void __user *ukey = u64_to_user_ptr(attr->key);
+        void __user *uvalue = u64_to_user_ptr(attr->value);
+        int ufd = attr->map_fd;
+        struct bpf_map *map;
+        void *key, *value;
+        u32 value_size;
+        struct fd f;
+        int err;
+
+        if (CHECK_ATTR(BPF_MAP_LOOKUP_AND_DELETE_ELEM))
+                return -EINVAL;
+
+        f = fdget(ufd);
+        map = __bpf_map_get(f);
+        if (IS_ERR(map))
+                return PTR_ERR(map);
+
+        if (!(f.file->f_mode & FMODE_CAN_WRITE)) {
+                err = -EPERM;
+                goto err_put;
+        }
+
+        key = __bpf_copy_key(ukey, map->key_size);
+        if (IS_ERR(key)) {
+                err = PTR_ERR(key);
+                goto err_put;
+        }
+
+        value_size = map->value_size;
+
+        err = -ENOMEM;
+        value = kmalloc(value_size, GFP_USER | __GFP_NOWARN);
+        if (!value)
+                goto free_key;
+
+        if (map->map_type == BPF_MAP_TYPE_QUEUE ||
+            map->map_type == BPF_MAP_TYPE_STACK) {
+                err = map->ops->map_pop_elem(map, value);
+        } else {
+                err = -ENOTSUPP;
+        }
+
+        if (err)
+                goto free_value;
+
+        if (copy_to_user(uvalue, value, value_size) != 0)
+                goto free_value;
+
+        err = 0;
+
+free_value:
+        kfree(value);
+free_key:
+        kfree(key);
+err_put:
+        fdput(f);
+        return err;
+}
+
 static const struct bpf_prog_ops * const bpf_prog_types[] = {
 #define BPF_PROG_TYPE(_id, _name) \
         [_id] = & _name ## _prog_ops,
@@ -2455,6 +2535,9 @@ SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, siz
         case BPF_TASK_FD_QUERY:
                 err = bpf_task_fd_query(&attr, uattr);
                 break;
+        case BPF_MAP_LOOKUP_AND_DELETE_ELEM:
+                err = map_lookup_and_delete_elem(&attr);
+                break;
         default:
                 err = -EINVAL;
                 break;
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 3f93a548a642..98fa0be35370 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -1528,14 +1528,19 @@ static bool __is_pointer_value(bool allow_ptr_leaks,
         return reg->type != SCALAR_VALUE;
 }
 
+static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno)
+{
+        return cur_regs(env) + regno;
+}
+
 static bool is_pointer_value(struct bpf_verifier_env *env, int regno)
 {
-        return __is_pointer_value(env->allow_ptr_leaks, cur_regs(env) + regno);
+        return __is_pointer_value(env->allow_ptr_leaks, reg_state(env, regno));
 }
 
 static bool is_ctx_reg(struct bpf_verifier_env *env, int regno)
 {
-        const struct bpf_reg_state *reg = cur_regs(env) + regno;
+        const struct bpf_reg_state *reg = reg_state(env, regno);
 
         return reg->type == PTR_TO_CTX ||
                reg->type == PTR_TO_SOCKET;
@@ -1543,11 +1548,19 @@ static bool is_ctx_reg(struct bpf_verifier_env *env, int regno)
 
 static bool is_pkt_reg(struct bpf_verifier_env *env, int regno)
 {
-        const struct bpf_reg_state *reg = cur_regs(env) + regno;
+        const struct bpf_reg_state *reg = reg_state(env, regno);
 
         return type_is_pkt_pointer(reg->type);
 }
 
+static bool is_flow_key_reg(struct bpf_verifier_env *env, int regno)
+{
+        const struct bpf_reg_state *reg = reg_state(env, regno);
+
+        /* Separate to is_ctx_reg() since we still want to allow BPF_ST here. */
+        return reg->type == PTR_TO_FLOW_KEYS;
+}
+
 static int check_pkt_ptr_alignment(struct bpf_verifier_env *env,
                                    const struct bpf_reg_state *reg,
                                    int off, int size, bool strict)
@@ -1956,9 +1969,11 @@ static int check_xadd(struct bpf_verifier_env *env, int insn_idx, struct bpf_ins
         }
 
         if (is_ctx_reg(env, insn->dst_reg) ||
-            is_pkt_reg(env, insn->dst_reg)) {
+            is_pkt_reg(env, insn->dst_reg) ||
+            is_flow_key_reg(env, insn->dst_reg)) {
                 verbose(env, "BPF_XADD stores into R%d %s is not allowed\n",
-                        insn->dst_reg, reg_type_str[insn->dst_reg]);
+                        insn->dst_reg,
+                        reg_type_str[reg_state(env, insn->dst_reg)->type]);
                 return -EACCES;
         }
 
@@ -1983,7 +1998,7 @@ static int check_stack_boundary(struct bpf_verifier_env *env, int regno,
                                 int access_size, bool zero_size_allowed,
                                 struct bpf_call_arg_meta *meta)
 {
-        struct bpf_reg_state *reg = cur_regs(env) + regno;
+        struct bpf_reg_state *reg = reg_state(env, regno);
         struct bpf_func_state *state = func(env, reg);
         int off, i, slot, spi;
 
@@ -2062,8 +2077,6 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, int regno,
         case PTR_TO_PACKET_META:
                 return check_packet_access(env, regno, reg->off, access_size,
                                            zero_size_allowed);
-        case PTR_TO_FLOW_KEYS:
-                return check_flow_keys_access(env, reg->off, access_size);
         case PTR_TO_MAP_VALUE:
                 return check_map_access(env, regno, reg->off, access_size,
                                         zero_size_allowed);
@@ -2117,7 +2130,8 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 regno,
         }
 
         if (arg_type == ARG_PTR_TO_MAP_KEY ||
-            arg_type == ARG_PTR_TO_MAP_VALUE) {
+            arg_type == ARG_PTR_TO_MAP_VALUE ||
+            arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE) {
                 expected_type = PTR_TO_STACK;
                 if (!type_is_pkt_pointer(type) && type != PTR_TO_MAP_VALUE &&
                     type != expected_type)
@@ -2187,7 +2201,8 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 regno,
                 err = check_helper_mem_access(env, regno,
                                               meta->map_ptr->key_size, false,
                                               NULL);
-        } else if (arg_type == ARG_PTR_TO_MAP_VALUE) {
+        } else if (arg_type == ARG_PTR_TO_MAP_VALUE ||
+                   arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE) {
                 /* bpf_map_xxx(..., map_ptr, ..., value) call:
                  * check [value, value + map->value_size) validity
                  */
@@ -2196,9 +2211,10 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 regno,
                         verbose(env, "invalid map_ptr to access map->value\n");
                         return -EACCES;
                 }
+                meta->raw_mode = (arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE);
                 err = check_helper_mem_access(env, regno,
                                               meta->map_ptr->value_size, false,
-                                              NULL);
+                                              meta);
         } else if (arg_type_is_mem_size(arg_type)) {
                 bool zero_size_allowed = (arg_type == ARG_CONST_SIZE_OR_ZERO);
 
@@ -2321,6 +2337,13 @@ static int check_map_func_compatibility(struct bpf_verifier_env *env,
                 if (func_id != BPF_FUNC_sk_select_reuseport)
                         goto error;
                 break;
+        case BPF_MAP_TYPE_QUEUE:
+        case BPF_MAP_TYPE_STACK:
+                if (func_id != BPF_FUNC_map_peek_elem &&
+                    func_id != BPF_FUNC_map_pop_elem &&
+                    func_id != BPF_FUNC_map_push_elem)
+                        goto error;
+                break;
         default:
                 break;
         }
@@ -2377,6 +2400,13 @@ static int check_map_func_compatibility(struct bpf_verifier_env *env,
                 if (map->map_type != BPF_MAP_TYPE_REUSEPORT_SOCKARRAY)
                         goto error;
                 break;
+        case BPF_FUNC_map_peek_elem:
+        case BPF_FUNC_map_pop_elem:
+        case BPF_FUNC_map_push_elem:
+                if (map->map_type != BPF_MAP_TYPE_QUEUE &&
+                    map->map_type != BPF_MAP_TYPE_STACK)
+                        goto error;
+                break;
         default:
                 break;
         }
@@ -2672,7 +2702,10 @@ record_func_map(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
         if (func_id != BPF_FUNC_tail_call &&
             func_id != BPF_FUNC_map_lookup_elem &&
             func_id != BPF_FUNC_map_update_elem &&
-            func_id != BPF_FUNC_map_delete_elem)
+            func_id != BPF_FUNC_map_delete_elem &&
+            func_id != BPF_FUNC_map_push_elem &&
+            func_id != BPF_FUNC_map_pop_elem &&
+            func_id != BPF_FUNC_map_peek_elem)
                 return 0;
 
         if (meta->map_ptr == NULL) {
@@ -5244,7 +5277,8 @@ static int do_check(struct bpf_verifier_env *env)
 
                         if (is_ctx_reg(env, insn->dst_reg)) {
                                 verbose(env, "BPF_ST stores into R%d %s is not allowed\n",
-                                        insn->dst_reg, reg_type_str[insn->dst_reg]);
+                                        insn->dst_reg,
+                                        reg_type_str[reg_state(env, insn->dst_reg)->type]);
                                 return -EACCES;
                         }
 
@@ -6144,7 +6178,10 @@ static int fixup_bpf_calls(struct bpf_verifier_env *env)
                 if (prog->jit_requested && BITS_PER_LONG == 64 &&
                     (insn->imm == BPF_FUNC_map_lookup_elem ||
                      insn->imm == BPF_FUNC_map_update_elem ||
-                     insn->imm == BPF_FUNC_map_delete_elem)) {
+                     insn->imm == BPF_FUNC_map_delete_elem ||
+                     insn->imm == BPF_FUNC_map_push_elem   ||
+                     insn->imm == BPF_FUNC_map_pop_elem    ||
+                     insn->imm == BPF_FUNC_map_peek_elem)) {
                         aux = &env->insn_aux_data[i + delta];
                         if (bpf_map_ptr_poisoned(aux))
                                 goto patch_call_imm;
@@ -6177,6 +6214,14 @@ static int fixup_bpf_calls(struct bpf_verifier_env *env)
                         BUILD_BUG_ON(!__same_type(ops->map_update_elem,
                                      (int (*)(struct bpf_map *map, void *key, void *value,
                                               u64 flags))NULL));
+                        BUILD_BUG_ON(!__same_type(ops->map_push_elem,
+                                     (int (*)(struct bpf_map *map, void *value,
+                                              u64 flags))NULL));
+                        BUILD_BUG_ON(!__same_type(ops->map_pop_elem,
+                                     (int (*)(struct bpf_map *map, void *value))NULL));
+                        BUILD_BUG_ON(!__same_type(ops->map_peek_elem,
+                                     (int (*)(struct bpf_map *map, void *value))NULL));
+
                         switch (insn->imm) {
                         case BPF_FUNC_map_lookup_elem:
                                 insn->imm = BPF_CAST_CALL(ops->map_lookup_elem) -
@@ -6190,6 +6235,18 @@ static int fixup_bpf_calls(struct bpf_verifier_env *env)
                                 insn->imm = BPF_CAST_CALL(ops->map_delete_elem) -
                                             __bpf_call_base;
                                 continue;
+                        case BPF_FUNC_map_push_elem:
+                                insn->imm = BPF_CAST_CALL(ops->map_push_elem) -
+                                            __bpf_call_base;
+                                continue;
+                        case BPF_FUNC_map_pop_elem:
+                                insn->imm = BPF_CAST_CALL(ops->map_pop_elem) -
+                                            __bpf_call_base;
+                                continue;
+                        case BPF_FUNC_map_peek_elem:
+                                insn->imm = BPF_CAST_CALL(ops->map_peek_elem) -
+                                            __bpf_call_base;
+                                continue;
                         }
 
                         goto patch_call_imm;