5 files changed, 208 insertions, 135 deletions

diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index f45827e205d3..b4f1cb0c5ac7 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -1162,12 +1162,12 @@ out:
         LD_ABS_W: /* BPF_R0 = ntohl(*(u32 *) (skb->data + imm32)) */
                 off = IMM;
 load_word:
-                /* BPF_LD + BPD_ABS and BPF_LD + BPF_IND insns are
-                 * only appearing in the programs where ctx ==
-                 * skb. All programs keep 'ctx' in regs[BPF_REG_CTX]
-                 * == BPF_R6, bpf_convert_filter() saves it in BPF_R6,
-                 * internal BPF verifier will check that BPF_R6 ==
-                 * ctx.
+                /* BPF_LD + BPD_ABS and BPF_LD + BPF_IND insns are only
+                 * appearing in the programs where ctx == skb
+                 * (see may_access_skb() in the verifier). All programs
+                 * keep 'ctx' in regs[BPF_REG_CTX] == BPF_R6,
+                 * bpf_convert_filter() saves it in BPF_R6, internal BPF
+                 * verifier will check that BPF_R6 == ctx.
                  *
                  * BPF_ABS and BPF_IND are wrappers of function calls,
                  * so they scratch BPF_R1-BPF_R5 registers, preserve
diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c
index 3ea87fb19a94..361a69dfe543 100644
--- a/kernel/bpf/hashtab.c
+++ b/kernel/bpf/hashtab.c
@@ -13,11 +13,12 @@
 #include <linux/bpf.h>
 #include <linux/jhash.h>
 #include <linux/filter.h>
+#include <linux/rculist_nulls.h>
 #include "percpu_freelist.h"
 #include "bpf_lru_list.h"
 
 struct bucket {
-        struct hlist_head head;
+        struct hlist_nulls_head head;
         raw_spinlock_t lock;
 };
 
@@ -29,28 +30,26 @@ struct bpf_htab {
                 struct pcpu_freelist freelist;
                 struct bpf_lru lru;
         };
-        void __percpu *extra_elems;
+        struct htab_elem *__percpu *extra_elems;
         atomic_t count; /* number of elements in this hashtable */
         u32 n_buckets;  /* number of hash buckets */
         u32 elem_size;  /* size of each element in bytes */
 };
 
-enum extra_elem_state {
-        HTAB_NOT_AN_EXTRA_ELEM = 0,
-        HTAB_EXTRA_ELEM_FREE,
-        HTAB_EXTRA_ELEM_USED
-};
-
 /* each htab element is struct htab_elem + key + value */
 struct htab_elem {
         union {
-                struct hlist_node hash_node;
-                struct bpf_htab *htab;
-                struct pcpu_freelist_node fnode;
+                struct hlist_nulls_node hash_node;
+                struct {
+                        void *padding;
+                        union {
+                                struct bpf_htab *htab;
+                                struct pcpu_freelist_node fnode;
+                        };
+                };
         };
         union {
                 struct rcu_head rcu;
-                enum extra_elem_state state;
                 struct bpf_lru_node lru_node;
         };
         u32 hash;
@@ -71,6 +70,11 @@ static bool htab_is_percpu(const struct bpf_htab *htab)
                 htab->map.map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH;
 }
 
+static bool htab_is_prealloc(const struct bpf_htab *htab)
+{
+        return !(htab->map.map_flags & BPF_F_NO_PREALLOC);
+}
+
 static inline void htab_elem_set_ptr(struct htab_elem *l, u32 key_size,
                                      void __percpu *pptr)
 {
@@ -122,17 +126,20 @@ static struct htab_elem *prealloc_lru_pop(struct bpf_htab *htab, void *key,
 
 static int prealloc_init(struct bpf_htab *htab)
 {
+        u32 num_entries = htab->map.max_entries;
         int err = -ENOMEM, i;
 
-        htab->elems = bpf_map_area_alloc(htab->elem_size *
-                                         htab->map.max_entries);
+        if (!htab_is_percpu(htab) && !htab_is_lru(htab))
+                num_entries += num_possible_cpus();
+
+        htab->elems = bpf_map_area_alloc(htab->elem_size * num_entries);
         if (!htab->elems)
                 return -ENOMEM;
 
         if (!htab_is_percpu(htab))
                 goto skip_percpu_elems;
 
-        for (i = 0; i < htab->map.max_entries; i++) {
+        for (i = 0; i < num_entries; i++) {
                 u32 size = round_up(htab->map.value_size, 8);
                 void __percpu *pptr;
 
@@ -160,10 +167,11 @@ skip_percpu_elems:
         if (htab_is_lru(htab))
                 bpf_lru_populate(&htab->lru, htab->elems,
                                  offsetof(struct htab_elem, lru_node),
-                                 htab->elem_size, htab->map.max_entries);
+                                 htab->elem_size, num_entries);
         else
-                pcpu_freelist_populate(&htab->freelist, htab->elems,
-                                       htab->elem_size, htab->map.max_entries);
+                pcpu_freelist_populate(&htab->freelist,
+                                       htab->elems + offsetof(struct htab_elem, fnode),
+                                       htab->elem_size, num_entries);
 
         return 0;
 
@@ -184,16 +192,22 @@ static void prealloc_destroy(struct bpf_htab *htab)
 
 static int alloc_extra_elems(struct bpf_htab *htab)
 {
-        void __percpu *pptr;
+        struct htab_elem *__percpu *pptr, *l_new;
+        struct pcpu_freelist_node *l;
         int cpu;
 
-        pptr = __alloc_percpu_gfp(htab->elem_size, 8, GFP_USER | __GFP_NOWARN);
+        pptr = __alloc_percpu_gfp(sizeof(struct htab_elem *), 8,
+                                  GFP_USER | __GFP_NOWARN);
         if (!pptr)
                 return -ENOMEM;
 
         for_each_possible_cpu(cpu) {
-                ((struct htab_elem *)per_cpu_ptr(pptr, cpu))->state =
-                        HTAB_EXTRA_ELEM_FREE;
+                l = pcpu_freelist_pop(&htab->freelist);
+                /* pop will succeed, since prealloc_init()
+                 * preallocated extra num_possible_cpus elements
+                 */
+                l_new = container_of(l, struct htab_elem, fnode);
+                *per_cpu_ptr(pptr, cpu) = l_new;
         }
         htab->extra_elems = pptr;
         return 0;
@@ -217,6 +231,11 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr)
         int err, i;
         u64 cost;
 
+        BUILD_BUG_ON(offsetof(struct htab_elem, htab) !=
+                     offsetof(struct htab_elem, hash_node.pprev));
+        BUILD_BUG_ON(offsetof(struct htab_elem, fnode.next) !=
+                     offsetof(struct htab_elem, hash_node.pprev));
+
         if (lru && !capable(CAP_SYS_ADMIN))
                 /* LRU implementation is much complicated than other
                  * maps.  Hence, limit to CAP_SYS_ADMIN for now.
@@ -326,29 +345,29 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr)
                 goto free_htab;
 
         for (i = 0; i < htab->n_buckets; i++) {
-                INIT_HLIST_HEAD(&htab->buckets[i].head);
+                INIT_HLIST_NULLS_HEAD(&htab->buckets[i].head, i);
                 raw_spin_lock_init(&htab->buckets[i].lock);
         }
 
-        if (!percpu && !lru) {
-                /* lru itself can remove the least used element, so
-                 * there is no need for an extra elem during map_update.
-                 */
-                err = alloc_extra_elems(htab);
-                if (err)
-                        goto free_buckets;
-        }
-
         if (prealloc) {
                 err = prealloc_init(htab);
                 if (err)
-                        goto free_extra_elems;
+                        goto free_buckets;
+
+                if (!percpu && !lru) {
+                        /* lru itself can remove the least used element, so
+                         * there is no need for an extra elem during map_update.
+                         */
+                        err = alloc_extra_elems(htab);
+                        if (err)
+                                goto free_prealloc;
+                }
         }
 
         return &htab->map;
 
-free_extra_elems:
-        free_percpu(htab->extra_elems);
+free_prealloc:
+        prealloc_destroy(htab);
 free_buckets:
         bpf_map_area_free(htab->buckets);
 free_htab:
@@ -366,20 +385,44 @@ static inline struct bucket *__select_bucket(struct bpf_htab *htab, u32 hash)
         return &htab->buckets[hash & (htab->n_buckets - 1)];
 }
 
-static inline struct hlist_head *select_bucket(struct bpf_htab *htab, u32 hash)
+static inline struct hlist_nulls_head *select_bucket(struct bpf_htab *htab, u32 hash)
 {
         return &__select_bucket(htab, hash)->head;
 }
 
-static struct htab_elem *lookup_elem_raw(struct hlist_head *head, u32 hash,
+/* this lookup function can only be called with bucket lock taken */
+static struct htab_elem *lookup_elem_raw(struct hlist_nulls_head *head, u32 hash,
                                          void *key, u32 key_size)
 {
+        struct hlist_nulls_node *n;
+        struct htab_elem *l;
+
+        hlist_nulls_for_each_entry_rcu(l, n, head, hash_node)
+                if (l->hash == hash && !memcmp(&l->key, key, key_size))
+                        return l;
+
+        return NULL;
+}
+
+/* can be called without bucket lock. it will repeat the loop in
+ * the unlikely event when elements moved from one bucket into another
+ * while link list is being walked
+ */
+static struct htab_elem *lookup_nulls_elem_raw(struct hlist_nulls_head *head,
+                                               u32 hash, void *key,
+                                               u32 key_size, u32 n_buckets)
+{
+        struct hlist_nulls_node *n;
         struct htab_elem *l;
 
-        hlist_for_each_entry_rcu(l, head, hash_node)
+again:
+        hlist_nulls_for_each_entry_rcu(l, n, head, hash_node)
                 if (l->hash == hash && !memcmp(&l->key, key, key_size))
                         return l;
 
+        if (unlikely(get_nulls_value(n) != (hash & (n_buckets - 1))))
+                goto again;
+
         return NULL;
 }
 
@@ -387,7 +430,7 @@ static struct htab_elem *lookup_elem_raw(struct hlist_head *head, u32 hash,
 static void *__htab_map_lookup_elem(struct bpf_map *map, void *key)
 {
         struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
-        struct hlist_head *head;
+        struct hlist_nulls_head *head;
         struct htab_elem *l;
         u32 hash, key_size;
 
@@ -400,7 +443,7 @@ static void *__htab_map_lookup_elem(struct bpf_map *map, void *key)
 
         head = select_bucket(htab, hash);
 
-        l = lookup_elem_raw(head, hash, key, key_size);
+        l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets);
 
         return l;
 }
@@ -433,8 +476,9 @@ static void *htab_lru_map_lookup_elem(struct bpf_map *map, void *key)
 static bool htab_lru_map_delete_node(void *arg, struct bpf_lru_node *node)
 {
         struct bpf_htab *htab = (struct bpf_htab *)arg;
-        struct htab_elem *l, *tgt_l;
-        struct hlist_head *head;
+        struct htab_elem *l = NULL, *tgt_l;
+        struct hlist_nulls_head *head;
+        struct hlist_nulls_node *n;
         unsigned long flags;
         struct bucket *b;
 
@@ -444,9 +488,9 @@ static bool htab_lru_map_delete_node(void *arg, struct bpf_lru_node *node)
 
         raw_spin_lock_irqsave(&b->lock, flags);
 
-        hlist_for_each_entry_rcu(l, head, hash_node)
+        hlist_nulls_for_each_entry_rcu(l, n, head, hash_node)
                 if (l == tgt_l) {
-                        hlist_del_rcu(&l->hash_node);
+                        hlist_nulls_del_rcu(&l->hash_node);
                         break;
                 }
 
@@ -459,7 +503,7 @@ static bool htab_lru_map_delete_node(void *arg, struct bpf_lru_node *node)
 static int htab_map_get_next_key(struct bpf_map *map, void *key, void *next_key)
 {
         struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
-        struct hlist_head *head;
+        struct hlist_nulls_head *head;
         struct htab_elem *l, *next_l;
         u32 hash, key_size;
         int i;
@@ -473,7 +517,7 @@ static int htab_map_get_next_key(struct bpf_map *map, void *key, void *next_key)
         head = select_bucket(htab, hash);
 
         /* lookup the key */
-        l = lookup_elem_raw(head, hash, key, key_size);
+        l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets);
 
         if (!l) {
                 i = 0;
@@ -481,7 +525,7 @@ static int htab_map_get_next_key(struct bpf_map *map, void *key, void *next_key)
         }
 
         /* key was found, get next key in the same bucket */
-        next_l = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu(&l->hash_node)),
+        next_l = hlist_nulls_entry_safe(rcu_dereference_raw(hlist_nulls_next_rcu(&l->hash_node)),
                                   struct htab_elem, hash_node);
 
         if (next_l) {
@@ -500,7 +544,7 @@ find_first_elem:
                 head = select_bucket(htab, i);
 
                 /* pick first element in the bucket */
-                next_l = hlist_entry_safe(rcu_dereference_raw(hlist_first_rcu(head)),
+                next_l = hlist_nulls_entry_safe(rcu_dereference_raw(hlist_nulls_first_rcu(head)),
                                           struct htab_elem, hash_node);
                 if (next_l) {
                         /* if it's not empty, just return it */
@@ -538,12 +582,7 @@ static void htab_elem_free_rcu(struct rcu_head *head)
 
 static void free_htab_elem(struct bpf_htab *htab, struct htab_elem *l)
 {
-        if (l->state == HTAB_EXTRA_ELEM_USED) {
-                l->state = HTAB_EXTRA_ELEM_FREE;
-                return;
-        }
-
-        if (!(htab->map.map_flags & BPF_F_NO_PREALLOC)) {
+        if (htab_is_prealloc(htab)) {
                 pcpu_freelist_push(&htab->freelist, &l->fnode);
         } else {
                 atomic_dec(&htab->count);
@@ -573,43 +612,43 @@ static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr,
 static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key,
                                          void *value, u32 key_size, u32 hash,
                                          bool percpu, bool onallcpus,
-                                         bool old_elem_exists)
+                                         struct htab_elem *old_elem)
 {
         u32 size = htab->map.value_size;
-        bool prealloc = !(htab->map.map_flags & BPF_F_NO_PREALLOC);
-        struct htab_elem *l_new;
+        bool prealloc = htab_is_prealloc(htab);
+        struct htab_elem *l_new, **pl_new;
         void __percpu *pptr;
-        int err = 0;
 
         if (prealloc) {
-                l_new = (struct htab_elem *)pcpu_freelist_pop(&htab->freelist);
-                if (!l_new)
-                        err = -E2BIG;
-        } else {
-                if (atomic_inc_return(&htab->count) > htab->map.max_entries) {
-                        atomic_dec(&htab->count);
-                        err = -E2BIG;
+                if (old_elem) {
+                        /* if we're updating the existing element,
+                         * use per-cpu extra elems to avoid freelist_pop/push
+                         */
+                        pl_new = this_cpu_ptr(htab->extra_elems);
+                        l_new = *pl_new;
+                        *pl_new = old_elem;
                 } else {
-                        l_new = kmalloc(htab->elem_size,
-                                        GFP_ATOMIC | __GFP_NOWARN);
-                        if (!l_new)
-                                return ERR_PTR(-ENOMEM);
-                }
-        }
+                        struct pcpu_freelist_node *l;
 
-        if (err) {
-                if (!old_elem_exists)
-                        return ERR_PTR(err);
-
-                /* if we're updating the existing element and the hash table
-                 * is full, use per-cpu extra elems
-                 */
-                l_new = this_cpu_ptr(htab->extra_elems);
-                if (l_new->state != HTAB_EXTRA_ELEM_FREE)
-                        return ERR_PTR(-E2BIG);
-                l_new->state = HTAB_EXTRA_ELEM_USED;
+                        l = pcpu_freelist_pop(&htab->freelist);
+                        if (!l)
+                                return ERR_PTR(-E2BIG);
+                        l_new = container_of(l, struct htab_elem, fnode);
+                }
         } else {
-                l_new->state = HTAB_NOT_AN_EXTRA_ELEM;
+                if (atomic_inc_return(&htab->count) > htab->map.max_entries)
+                        if (!old_elem) {
+                                /* when map is full and update() is replacing
+                                 * old element, it's ok to allocate, since
+                                 * old element will be freed immediately.
+                                 * Otherwise return an error
+                                 */
+                                atomic_dec(&htab->count);
+                                return ERR_PTR(-E2BIG);
+                        }
+                l_new = kmalloc(htab->elem_size, GFP_ATOMIC | __GFP_NOWARN);
+                if (!l_new)
+                        return ERR_PTR(-ENOMEM);
         }
 
         memcpy(l_new->key, key, key_size);
@@ -661,7 +700,7 @@ static int htab_map_update_elem(struct bpf_map *map, void *key, void *value,
 {
         struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
         struct htab_elem *l_new = NULL, *l_old;
-        struct hlist_head *head;
+        struct hlist_nulls_head *head;
         unsigned long flags;
         struct bucket *b;
         u32 key_size, hash;
@@ -690,7 +729,7 @@ static int htab_map_update_elem(struct bpf_map *map, void *key, void *value,
                 goto err;
 
         l_new = alloc_htab_elem(htab, key, value, key_size, hash, false, false,
-                                !!l_old);
+                                l_old);
         if (IS_ERR(l_new)) {
                 /* all pre-allocated elements are in use or memory exhausted */
                 ret = PTR_ERR(l_new);
@@ -700,10 +739,11 @@ static int htab_map_update_elem(struct bpf_map *map, void *key, void *value,
         /* add new element to the head of the list, so that
          * concurrent search will find it before old elem
          */
-        hlist_add_head_rcu(&l_new->hash_node, head);
+        hlist_nulls_add_head_rcu(&l_new->hash_node, head);
         if (l_old) {
-                hlist_del_rcu(&l_old->hash_node);
-                free_htab_elem(htab, l_old);
+                hlist_nulls_del_rcu(&l_old->hash_node);
+                if (!htab_is_prealloc(htab))
+                        free_htab_elem(htab, l_old);
         }
         ret = 0;
 err:
@@ -716,7 +756,7 @@ static int htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value,
 {
         struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
         struct htab_elem *l_new, *l_old = NULL;
-        struct hlist_head *head;
+        struct hlist_nulls_head *head;
         unsigned long flags;
         struct bucket *b;
         u32 key_size, hash;
@@ -757,10 +797,10 @@ static int htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value,
         /* add new element to the head of the list, so that
          * concurrent search will find it before old elem
          */
-        hlist_add_head_rcu(&l_new->hash_node, head);
+        hlist_nulls_add_head_rcu(&l_new->hash_node, head);
         if (l_old) {
                 bpf_lru_node_set_ref(&l_new->lru_node);
-                hlist_del_rcu(&l_old->hash_node);
+                hlist_nulls_del_rcu(&l_old->hash_node);
         }
         ret = 0;
 
@@ -781,7 +821,7 @@ static int __htab_percpu_map_update_elem(struct bpf_map *map, void *key,
 {
         struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
         struct htab_elem *l_new = NULL, *l_old;
-        struct hlist_head *head;
+        struct hlist_nulls_head *head;
         unsigned long flags;
         struct bucket *b;
         u32 key_size, hash;
@@ -815,12 +855,12 @@ static int __htab_percpu_map_update_elem(struct bpf_map *map, void *key,
                                 value, onallcpus);
         } else {
                 l_new = alloc_htab_elem(htab, key, value, key_size,
-                                        hash, true, onallcpus, false);
+                                        hash, true, onallcpus, NULL);
                 if (IS_ERR(l_new)) {
                         ret = PTR_ERR(l_new);
                         goto err;
                 }
-                hlist_add_head_rcu(&l_new->hash_node, head);
+                hlist_nulls_add_head_rcu(&l_new->hash_node, head);
         }
         ret = 0;
 err:
@@ -834,7 +874,7 @@ static int __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key,
 {
         struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
         struct htab_elem *l_new = NULL, *l_old;
-        struct hlist_head *head;
+        struct hlist_nulls_head *head;
         unsigned long flags;
         struct bucket *b;
         u32 key_size, hash;
@@ -882,7 +922,7 @@ static int __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key,
         } else {
                 pcpu_copy_value(htab, htab_elem_get_ptr(l_new, key_size),
                                 value, onallcpus);
-                hlist_add_head_rcu(&l_new->hash_node, head);
+                hlist_nulls_add_head_rcu(&l_new->hash_node, head);
                 l_new = NULL;
         }
         ret = 0;
@@ -910,7 +950,7 @@ static int htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key,
 static int htab_map_delete_elem(struct bpf_map *map, void *key)
 {
         struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
-        struct hlist_head *head;
+        struct hlist_nulls_head *head;
         struct bucket *b;
         struct htab_elem *l;
         unsigned long flags;
@@ -930,7 +970,7 @@ static int htab_map_delete_elem(struct bpf_map *map, void *key)
         l = lookup_elem_raw(head, hash, key, key_size);
 
         if (l) {
-                hlist_del_rcu(&l->hash_node);
+                hlist_nulls_del_rcu(&l->hash_node);
                 free_htab_elem(htab, l);
                 ret = 0;
         }
@@ -942,7 +982,7 @@ static int htab_map_delete_elem(struct bpf_map *map, void *key)
 static int htab_lru_map_delete_elem(struct bpf_map *map, void *key)
 {
         struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
-        struct hlist_head *head;
+        struct hlist_nulls_head *head;
         struct bucket *b;
         struct htab_elem *l;
         unsigned long flags;
@@ -962,7 +1002,7 @@ static int htab_lru_map_delete_elem(struct bpf_map *map, void *key)
         l = lookup_elem_raw(head, hash, key, key_size);
 
         if (l) {
-                hlist_del_rcu(&l->hash_node);
+                hlist_nulls_del_rcu(&l->hash_node);
                 ret = 0;
         }
 
@@ -977,14 +1017,13 @@ static void delete_all_elements(struct bpf_htab *htab)
         int i;
 
         for (i = 0; i < htab->n_buckets; i++) {
-                struct hlist_head *head = select_bucket(htab, i);
-                struct hlist_node *n;
+                struct hlist_nulls_head *head = select_bucket(htab, i);
+                struct hlist_nulls_node *n;
                 struct htab_elem *l;
 
-                hlist_for_each_entry_safe(l, n, head, hash_node) {
-                        hlist_del_rcu(&l->hash_node);
-                        if (l->state != HTAB_EXTRA_ELEM_USED)
-                                htab_elem_free(htab, l);
+                hlist_nulls_for_each_entry_safe(l, n, head, hash_node) {
+                        hlist_nulls_del_rcu(&l->hash_node);
+                        htab_elem_free(htab, l);
                 }
         }
 }
@@ -1004,7 +1043,7 @@ static void htab_map_free(struct bpf_map *map)
          * not have executed. Wait for them.
          */
         rcu_barrier();
-        if (htab->map.map_flags & BPF_F_NO_PREALLOC)
+        if (!htab_is_prealloc(htab))
                 delete_all_elements(htab);
         else
                 prealloc_destroy(htab);
diff --git a/kernel/bpf/lpm_trie.c b/kernel/bpf/lpm_trie.c
index 8bfe0afaee10..b37bd9ab7f57 100644
--- a/kernel/bpf/lpm_trie.c
+++ b/kernel/bpf/lpm_trie.c
@@ -500,9 +500,15 @@ unlock:
         raw_spin_unlock(&trie->lock);
 }
 
+static int trie_get_next_key(struct bpf_map *map, void *key, void *next_key)
+{
+        return -ENOTSUPP;
+}
+
 static const struct bpf_map_ops trie_ops = {
         .map_alloc = trie_alloc,
         .map_free = trie_free,
+        .map_get_next_key = trie_get_next_key,
         .map_lookup_elem = trie_lookup_elem,
         .map_update_elem = trie_update_elem,
         .map_delete_elem = trie_delete_elem,
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 7af0dcc5d755..821f9e807de5 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -617,6 +617,14 @@ static void fixup_bpf_calls(struct bpf_prog *prog)
                         if (insn->imm == BPF_FUNC_xdp_adjust_head)
                                 prog->xdp_adjust_head = 1;
                         if (insn->imm == BPF_FUNC_tail_call) {
+                                /* If we tail call into other programs, we
+                                 * cannot make any assumptions since they
+                                 * can be replaced dynamically during runtime
+                                 * in the program array.
+                                 */
+                                prog->cb_access = 1;
+                                prog->xdp_adjust_head = 1;
+
                                 /* mark bpf_tail_call as different opcode
                                  * to avoid conditional branch in
                                  * interpeter for every normal call
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 796b68d00119..a834068a400e 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -765,38 +765,56 @@ static bool is_pointer_value(struct bpf_verifier_env *env, int regno)
         }
 }
 
-static int check_ptr_alignment(struct bpf_verifier_env *env,
-                               struct bpf_reg_state *reg, int off, int size)
+static int check_pkt_ptr_alignment(const struct bpf_reg_state *reg,
+                                   int off, int size)
 {
-        if (reg->type != PTR_TO_PACKET && reg->type != PTR_TO_MAP_VALUE_ADJ) {
-                if (off % size != 0) {
-                        verbose("misaligned access off %d size %d\n",
-                                off, size);
-                        return -EACCES;
-                } else {
-                        return 0;
-                }
-        }
-
-        if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS))
-                /* misaligned access to packet is ok on x86,arm,arm64 */
-                return 0;
-
         if (reg->id && size != 1) {
-                verbose("Unknown packet alignment. Only byte-sized access allowed\n");
+                verbose("Unknown alignment. Only byte-sized access allowed in packet access.\n");
                 return -EACCES;
         }
 
         /* skb->data is NET_IP_ALIGN-ed */
-        if (reg->type == PTR_TO_PACKET &&
-            (NET_IP_ALIGN + reg->off + off) % size != 0) {
+        if ((NET_IP_ALIGN + reg->off + off) % size != 0) {
                 verbose("misaligned packet access off %d+%d+%d size %d\n",
                         NET_IP_ALIGN, reg->off, off, size);
                 return -EACCES;
         }
+
         return 0;
 }
 
+static int check_val_ptr_alignment(const struct bpf_reg_state *reg,
+                                   int size)
+{
+        if (size != 1) {
+                verbose("Unknown alignment. Only byte-sized access allowed in value access.\n");
+                return -EACCES;
+        }
+
+        return 0;
+}
+
+static int check_ptr_alignment(const struct bpf_reg_state *reg,
+                               int off, int size)
+{
+        switch (reg->type) {
+        case PTR_TO_PACKET:
+                return IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) ? 0 :
+                       check_pkt_ptr_alignment(reg, off, size);
+        case PTR_TO_MAP_VALUE_ADJ:
+                return IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) ? 0 :
+                       check_val_ptr_alignment(reg, size);
+        default:
+                if (off % size != 0) {
+                        verbose("misaligned access off %d size %d\n",
+                                off, size);
+                        return -EACCES;
+                }
+
+                return 0;
+        }
+}
+
 /* check whether memory at (regno + off) is accessible for t = (read | write)
  * if t==write, value_regno is a register which value is stored into memory
  * if t==read, value_regno is a register which will receive the value from memory
@@ -818,7 +836,7 @@ static int check_mem_access(struct bpf_verifier_env *env, u32 regno, int off,
         if (size < 0)
                 return size;
 
-        err = check_ptr_alignment(env, reg, off, size);
+        err = check_ptr_alignment(reg, off, size);
         if (err)
                 return err;
 
@@ -1925,6 +1943,7 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn)
                  * register as unknown.
                  */
                 if (env->allow_ptr_leaks &&
+                    BPF_CLASS(insn->code) == BPF_ALU64 && opcode == BPF_ADD &&
                     (dst_reg->type == PTR_TO_MAP_VALUE ||
                      dst_reg->type == PTR_TO_MAP_VALUE_ADJ))
                         dst_reg->type = PTR_TO_MAP_VALUE_ADJ;
@@ -1973,14 +1992,15 @@ static void find_good_pkt_pointers(struct bpf_verifier_state *state,
 
         for (i = 0; i < MAX_BPF_REG; i++)
                 if (regs[i].type == PTR_TO_PACKET && regs[i].id == dst_reg->id)
-                        regs[i].range = dst_reg->off;
+                        /* keep the maximum range already checked */
+                        regs[i].range = max(regs[i].range, dst_reg->off);
 
         for (i = 0; i < MAX_BPF_STACK; i += BPF_REG_SIZE) {
                 if (state->stack_slot_type[i] != STACK_SPILL)
                         continue;
                 reg = &state->spilled_regs[i / BPF_REG_SIZE];
                 if (reg->type == PTR_TO_PACKET && reg->id == dst_reg->id)
-                        reg->range = dst_reg->off;
+                        reg->range = max(reg->range, dst_reg->off);
         }
 }