10 files changed, 143 insertions, 48 deletions

diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c
index 715f9fcf4712..c57bd10340ed 100644
--- a/kernel/bpf/btf.c
+++ b/kernel/bpf/btf.c
@@ -467,7 +467,7 @@ static const struct btf_kind_operations *btf_type_ops(const struct btf_type *t)
         return kind_ops[BTF_INFO_KIND(t->info)];
 }
 
-bool btf_name_offset_valid(const struct btf *btf, u32 offset)
+static bool btf_name_offset_valid(const struct btf *btf, u32 offset)
 {
         return BTF_STR_OFFSET_VALID(offset) &&
                 offset < btf->hdr.str_len;
@@ -1219,8 +1219,6 @@ static void btf_bitfield_seq_show(void *data, u8 bits_offset,
         u8 nr_copy_bits;
         u64 print_num;
 
-        data += BITS_ROUNDDOWN_BYTES(bits_offset);
-        bits_offset = BITS_PER_BYTE_MASKED(bits_offset);
         nr_copy_bits = nr_bits + bits_offset;
         nr_copy_bytes = BITS_ROUNDUP_BYTES(nr_copy_bits);
 
@@ -1255,7 +1253,9 @@ static void btf_int_bits_seq_show(const struct btf *btf,
          * BTF_INT_OFFSET() cannot exceed 64 bits.
          */
         total_bits_offset = bits_offset + BTF_INT_OFFSET(int_data);
-        btf_bitfield_seq_show(data, total_bits_offset, nr_bits, m);
+        data += BITS_ROUNDDOWN_BYTES(total_bits_offset);
+        bits_offset = BITS_PER_BYTE_MASKED(total_bits_offset);
+        btf_bitfield_seq_show(data, bits_offset, nr_bits, m);
 }
 
 static void btf_int_seq_show(const struct btf *btf, const struct btf_type *t,
@@ -1459,7 +1459,8 @@ static int btf_modifier_resolve(struct btf_verifier_env *env,
 
                 /* "typedef void new_void", "const void"...etc */
                 if (!btf_type_is_void(next_type) &&
-                    !btf_type_is_fwd(next_type)) {
+                    !btf_type_is_fwd(next_type) &&
+                    !btf_type_is_func_proto(next_type)) {
                         btf_verifier_log_type(env, v->t, "Invalid type_id");
                         return -EINVAL;
                 }
@@ -2001,12 +2002,12 @@ static void btf_struct_seq_show(const struct btf *btf, const struct btf_type *t,
 
                 member_offset = btf_member_bit_offset(t, member);
                 bitfield_size = btf_member_bitfield_size(t, member);
+                bytes_offset = BITS_ROUNDDOWN_BYTES(member_offset);
+                bits8_offset = BITS_PER_BYTE_MASKED(member_offset);
                 if (bitfield_size) {
-                        btf_bitfield_seq_show(data, member_offset,
+                        btf_bitfield_seq_show(data + bytes_offset, bits8_offset,
                                               bitfield_size, m);
                 } else {
-                        bytes_offset = BITS_ROUNDDOWN_BYTES(member_offset);
-                        bits8_offset = BITS_PER_BYTE_MASKED(member_offset);
                         ops = btf_type_ops(member_type);
                         ops->seq_show(btf, member_type, member->type,
                                       data + bytes_offset, bits8_offset, m);
diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c
index 9425c2fb872f..d17d05570a3f 100644
--- a/kernel/bpf/cgroup.c
+++ b/kernel/bpf/cgroup.c
@@ -572,7 +572,7 @@ int __cgroup_bpf_run_filter_skb(struct sock *sk,
         bpf_compute_and_save_data_end(skb, &saved_data_end);
 
         ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], skb,
-                                 bpf_prog_run_save_cb);
+                                 __bpf_prog_run_save_cb);
         bpf_restore_data_end(skb, saved_data_end);
         __skb_pull(skb, offset);
         skb->sk = save_sk;
@@ -718,6 +718,7 @@ cgroup_dev_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
         case BPF_FUNC_trace_printk:
                 if (capable(CAP_SYS_ADMIN))
                         return bpf_get_trace_printk_proto();
+                /* fall through */
         default:
                 return NULL;
         }
diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c
index 4b7c76765d9d..f9274114c88d 100644
--- a/kernel/bpf/hashtab.c
+++ b/kernel/bpf/hashtab.c
@@ -686,7 +686,7 @@ static void free_htab_elem(struct bpf_htab *htab, struct htab_elem *l)
         }
 
         if (htab_is_prealloc(htab)) {
-                pcpu_freelist_push(&htab->freelist, &l->fnode);
+                __pcpu_freelist_push(&htab->freelist, &l->fnode);
         } else {
                 atomic_dec(&htab->count);
                 l->htab = htab;
@@ -748,7 +748,7 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key,
                 } else {
                         struct pcpu_freelist_node *l;
 
-                        l = pcpu_freelist_pop(&htab->freelist);
+                        l = __pcpu_freelist_pop(&htab->freelist);
                         if (!l)
                                 return ERR_PTR(-E2BIG);
                         l_new = container_of(l, struct htab_elem, fnode);
diff --git a/kernel/bpf/lpm_trie.c b/kernel/bpf/lpm_trie.c
index abf1002080df..93a5cbbde421 100644
--- a/kernel/bpf/lpm_trie.c
+++ b/kernel/bpf/lpm_trie.c
@@ -471,6 +471,7 @@ static int trie_delete_elem(struct bpf_map *map, void *_key)
         }
 
         if (!node || node->prefixlen != key->prefixlen ||
+            node->prefixlen != matchlen ||
             (node->flags & LPM_TREE_NODE_FLAG_IM)) {
                 ret = -ENOENT;
                 goto out;
diff --git a/kernel/bpf/map_in_map.c b/kernel/bpf/map_in_map.c
index 99d243e1ad6e..52378d3e34b3 100644
--- a/kernel/bpf/map_in_map.c
+++ b/kernel/bpf/map_in_map.c
@@ -12,6 +12,7 @@
 struct bpf_map *bpf_map_meta_alloc(int inner_map_ufd)
 {
         struct bpf_map *inner_map, *inner_map_meta;
+        u32 inner_map_meta_size;
         struct fd f;
 
         f = fdget(inner_map_ufd);
@@ -36,7 +37,12 @@ struct bpf_map *bpf_map_meta_alloc(int inner_map_ufd)
                 return ERR_PTR(-EINVAL);
         }
 
-        inner_map_meta = kzalloc(sizeof(*inner_map_meta), GFP_USER);
+        inner_map_meta_size = sizeof(*inner_map_meta);
+        /* In some cases verifier needs to access beyond just base map. */
+        if (inner_map->ops == &array_map_ops)
+                inner_map_meta_size = sizeof(struct bpf_array);
+
+        inner_map_meta = kzalloc(inner_map_meta_size, GFP_USER);
         if (!inner_map_meta) {
                 fdput(f);
                 return ERR_PTR(-ENOMEM);
@@ -46,9 +52,16 @@ struct bpf_map *bpf_map_meta_alloc(int inner_map_ufd)
         inner_map_meta->key_size = inner_map->key_size;
         inner_map_meta->value_size = inner_map->value_size;
         inner_map_meta->map_flags = inner_map->map_flags;
-        inner_map_meta->ops = inner_map->ops;
         inner_map_meta->max_entries = inner_map->max_entries;
 
+        /* Misc members not needed in bpf_map_meta_equal() check. */
+        inner_map_meta->ops = inner_map->ops;
+        if (inner_map->ops == &array_map_ops) {
+                inner_map_meta->unpriv_array = inner_map->unpriv_array;
+                container_of(inner_map_meta, struct bpf_array, map)->index_mask =
+                     container_of(inner_map, struct bpf_array, map)->index_mask;
+        }
+
         fdput(f);
         return inner_map_meta;
 }
diff --git a/kernel/bpf/percpu_freelist.c b/kernel/bpf/percpu_freelist.c
index 673fa6fe2d73..0c1b4ba9e90e 100644
--- a/kernel/bpf/percpu_freelist.c
+++ b/kernel/bpf/percpu_freelist.c
@@ -28,8 +28,8 @@ void pcpu_freelist_destroy(struct pcpu_freelist *s)
         free_percpu(s->freelist);
 }
 
-static inline void __pcpu_freelist_push(struct pcpu_freelist_head *head,
-                                        struct pcpu_freelist_node *node)
+static inline void ___pcpu_freelist_push(struct pcpu_freelist_head *head,
+                                         struct pcpu_freelist_node *node)
 {
         raw_spin_lock(&head->lock);
         node->next = head->first;
@@ -37,12 +37,22 @@ static inline void __pcpu_freelist_push(struct pcpu_freelist_head *head,
         raw_spin_unlock(&head->lock);
 }
 
-void pcpu_freelist_push(struct pcpu_freelist *s,
+void __pcpu_freelist_push(struct pcpu_freelist *s,
                         struct pcpu_freelist_node *node)
 {
         struct pcpu_freelist_head *head = this_cpu_ptr(s->freelist);
 
-        __pcpu_freelist_push(head, node);
+        ___pcpu_freelist_push(head, node);
+}
+
+void pcpu_freelist_push(struct pcpu_freelist *s,
+                        struct pcpu_freelist_node *node)
+{
+        unsigned long flags;
+
+        local_irq_save(flags);
+        __pcpu_freelist_push(s, node);
+        local_irq_restore(flags);
 }
 
 void pcpu_freelist_populate(struct pcpu_freelist *s, void *buf, u32 elem_size,
@@ -63,7 +73,7 @@ void pcpu_freelist_populate(struct pcpu_freelist *s, void *buf, u32 elem_size,
         for_each_possible_cpu(cpu) {
 again:
                 head = per_cpu_ptr(s->freelist, cpu);
-                __pcpu_freelist_push(head, buf);
+                ___pcpu_freelist_push(head, buf);
                 i++;
                 buf += elem_size;
                 if (i == nr_elems)
@@ -74,14 +84,12 @@ again:
         local_irq_restore(flags);
 }
 
-struct pcpu_freelist_node *pcpu_freelist_pop(struct pcpu_freelist *s)
+struct pcpu_freelist_node *__pcpu_freelist_pop(struct pcpu_freelist *s)
 {
         struct pcpu_freelist_head *head;
         struct pcpu_freelist_node *node;
-        unsigned long flags;
         int orig_cpu, cpu;
 
-        local_irq_save(flags);
         orig_cpu = cpu = raw_smp_processor_id();
         while (1) {
                 head = per_cpu_ptr(s->freelist, cpu);
@@ -89,16 +97,25 @@ struct pcpu_freelist_node *pcpu_freelist_pop(struct pcpu_freelist *s)
                 node = head->first;
                 if (node) {
                         head->first = node->next;
-                        raw_spin_unlock_irqrestore(&head->lock, flags);
+                        raw_spin_unlock(&head->lock);
                         return node;
                 }
                 raw_spin_unlock(&head->lock);
                 cpu = cpumask_next(cpu, cpu_possible_mask);
                 if (cpu >= nr_cpu_ids)
                         cpu = 0;
-                if (cpu == orig_cpu) {
-                        local_irq_restore(flags);
+                if (cpu == orig_cpu)
                         return NULL;
-                }
         }
 }
+
+struct pcpu_freelist_node *pcpu_freelist_pop(struct pcpu_freelist *s)
+{
+        struct pcpu_freelist_node *ret;
+        unsigned long flags;
+
+        local_irq_save(flags);
+        ret = __pcpu_freelist_pop(s);
+        local_irq_restore(flags);
+        return ret;
+}
diff --git a/kernel/bpf/percpu_freelist.h b/kernel/bpf/percpu_freelist.h
index 3049aae8ea1e..c3960118e617 100644
--- a/kernel/bpf/percpu_freelist.h
+++ b/kernel/bpf/percpu_freelist.h
@@ -22,8 +22,12 @@ struct pcpu_freelist_node {
         struct pcpu_freelist_node *next;
 };
 
+/* pcpu_freelist_* do spin_lock_irqsave. */
 void pcpu_freelist_push(struct pcpu_freelist *, struct pcpu_freelist_node *);
 struct pcpu_freelist_node *pcpu_freelist_pop(struct pcpu_freelist *);
+/* __pcpu_freelist_* do spin_lock only. caller must disable irqs. */
+void __pcpu_freelist_push(struct pcpu_freelist *, struct pcpu_freelist_node *);
+struct pcpu_freelist_node *__pcpu_freelist_pop(struct pcpu_freelist *);
 void pcpu_freelist_populate(struct pcpu_freelist *s, void *buf, u32 elem_size,
                             u32 nr_elems);
 int pcpu_freelist_init(struct pcpu_freelist *);
diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c
index 90daf285de03..950ab2f28922 100644
--- a/kernel/bpf/stackmap.c
+++ b/kernel/bpf/stackmap.c
@@ -44,7 +44,7 @@ static void do_up_read(struct irq_work *entry)
         struct stack_map_irq_work *work;
 
         work = container_of(entry, struct stack_map_irq_work, irq_work);
-        up_read(work->sem);
+        up_read_non_owner(work->sem);
         work->sem = NULL;
 }
 
@@ -180,11 +180,14 @@ static inline int stack_map_parse_build_id(void *page_addr,
 
                 if (nhdr->n_type == BPF_BUILD_ID &&
                     nhdr->n_namesz == sizeof("GNU") &&
-                    nhdr->n_descsz == BPF_BUILD_ID_SIZE) {
+                    nhdr->n_descsz > 0 &&
+                    nhdr->n_descsz <= BPF_BUILD_ID_SIZE) {
                         memcpy(build_id,
                                note_start + note_offs +
                                ALIGN(sizeof("GNU"), 4) + sizeof(Elf32_Nhdr),
-                               BPF_BUILD_ID_SIZE);
+                               nhdr->n_descsz);
+                        memset(build_id + nhdr->n_descsz, 0,
+                               BPF_BUILD_ID_SIZE - nhdr->n_descsz);
                         return 0;
                 }
                 new_offs = note_offs + sizeof(Elf32_Nhdr) +
@@ -260,7 +263,7 @@ static int stack_map_get_build_id(struct vm_area_struct *vma,
                 return -EFAULT; /* page not mapped */
 
         ret = -EINVAL;
-        page_addr = page_address(page);
+        page_addr = kmap_atomic(page);
         ehdr = (Elf32_Ehdr *)page_addr;
 
         /* compare magic x7f "ELF" */
@@ -276,6 +279,7 @@ static int stack_map_get_build_id(struct vm_area_struct *vma,
         else if (ehdr->e_ident[EI_CLASS] == ELFCLASS64)
                 ret = stack_map_get_build_id_64(page_addr, build_id);
 out:
+        kunmap_atomic(page_addr);
         put_page(page);
         return ret;
 }
@@ -310,6 +314,7 @@ static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs,
                 for (i = 0; i < trace_nr; i++) {
                         id_offs[i].status = BPF_STACK_BUILD_ID_IP;
                         id_offs[i].ip = ips[i];
+                        memset(id_offs[i].build_id, 0, BPF_BUILD_ID_SIZE);
                 }
                 return;
         }
@@ -320,6 +325,7 @@ static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs,
                         /* per entry fall back to ips */
                         id_offs[i].status = BPF_STACK_BUILD_ID_IP;
                         id_offs[i].ip = ips[i];
+                        memset(id_offs[i].build_id, 0, BPF_BUILD_ID_SIZE);
                         continue;
                 }
                 id_offs[i].offset = (vma->vm_pgoff << PAGE_SHIFT) + ips[i]
@@ -332,6 +338,12 @@ static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs,
         } else {
                 work->sem = &current->mm->mmap_sem;
                 irq_work_queue(&work->irq_work);
+                /*
+                 * The irq_work will release the mmap_sem with
+                 * up_read_non_owner(). The rwsem_release() is called
+                 * here to release the lock from lockdep's perspective.
+                 */
+                rwsem_release(&current->mm->mmap_sem.dep_map, 1, _RET_IP_);
         }
 }
 
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index b155cd17c1bd..8577bb7f8be6 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -713,8 +713,13 @@ static int map_lookup_elem(union bpf_attr *attr)
 
         if (bpf_map_is_dev_bound(map)) {
                 err = bpf_map_offload_lookup_elem(map, key, value);
-        } else if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
-                   map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
+                goto done;
+        }
+
+        preempt_disable();
+        this_cpu_inc(bpf_prog_active);
+        if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
+            map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
                 err = bpf_percpu_hash_copy(map, key, value);
         } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
                 err = bpf_percpu_array_copy(map, key, value);
@@ -744,7 +749,10 @@ static int map_lookup_elem(union bpf_attr *attr)
                 }
                 rcu_read_unlock();
         }
+        this_cpu_dec(bpf_prog_active);
+        preempt_enable();
 
+done:
         if (err)
                 goto free_value;
 
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index f6bc62a9ee8e..8f295b790297 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -1617,12 +1617,13 @@ static int check_flow_keys_access(struct bpf_verifier_env *env, int off,
         return 0;
 }
 
-static int check_sock_access(struct bpf_verifier_env *env, u32 regno, int off,
-                             int size, enum bpf_access_type t)
+static int check_sock_access(struct bpf_verifier_env *env, int insn_idx,
+                             u32 regno, int off, int size,
+                             enum bpf_access_type t)
 {
         struct bpf_reg_state *regs = cur_regs(env);
         struct bpf_reg_state *reg = &regs[regno];
-        struct bpf_insn_access_aux info;
+        struct bpf_insn_access_aux info = {};
 
         if (reg->smin_value < 0) {
                 verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n",
@@ -1636,6 +1637,8 @@ static int check_sock_access(struct bpf_verifier_env *env, u32 regno, int off,
                 return -EACCES;
         }
 
+        env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size;
+
         return 0;
 }
 
@@ -2032,7 +2035,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn
                         verbose(env, "cannot write into socket\n");
                         return -EACCES;
                 }
-                err = check_sock_access(env, regno, off, size, t);
+                err = check_sock_access(env, insn_idx, regno, off, size, t);
                 if (!err && value_regno >= 0)
                         mark_reg_unknown(env, regs, value_regno);
         } else {
@@ -3103,6 +3106,40 @@ static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg,
         }
 }
 
+static bool can_skip_alu_sanitation(const struct bpf_verifier_env *env,
+                                    const struct bpf_insn *insn)
+{
+        return env->allow_ptr_leaks || BPF_SRC(insn->code) == BPF_K;
+}
+
+static int update_alu_sanitation_state(struct bpf_insn_aux_data *aux,
+                                       u32 alu_state, u32 alu_limit)
+{
+        /* If we arrived here from different branches with different
+         * state or limits to sanitize, then this won't work.
+         */
+        if (aux->alu_state &&
+            (aux->alu_state != alu_state ||
+             aux->alu_limit != alu_limit))
+                return -EACCES;
+
+        /* Corresponding fixup done in fixup_bpf_calls(). */
+        aux->alu_state = alu_state;
+        aux->alu_limit = alu_limit;
+        return 0;
+}
+
+static int sanitize_val_alu(struct bpf_verifier_env *env,
+                            struct bpf_insn *insn)
+{
+        struct bpf_insn_aux_data *aux = cur_aux(env);
+
+        if (can_skip_alu_sanitation(env, insn))
+                return 0;
+
+        return update_alu_sanitation_state(aux, BPF_ALU_NON_POINTER, 0);
+}
+
 static int sanitize_ptr_alu(struct bpf_verifier_env *env,
                             struct bpf_insn *insn,
                             const struct bpf_reg_state *ptr_reg,
@@ -3117,7 +3154,7 @@ static int sanitize_ptr_alu(struct bpf_verifier_env *env,
         struct bpf_reg_state tmp;
         bool ret;
 
-        if (env->allow_ptr_leaks || BPF_SRC(insn->code) == BPF_K)
+        if (can_skip_alu_sanitation(env, insn))
                 return 0;
 
         /* We already marked aux for masking from non-speculative
@@ -3133,19 +3170,8 @@ static int sanitize_ptr_alu(struct bpf_verifier_env *env,
 
         if (retrieve_ptr_limit(ptr_reg, &alu_limit, opcode, off_is_neg))
                 return 0;
-
-        /* If we arrived here from different branches with different
-         * limits to sanitize, then this won't work.
-         */
-        if (aux->alu_state &&
-            (aux->alu_state != alu_state ||
-             aux->alu_limit != alu_limit))
+        if (update_alu_sanitation_state(aux, alu_state, alu_limit))
                 return -EACCES;
-
-        /* Corresponding fixup done in fixup_bpf_calls(). */
-        aux->alu_state = alu_state;
-        aux->alu_limit = alu_limit;
-
 do_sim:
         /* Simulate and find potential out-of-bounds access under
          * speculative execution from truncation as a result of
@@ -3418,6 +3444,8 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env,
         s64 smin_val, smax_val;
         u64 umin_val, umax_val;
         u64 insn_bitness = (BPF_CLASS(insn->code) == BPF_ALU64) ? 64 : 32;
+        u32 dst = insn->dst_reg;
+        int ret;
 
         if (insn_bitness == 32) {
                 /* Relevant for 32-bit RSH: Information can propagate towards
@@ -3452,6 +3480,11 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env,
 
         switch (opcode) {
         case BPF_ADD:
+                ret = sanitize_val_alu(env, insn);
+                if (ret < 0) {
+                        verbose(env, "R%d tried to add from different pointers or scalars\n", dst);
+                        return ret;
+                }
                 if (signed_add_overflows(dst_reg->smin_value, smin_val) ||
                     signed_add_overflows(dst_reg->smax_value, smax_val)) {
                         dst_reg->smin_value = S64_MIN;
@@ -3471,6 +3504,11 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env,
                 dst_reg->var_off = tnum_add(dst_reg->var_off, src_reg.var_off);
                 break;
         case BPF_SUB:
+                ret = sanitize_val_alu(env, insn);
+                if (ret < 0) {
+                        verbose(env, "R%d tried to sub from different pointers or scalars\n", dst);
+                        return ret;
+                }
                 if (signed_sub_overflows(dst_reg->smin_value, smax_val) ||
                     signed_sub_overflows(dst_reg->smax_value, smin_val)) {
                         /* Overflow possible, we know nothing */