6 files changed, 48 insertions, 18 deletions

diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c
index 7019c1f05cab..bd3921b1514b 100644
--- a/kernel/bpf/btf.c
+++ b/kernel/bpf/btf.c
@@ -1530,7 +1530,8 @@ static int btf_modifier_resolve(struct btf_verifier_env *env,
 
                 /* "typedef void new_void", "const void"...etc */
                 if (!btf_type_is_void(next_type) &&
-                    !btf_type_is_fwd(next_type)) {
+                    !btf_type_is_fwd(next_type) &&
+                    !btf_type_is_func_proto(next_type)) {
                         btf_verifier_log_type(env, v->t, "Invalid type_id");
                         return -EINVAL;
                 }
diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c
index d78cfec5807d..4e807973aa80 100644
--- a/kernel/bpf/cgroup.c
+++ b/kernel/bpf/cgroup.c
@@ -573,7 +573,7 @@ int __cgroup_bpf_run_filter_skb(struct sock *sk,
         bpf_compute_and_save_data_end(skb, &saved_data_end);
 
         ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], skb,
-                                 bpf_prog_run_save_cb);
+                                 __bpf_prog_run_save_cb);
         bpf_restore_data_end(skb, saved_data_end);
         __skb_pull(skb, offset);
         skb->sk = save_sk;
diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c
index 937776531998..fed15cf94dca 100644
--- a/kernel/bpf/hashtab.c
+++ b/kernel/bpf/hashtab.c
@@ -686,7 +686,7 @@ static void free_htab_elem(struct bpf_htab *htab, struct htab_elem *l)
         }
 
         if (htab_is_prealloc(htab)) {
-                pcpu_freelist_push(&htab->freelist, &l->fnode);
+                __pcpu_freelist_push(&htab->freelist, &l->fnode);
         } else {
                 atomic_dec(&htab->count);
                 l->htab = htab;
@@ -739,7 +739,7 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key,
                 } else {
                         struct pcpu_freelist_node *l;
 
-                        l = pcpu_freelist_pop(&htab->freelist);
+                        l = __pcpu_freelist_pop(&htab->freelist);
                         if (!l)
                                 return ERR_PTR(-E2BIG);
                         l_new = container_of(l, struct htab_elem, fnode);
diff --git a/kernel/bpf/percpu_freelist.c b/kernel/bpf/percpu_freelist.c
index 673fa6fe2d73..0c1b4ba9e90e 100644
--- a/kernel/bpf/percpu_freelist.c
+++ b/kernel/bpf/percpu_freelist.c
@@ -28,8 +28,8 @@ void pcpu_freelist_destroy(struct pcpu_freelist *s)
         free_percpu(s->freelist);
 }
 
-static inline void __pcpu_freelist_push(struct pcpu_freelist_head *head,
-                                        struct pcpu_freelist_node *node)
+static inline void ___pcpu_freelist_push(struct pcpu_freelist_head *head,
+                                         struct pcpu_freelist_node *node)
 {
         raw_spin_lock(&head->lock);
         node->next = head->first;
@@ -37,12 +37,22 @@ static inline void __pcpu_freelist_push(struct pcpu_freelist_head *head,
         raw_spin_unlock(&head->lock);
 }
 
-void pcpu_freelist_push(struct pcpu_freelist *s,
+void __pcpu_freelist_push(struct pcpu_freelist *s,
                         struct pcpu_freelist_node *node)
 {
         struct pcpu_freelist_head *head = this_cpu_ptr(s->freelist);
 
-        __pcpu_freelist_push(head, node);
+        ___pcpu_freelist_push(head, node);
+}
+
+void pcpu_freelist_push(struct pcpu_freelist *s,
+                        struct pcpu_freelist_node *node)
+{
+        unsigned long flags;
+
+        local_irq_save(flags);
+        __pcpu_freelist_push(s, node);
+        local_irq_restore(flags);
 }
 
 void pcpu_freelist_populate(struct pcpu_freelist *s, void *buf, u32 elem_size,
@@ -63,7 +73,7 @@ void pcpu_freelist_populate(struct pcpu_freelist *s, void *buf, u32 elem_size,
         for_each_possible_cpu(cpu) {
 again:
                 head = per_cpu_ptr(s->freelist, cpu);
-                __pcpu_freelist_push(head, buf);
+                ___pcpu_freelist_push(head, buf);
                 i++;
                 buf += elem_size;
                 if (i == nr_elems)
@@ -74,14 +84,12 @@ again:
         local_irq_restore(flags);
 }
 
-struct pcpu_freelist_node *pcpu_freelist_pop(struct pcpu_freelist *s)
+struct pcpu_freelist_node *__pcpu_freelist_pop(struct pcpu_freelist *s)
 {
         struct pcpu_freelist_head *head;
         struct pcpu_freelist_node *node;
-        unsigned long flags;
         int orig_cpu, cpu;
 
-        local_irq_save(flags);
         orig_cpu = cpu = raw_smp_processor_id();
         while (1) {
                 head = per_cpu_ptr(s->freelist, cpu);
@@ -89,16 +97,25 @@ struct pcpu_freelist_node *pcpu_freelist_pop(struct pcpu_freelist *s)
                 node = head->first;
                 if (node) {
                         head->first = node->next;
-                        raw_spin_unlock_irqrestore(&head->lock, flags);
+                        raw_spin_unlock(&head->lock);
                         return node;
                 }
                 raw_spin_unlock(&head->lock);
                 cpu = cpumask_next(cpu, cpu_possible_mask);
                 if (cpu >= nr_cpu_ids)
                         cpu = 0;
-                if (cpu == orig_cpu) {
-                        local_irq_restore(flags);
+                if (cpu == orig_cpu)
                         return NULL;
-                }
         }
 }
+
+struct pcpu_freelist_node *pcpu_freelist_pop(struct pcpu_freelist *s)
+{
+        struct pcpu_freelist_node *ret;
+        unsigned long flags;
+
+        local_irq_save(flags);
+        ret = __pcpu_freelist_pop(s);
+        local_irq_restore(flags);
+        return ret;
+}
diff --git a/kernel/bpf/percpu_freelist.h b/kernel/bpf/percpu_freelist.h
index 3049aae8ea1e..c3960118e617 100644
--- a/kernel/bpf/percpu_freelist.h
+++ b/kernel/bpf/percpu_freelist.h
@@ -22,8 +22,12 @@ struct pcpu_freelist_node {
         struct pcpu_freelist_node *next;
 };
 
+/* pcpu_freelist_* do spin_lock_irqsave. */
 void pcpu_freelist_push(struct pcpu_freelist *, struct pcpu_freelist_node *);
 struct pcpu_freelist_node *pcpu_freelist_pop(struct pcpu_freelist *);
+/* __pcpu_freelist_* do spin_lock only. caller must disable irqs. */
+void __pcpu_freelist_push(struct pcpu_freelist *, struct pcpu_freelist_node *);
+struct pcpu_freelist_node *__pcpu_freelist_pop(struct pcpu_freelist *);
 void pcpu_freelist_populate(struct pcpu_freelist *s, void *buf, u32 elem_size,
                             u32 nr_elems);
 int pcpu_freelist_init(struct pcpu_freelist *);
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 0834958f1dc4..ec7c552af76b 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -740,8 +740,13 @@ static int map_lookup_elem(union bpf_attr *attr)
 
         if (bpf_map_is_dev_bound(map)) {
                 err = bpf_map_offload_lookup_elem(map, key, value);
-        } else if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
-                   map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
+                goto done;
+        }
+
+        preempt_disable();
+        this_cpu_inc(bpf_prog_active);
+        if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
+            map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
                 err = bpf_percpu_hash_copy(map, key, value);
         } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
                 err = bpf_percpu_array_copy(map, key, value);
@@ -777,7 +782,10 @@ static int map_lookup_elem(union bpf_attr *attr)
                 }
                 rcu_read_unlock();
         }
+        this_cpu_dec(bpf_prog_active);
+        preempt_enable();
 
+done:
         if (err)
                 goto free_value;