bpf: extend cgroup bpf core to allow multiple cgroup storage types

In order to introduce per-cpu cgroup storage, let's generalize
bpf cgroup core to support multiple cgroup storage types.
Potentially, per-node cgroup storage can be added later.

This commit is mostly a formal change that replaces
cgroup_storage pointer with a array of cgroup_storage pointers.
It doesn't actually introduce a new storage type,
it will be done later.

Each bpf program is now able to have one cgroup storage of each type.

Signed-off-by: Roman Gushchin <guro@fb.com>
Acked-by: Song Liu <songliubraving@fb.com>
Cc: Daniel Borkmann <daniel@iogearbox.net>
Cc: Alexei Starovoitov <ast@kernel.org>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>

5 files changed, 85 insertions, 39 deletions

diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c
index 549f6fbcc461..00f6ed2e4f9a 100644
--- a/kernel/bpf/cgroup.c
+++ b/kernel/bpf/cgroup.c
@@ -25,6 +25,7 @@ EXPORT_SYMBOL(cgroup_bpf_enabled_key);
  */
 void cgroup_bpf_put(struct cgroup *cgrp)
 {
+        enum bpf_cgroup_storage_type stype;
         unsigned int type;
 
         for (type = 0; type < ARRAY_SIZE(cgrp->bpf.progs); type++) {
@@ -34,8 +35,10 @@ void cgroup_bpf_put(struct cgroup *cgrp)
                 list_for_each_entry_safe(pl, tmp, progs, node) {
                         list_del(&pl->node);
                         bpf_prog_put(pl->prog);
-                        bpf_cgroup_storage_unlink(pl->storage);
-                        bpf_cgroup_storage_free(pl->storage);
+                        for_each_cgroup_storage_type(stype) {
+                                bpf_cgroup_storage_unlink(pl->storage[stype]);
+                                bpf_cgroup_storage_free(pl->storage[stype]);
+                        }
                         kfree(pl);
                         static_branch_dec(&cgroup_bpf_enabled_key);
                 }
@@ -97,6 +100,7 @@ static int compute_effective_progs(struct cgroup *cgrp,
                                    enum bpf_attach_type type,
                                    struct bpf_prog_array __rcu **array)
 {
+        enum bpf_cgroup_storage_type stype;
         struct bpf_prog_array *progs;
         struct bpf_prog_list *pl;
         struct cgroup *p = cgrp;
@@ -125,7 +129,9 @@ static int compute_effective_progs(struct cgroup *cgrp,
                                 continue;
 
                         progs->items[cnt].prog = pl->prog;
-                        progs->items[cnt].cgroup_storage = pl->storage;
+                        for_each_cgroup_storage_type(stype)
+                                progs->items[cnt].cgroup_storage[stype] =
+                                        pl->storage[stype];
                         cnt++;
                 }
         } while ((p = cgroup_parent(p)));
@@ -232,7 +238,9 @@ int __cgroup_bpf_attach(struct cgroup *cgrp, struct bpf_prog *prog,
 {
         struct list_head *progs = &cgrp->bpf.progs[type];
         struct bpf_prog *old_prog = NULL;
-        struct bpf_cgroup_storage *storage, *old_storage = NULL;
+        struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE],
+                *old_storage[MAX_BPF_CGROUP_STORAGE_TYPE] = {NULL};
+        enum bpf_cgroup_storage_type stype;
         struct bpf_prog_list *pl;
         bool pl_was_allocated;
         int err;
@@ -254,34 +262,44 @@ int __cgroup_bpf_attach(struct cgroup *cgrp, struct bpf_prog *prog,
         if (prog_list_length(progs) >= BPF_CGROUP_MAX_PROGS)
                 return -E2BIG;
 
-        storage = bpf_cgroup_storage_alloc(prog);
-        if (IS_ERR(storage))
-                return -ENOMEM;
+        for_each_cgroup_storage_type(stype) {
+                storage[stype] = bpf_cgroup_storage_alloc(prog, stype);
+                if (IS_ERR(storage[stype])) {
+                        storage[stype] = NULL;
+                        for_each_cgroup_storage_type(stype)
+                                bpf_cgroup_storage_free(storage[stype]);
+                        return -ENOMEM;
+                }
+        }
 
         if (flags & BPF_F_ALLOW_MULTI) {
                 list_for_each_entry(pl, progs, node) {
                         if (pl->prog == prog) {
                                 /* disallow attaching the same prog twice */
-                                bpf_cgroup_storage_free(storage);
+                                for_each_cgroup_storage_type(stype)
+                                        bpf_cgroup_storage_free(storage[stype]);
                                 return -EINVAL;
                         }
                 }
 
                 pl = kmalloc(sizeof(*pl), GFP_KERNEL);
                 if (!pl) {
-                        bpf_cgroup_storage_free(storage);
+                        for_each_cgroup_storage_type(stype)
+                                bpf_cgroup_storage_free(storage[stype]);
                         return -ENOMEM;
                 }
 
                 pl_was_allocated = true;
                 pl->prog = prog;
-                pl->storage = storage;
+                for_each_cgroup_storage_type(stype)
+                        pl->storage[stype] = storage[stype];
                 list_add_tail(&pl->node, progs);
         } else {
                 if (list_empty(progs)) {
                         pl = kmalloc(sizeof(*pl), GFP_KERNEL);
                         if (!pl) {
-                                bpf_cgroup_storage_free(storage);
+                                for_each_cgroup_storage_type(stype)
+                                        bpf_cgroup_storage_free(storage[stype]);
                                 return -ENOMEM;
                         }
                         pl_was_allocated = true;
@@ -289,12 +307,15 @@ int __cgroup_bpf_attach(struct cgroup *cgrp, struct bpf_prog *prog,
                 } else {
                         pl = list_first_entry(progs, typeof(*pl), node);
                         old_prog = pl->prog;
-                        old_storage = pl->storage;
-                        bpf_cgroup_storage_unlink(old_storage);
+                        for_each_cgroup_storage_type(stype) {
+                                old_storage[stype] = pl->storage[stype];
+                                bpf_cgroup_storage_unlink(old_storage[stype]);
+                        }
                         pl_was_allocated = false;
                 }
                 pl->prog = prog;
-                pl->storage = storage;
+                for_each_cgroup_storage_type(stype)
+                        pl->storage[stype] = storage[stype];
         }
 
         cgrp->bpf.flags[type] = flags;
@@ -304,21 +325,27 @@ int __cgroup_bpf_attach(struct cgroup *cgrp, struct bpf_prog *prog,
                 goto cleanup;
 
         static_branch_inc(&cgroup_bpf_enabled_key);
-        if (old_storage)
-                bpf_cgroup_storage_free(old_storage);
+        for_each_cgroup_storage_type(stype) {
+                if (!old_storage[stype])
+                        continue;
+                bpf_cgroup_storage_free(old_storage[stype]);
+        }
         if (old_prog) {
                 bpf_prog_put(old_prog);
                 static_branch_dec(&cgroup_bpf_enabled_key);
         }
-        bpf_cgroup_storage_link(storage, cgrp, type);
+        for_each_cgroup_storage_type(stype)
+                bpf_cgroup_storage_link(storage[stype], cgrp, type);
         return 0;
 
 cleanup:
         /* and cleanup the prog list */
         pl->prog = old_prog;
-        bpf_cgroup_storage_free(pl->storage);
-        pl->storage = old_storage;
-        bpf_cgroup_storage_link(old_storage, cgrp, type);
+        for_each_cgroup_storage_type(stype) {
+                bpf_cgroup_storage_free(pl->storage[stype]);
+                pl->storage[stype] = old_storage[stype];
+                bpf_cgroup_storage_link(old_storage[stype], cgrp, type);
+        }
         if (pl_was_allocated) {
                 list_del(&pl->node);
                 kfree(pl);
@@ -339,6 +366,7 @@ int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,
                         enum bpf_attach_type type, u32 unused_flags)
 {
         struct list_head *progs = &cgrp->bpf.progs[type];
+        enum bpf_cgroup_storage_type stype;
         u32 flags = cgrp->bpf.flags[type];
         struct bpf_prog *old_prog = NULL;
         struct bpf_prog_list *pl;
@@ -385,8 +413,10 @@ int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,
 
         /* now can actually delete it from this cgroup list */
         list_del(&pl->node);
-        bpf_cgroup_storage_unlink(pl->storage);
-        bpf_cgroup_storage_free(pl->storage);
+        for_each_cgroup_storage_type(stype) {
+                bpf_cgroup_storage_unlink(pl->storage[stype]);
+                bpf_cgroup_storage_free(pl->storage[stype]);
+        }
         kfree(pl);
         if (list_empty(progs))
                 /* last program was detached, reset flags to zero */
diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c
index 1991466b8327..9070b2ace6aa 100644
--- a/kernel/bpf/helpers.c
+++ b/kernel/bpf/helpers.c
@@ -194,16 +194,18 @@ const struct bpf_func_proto bpf_get_current_cgroup_id_proto = {
         .ret_type       = RET_INTEGER,
 };
 
-DECLARE_PER_CPU(void*, bpf_cgroup_storage);
+#ifdef CONFIG_CGROUP_BPF
+DECLARE_PER_CPU(void*, bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]);
 
 BPF_CALL_2(bpf_get_local_storage, struct bpf_map *, map, u64, flags)
 {
-        /* map and flags arguments are not used now,
-         * but provide an ability to extend the API
-         * for other types of local storages.
-         * verifier checks that their values are correct.
+        /* flags argument is not used now,
+         * but provides an ability to extend the API.
+         * verifier checks that its value is correct.
          */
-        return (unsigned long) this_cpu_read(bpf_cgroup_storage);
+        enum bpf_cgroup_storage_type stype = cgroup_storage_type(map);
+
+        return (unsigned long) this_cpu_read(bpf_cgroup_storage[stype]);
 }
 
 const struct bpf_func_proto bpf_get_local_storage_proto = {
@@ -214,3 +216,4 @@ const struct bpf_func_proto bpf_get_local_storage_proto = {
         .arg2_type      = ARG_ANYTHING,
 };
 #endif
+#endif
diff --git a/kernel/bpf/local_storage.c b/kernel/bpf/local_storage.c
index 22ad967d1e5f..0bd9f19fc557 100644
--- a/kernel/bpf/local_storage.c
+++ b/kernel/bpf/local_storage.c
@@ -7,7 +7,7 @@
 #include <linux/rbtree.h>
 #include <linux/slab.h>
 
-DEFINE_PER_CPU(void*, bpf_cgroup_storage);
+DEFINE_PER_CPU(void*, bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]);
 
 #ifdef CONFIG_CGROUP_BPF
 
@@ -251,6 +251,7 @@ const struct bpf_map_ops cgroup_storage_map_ops = {
 
 int bpf_cgroup_storage_assign(struct bpf_prog *prog, struct bpf_map *_map)
 {
+        enum bpf_cgroup_storage_type stype = cgroup_storage_type(_map);
         struct bpf_cgroup_storage_map *map = map_to_storage(_map);
         int ret = -EBUSY;
 
@@ -258,11 +259,12 @@ int bpf_cgroup_storage_assign(struct bpf_prog *prog, struct bpf_map *_map)
 
         if (map->prog && map->prog != prog)
                 goto unlock;
-        if (prog->aux->cgroup_storage && prog->aux->cgroup_storage != _map)
+        if (prog->aux->cgroup_storage[stype] &&
+            prog->aux->cgroup_storage[stype] != _map)
                 goto unlock;
 
         map->prog = prog;
-        prog->aux->cgroup_storage = _map;
+        prog->aux->cgroup_storage[stype] = _map;
         ret = 0;
 unlock:
         spin_unlock_bh(&map->lock);
@@ -272,24 +274,26 @@ unlock:
 
 void bpf_cgroup_storage_release(struct bpf_prog *prog, struct bpf_map *_map)
 {
+        enum bpf_cgroup_storage_type stype = cgroup_storage_type(_map);
         struct bpf_cgroup_storage_map *map = map_to_storage(_map);
 
         spin_lock_bh(&map->lock);
         if (map->prog == prog) {
-                WARN_ON(prog->aux->cgroup_storage != _map);
+                WARN_ON(prog->aux->cgroup_storage[stype] != _map);
                 map->prog = NULL;
-                prog->aux->cgroup_storage = NULL;
+                prog->aux->cgroup_storage[stype] = NULL;
         }
         spin_unlock_bh(&map->lock);
 }
 
-struct bpf_cgroup_storage *bpf_cgroup_storage_alloc(struct bpf_prog *prog)
+struct bpf_cgroup_storage *bpf_cgroup_storage_alloc(struct bpf_prog *prog,
+                                        enum bpf_cgroup_storage_type stype)
 {
         struct bpf_cgroup_storage *storage;
         struct bpf_map *map;
         u32 pages;
 
-        map = prog->aux->cgroup_storage;
+        map = prog->aux->cgroup_storage[stype];
         if (!map)
                 return NULL;
 
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index b3c2d09bcf7a..8c91d2b41b1e 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -988,10 +988,15 @@ static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog)
 /* drop refcnt on maps used by eBPF program and free auxilary data */
 static void free_used_maps(struct bpf_prog_aux *aux)
 {
+        enum bpf_cgroup_storage_type stype;
         int i;
 
-        if (aux->cgroup_storage)
-                bpf_cgroup_storage_release(aux->prog, aux->cgroup_storage);
+        for_each_cgroup_storage_type(stype) {
+                if (!aux->cgroup_storage[stype])
+                        continue;
+                bpf_cgroup_storage_release(aux->prog,
+                                           aux->cgroup_storage[stype]);
+        }
 
         for (i = 0; i < aux->used_map_cnt; i++)
                 bpf_map_put(aux->used_maps[i]);
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index e986518d7bc3..e90899df585d 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -5171,11 +5171,15 @@ next_insn:
 /* drop refcnt of maps used by the rejected program */
 static void release_maps(struct bpf_verifier_env *env)
 {
+        enum bpf_cgroup_storage_type stype;
         int i;
 
-        if (env->prog->aux->cgroup_storage)
+        for_each_cgroup_storage_type(stype) {
+                if (!env->prog->aux->cgroup_storage[stype])
+                        continue;
                 bpf_cgroup_storage_release(env->prog,
-                                           env->prog->aux->cgroup_storage);
+                        env->prog->aux->cgroup_storage[stype]);
+        }
 
         for (i = 0; i < env->used_map_cnt; i++)
                 bpf_map_put(env->used_maps[i]);