slab/slub: consider a memcg parameter in kmem_create_cache

Allow a memcg parameter to be passed during cache creation.  When the slub
allocator is being used, it will only merge caches that belong to the same
memcg.  We'll do this by scanning the global list, and then translating
the cache to a memcg-specific cache

Default function is created as a wrapper, passing NULL to the memcg
version.  We only merge caches that belong to the same memcg.

A helper is provided, memcg_css_id: because slub needs a unique cache name
for sysfs.  Since this is visible, but not the canonical location for slab
data, the cache name is not used, the css_id should suffice.

Signed-off-by: Glauber Costa <glommer@parallels.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Frederic Weisbecker <fweisbec@redhat.com>
Cc: Greg Thelen <gthelen@google.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: JoonSoo Kim <js1304@gmail.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Michal Hocko <mhocko@suse.cz>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: Rik van Riel <riel@redhat.com>
Cc: Suleiman Souhlal <suleiman@google.com>
Cc: Tejun Heo <tj@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

1 files changed, 33 insertions, 9 deletions

diff --git a/mm/slab_common.c b/mm/slab_common.c
index a8e76d79ee65..3031badcc577 100644
--- a/mm/slab_common.c
+++ b/mm/slab_common.c
@@ -18,6 +18,7 @@
 #include <asm/cacheflush.h>
 #include <asm/tlbflush.h>
 #include <asm/page.h>
+#include <linux/memcontrol.h>
 
 #include "slab.h"
 
@@ -27,7 +28,8 @@ DEFINE_MUTEX(slab_mutex);
 struct kmem_cache *kmem_cache;
 
 #ifdef CONFIG_DEBUG_VM
-static int kmem_cache_sanity_check(const char *name, size_t size)
+static int kmem_cache_sanity_check(struct mem_cgroup *memcg, const char *name,
+                                   size_t size)
 {
         struct kmem_cache *s = NULL;
 
@@ -53,7 +55,13 @@ static int kmem_cache_sanity_check(const char *name, size_t size)
                         continue;
                 }
 
-                if (!strcmp(s->name, name)) {
+                /*
+                 * For simplicity, we won't check this in the list of memcg
+                 * caches. We have control over memcg naming, and if there
+                 * aren't duplicates in the global list, there won't be any
+                 * duplicates in the memcg lists as well.
+                 */
+                if (!memcg && !strcmp(s->name, name)) {
                         pr_err("%s (%s): Cache name already exists.\n",
                                __func__, name);
                         dump_stack();
@@ -66,7 +74,8 @@ static int kmem_cache_sanity_check(const char *name, size_t size)
         return 0;
 }
 #else
-static inline int kmem_cache_sanity_check(const char *name, size_t size)
+static inline int kmem_cache_sanity_check(struct mem_cgroup *memcg,
+                                          const char *name, size_t size)
 {
         return 0;
 }
@@ -125,8 +134,9 @@ unsigned long calculate_alignment(unsigned long flags,
  * as davem.
  */
 
-struct kmem_cache *kmem_cache_create(const char *name, size_t size, size_t align,
-                unsigned long flags, void (*ctor)(void *))
+struct kmem_cache *
+kmem_cache_create_memcg(struct mem_cgroup *memcg, const char *name, size_t size,
+                        size_t align, unsigned long flags, void (*ctor)(void *))
 {
         struct kmem_cache *s = NULL;
         int err = 0;
@@ -134,7 +144,7 @@ struct kmem_cache *kmem_cache_create(const char *name, size_t size, size_t align
         get_online_cpus();
         mutex_lock(&slab_mutex);
 
-        if (!kmem_cache_sanity_check(name, size) == 0)
+        if (!kmem_cache_sanity_check(memcg, name, size) == 0)
                 goto out_locked;
 
         /*
@@ -145,7 +155,7 @@ struct kmem_cache *kmem_cache_create(const char *name, size_t size, size_t align
          */
         flags &= CACHE_CREATE_MASK;
 
-        s = __kmem_cache_alias(name, size, align, flags, ctor);
+        s = __kmem_cache_alias(memcg, name, size, align, flags, ctor);
         if (s)
                 goto out_locked;
 
@@ -154,6 +164,13 @@ struct kmem_cache *kmem_cache_create(const char *name, size_t size, size_t align
                 s->object_size = s->size = size;
                 s->align = calculate_alignment(flags, align, size);
                 s->ctor = ctor;
+
+                if (memcg_register_cache(memcg, s)) {
+                        kmem_cache_free(kmem_cache, s);
+                        err = -ENOMEM;
+                        goto out_locked;
+                }
+
                 s->name = kstrdup(name, GFP_KERNEL);
                 if (!s->name) {
                         kmem_cache_free(kmem_cache, s);
@@ -163,10 +180,9 @@ struct kmem_cache *kmem_cache_create(const char *name, size_t size, size_t align
 
                 err = __kmem_cache_create(s, flags);
                 if (!err) {
-
                         s->refcount = 1;
                         list_add(&s->list, &slab_caches);
-
+                        memcg_cache_list_add(memcg, s);
                 } else {
                         kfree(s->name);
                         kmem_cache_free(kmem_cache, s);
@@ -194,6 +210,13 @@ out_locked:
 
         return s;
 }
+
+struct kmem_cache *
+kmem_cache_create(const char *name, size_t size, size_t align,
+                  unsigned long flags, void (*ctor)(void *))
+{
+        return kmem_cache_create_memcg(NULL, name, size, align, flags, ctor);
+}
 EXPORT_SYMBOL(kmem_cache_create);
 
 void kmem_cache_destroy(struct kmem_cache *s)
@@ -209,6 +232,7 @@ void kmem_cache_destroy(struct kmem_cache *s)
                         if (s->flags & SLAB_DESTROY_BY_RCU)
                                 rcu_barrier();
 
+                        memcg_release_cache(s);
                         kfree(s->name);
                         kmem_cache_free(kmem_cache, s);
                 } else {