1 files changed, 154 insertions, 96 deletions

diff --git a/mm/slab_common.c b/mm/slab_common.c
index 1ec3c619ba04..f3cfccf76dda 100644
--- a/mm/slab_common.c
+++ b/mm/slab_common.c
@@ -29,8 +29,7 @@ DEFINE_MUTEX(slab_mutex);
 struct kmem_cache *kmem_cache;
 
 #ifdef CONFIG_DEBUG_VM
-static int kmem_cache_sanity_check(struct mem_cgroup *memcg, const char *name,
-                                   size_t size)
+static int kmem_cache_sanity_check(const char *name, size_t size)
 {
         struct kmem_cache *s = NULL;
 
@@ -57,13 +56,7 @@ static int kmem_cache_sanity_check(struct mem_cgroup *memcg, const char *name,
                 }
 
 #if !defined(CONFIG_SLUB) || !defined(CONFIG_SLUB_DEBUG_ON)
-                /*
-                 * For simplicity, we won't check this in the list of memcg
-                 * caches. We have control over memcg naming, and if there
-                 * aren't duplicates in the global list, there won't be any
-                 * duplicates in the memcg lists as well.
-                 */
-                if (!memcg && !strcmp(s->name, name)) {
+                if (!strcmp(s->name, name)) {
                         pr_err("%s (%s): Cache name already exists.\n",
                                __func__, name);
                         dump_stack();
@@ -77,8 +70,7 @@ static int kmem_cache_sanity_check(struct mem_cgroup *memcg, const char *name,
         return 0;
 }
 #else
-static inline int kmem_cache_sanity_check(struct mem_cgroup *memcg,
-                                          const char *name, size_t size)
+static inline int kmem_cache_sanity_check(const char *name, size_t size)
 {
         return 0;
 }
@@ -139,6 +131,46 @@ unsigned long calculate_alignment(unsigned long flags,
         return ALIGN(align, sizeof(void *));
 }
 
+static struct kmem_cache *
+do_kmem_cache_create(char *name, size_t object_size, size_t size, size_t align,
+                     unsigned long flags, void (*ctor)(void *),
+                     struct mem_cgroup *memcg, struct kmem_cache *root_cache)
+{
+        struct kmem_cache *s;
+        int err;
+
+        err = -ENOMEM;
+        s = kmem_cache_zalloc(kmem_cache, GFP_KERNEL);
+        if (!s)
+                goto out;
+
+        s->name = name;
+        s->object_size = object_size;
+        s->size = size;
+        s->align = align;
+        s->ctor = ctor;
+
+        err = memcg_alloc_cache_params(memcg, s, root_cache);
+        if (err)
+                goto out_free_cache;
+
+        err = __kmem_cache_create(s, flags);
+        if (err)
+                goto out_free_cache;
+
+        s->refcount = 1;
+        list_add(&s->list, &slab_caches);
+        memcg_register_cache(s);
+out:
+        if (err)
+                return ERR_PTR(err);
+        return s;
+
+out_free_cache:
+        memcg_free_cache_params(s);
+        kfree(s);
+        goto out;
+}
 
 /*
  * kmem_cache_create - Create a cache.
@@ -164,34 +196,21 @@ unsigned long calculate_alignment(unsigned long flags,
  * cacheline.  This can be beneficial if you're counting cycles as closely
  * as davem.
  */
-
 struct kmem_cache *
-kmem_cache_create_memcg(struct mem_cgroup *memcg, const char *name, size_t size,
-                        size_t align, unsigned long flags, void (*ctor)(void *),
-                        struct kmem_cache *parent_cache)
+kmem_cache_create(const char *name, size_t size, size_t align,
+                  unsigned long flags, void (*ctor)(void *))
 {
-        struct kmem_cache *s = NULL;
+        struct kmem_cache *s;
+        char *cache_name;
         int err;
 
         get_online_cpus();
         mutex_lock(&slab_mutex);
 
-        err = kmem_cache_sanity_check(memcg, name, size);
+        err = kmem_cache_sanity_check(name, size);
         if (err)
                 goto out_unlock;
 
-        if (memcg) {
-                /*
-                 * Since per-memcg caches are created asynchronously on first
-                 * allocation (see memcg_kmem_get_cache()), several threads can
-                 * try to create the same cache, but only one of them may
-                 * succeed. Therefore if we get here and see the cache has
-                 * already been created, we silently return NULL.
-                 */
-                if (cache_from_memcg_idx(parent_cache, memcg_cache_id(memcg)))
-                        goto out_unlock;
-        }
-
         /*
          * Some allocators will constraint the set of valid flags to a subset
          * of all flags. We expect them to define CACHE_CREATE_MASK in this
@@ -200,50 +219,29 @@ kmem_cache_create_memcg(struct mem_cgroup *memcg, const char *name, size_t size,
          */
         flags &= CACHE_CREATE_MASK;
 
-        s = __kmem_cache_alias(memcg, name, size, align, flags, ctor);
+        s = __kmem_cache_alias(name, size, align, flags, ctor);
         if (s)
                 goto out_unlock;
 
-        err = -ENOMEM;
-        s = kmem_cache_zalloc(kmem_cache, GFP_KERNEL);
-        if (!s)
+        cache_name = kstrdup(name, GFP_KERNEL);
+        if (!cache_name) {
+                err = -ENOMEM;
                 goto out_unlock;
+        }
 
-        s->object_size = s->size = size;
-        s->align = calculate_alignment(flags, align, size);
-        s->ctor = ctor;
-
-        s->name = kstrdup(name, GFP_KERNEL);
-        if (!s->name)
-                goto out_free_cache;
-
-        err = memcg_alloc_cache_params(memcg, s, parent_cache);
-        if (err)
-                goto out_free_cache;
-
-        err = __kmem_cache_create(s, flags);
-        if (err)
-                goto out_free_cache;
-
-        s->refcount = 1;
-        list_add(&s->list, &slab_caches);
-        memcg_register_cache(s);
+        s = do_kmem_cache_create(cache_name, size, size,
+                                 calculate_alignment(flags, align, size),
+                                 flags, ctor, NULL, NULL);
+        if (IS_ERR(s)) {
+                err = PTR_ERR(s);
+                kfree(cache_name);
+        }
 
 out_unlock:
         mutex_unlock(&slab_mutex);
         put_online_cpus();
 
         if (err) {
-                /*
-                 * There is no point in flooding logs with warnings or
-                 * especially crashing the system if we fail to create a cache
-                 * for a memcg. In this case we will be accounting the memcg
-                 * allocation to the root cgroup until we succeed to create its
-                 * own cache, but it isn't that critical.
-                 */
-                if (!memcg)
-                        return NULL;
-
                 if (flags & SLAB_PANIC)
                         panic("kmem_cache_create: Failed to create slab '%s'. Error %d\n",
                                 name, err);
@@ -255,52 +253,112 @@ out_unlock:
                 return NULL;
         }
         return s;
+}
+EXPORT_SYMBOL(kmem_cache_create);
 
-out_free_cache:
-        memcg_free_cache_params(s);
-        kfree(s->name);
-        kmem_cache_free(kmem_cache, s);
-        goto out_unlock;
+#ifdef CONFIG_MEMCG_KMEM
+/*
+ * kmem_cache_create_memcg - Create a cache for a memory cgroup.
+ * @memcg: The memory cgroup the new cache is for.
+ * @root_cache: The parent of the new cache.
+ *
+ * This function attempts to create a kmem cache that will serve allocation
+ * requests going from @memcg to @root_cache. The new cache inherits properties
+ * from its parent.
+ */
+void kmem_cache_create_memcg(struct mem_cgroup *memcg, struct kmem_cache *root_cache)
+{
+        struct kmem_cache *s;
+        char *cache_name;
+
+        get_online_cpus();
+        mutex_lock(&slab_mutex);
+
+        /*
+         * Since per-memcg caches are created asynchronously on first
+         * allocation (see memcg_kmem_get_cache()), several threads can try to
+         * create the same cache, but only one of them may succeed.
+         */
+        if (cache_from_memcg_idx(root_cache, memcg_cache_id(memcg)))
+                goto out_unlock;
+
+        cache_name = memcg_create_cache_name(memcg, root_cache);
+        if (!cache_name)
+                goto out_unlock;
+
+        s = do_kmem_cache_create(cache_name, root_cache->object_size,
+                                 root_cache->size, root_cache->align,
+                                 root_cache->flags, root_cache->ctor,
+                                 memcg, root_cache);
+        if (IS_ERR(s)) {
+                kfree(cache_name);
+                goto out_unlock;
+        }
+
+        s->allocflags |= __GFP_KMEMCG;
+
+out_unlock:
+        mutex_unlock(&slab_mutex);
+        put_online_cpus();
 }
 
-struct kmem_cache *
-kmem_cache_create(const char *name, size_t size, size_t align,
-                  unsigned long flags, void (*ctor)(void *))
+static int kmem_cache_destroy_memcg_children(struct kmem_cache *s)
 {
-        return kmem_cache_create_memcg(NULL, name, size, align, flags, ctor, NULL);
+        int rc;
+
+        if (!s->memcg_params ||
+            !s->memcg_params->is_root_cache)
+                return 0;
+
+        mutex_unlock(&slab_mutex);
+        rc = __kmem_cache_destroy_memcg_children(s);
+        mutex_lock(&slab_mutex);
+
+        return rc;
 }
-EXPORT_SYMBOL(kmem_cache_create);
+#else
+static int kmem_cache_destroy_memcg_children(struct kmem_cache *s)
+{
+        return 0;
+}
+#endif /* CONFIG_MEMCG_KMEM */
 
 void kmem_cache_destroy(struct kmem_cache *s)
 {
-        /* Destroy all the children caches if we aren't a memcg cache */
-        kmem_cache_destroy_memcg_children(s);
-
         get_online_cpus();
         mutex_lock(&slab_mutex);
+
         s->refcount--;
-        if (!s->refcount) {
-                list_del(&s->list);
-
-                if (!__kmem_cache_shutdown(s)) {
-                        memcg_unregister_cache(s);
-                        mutex_unlock(&slab_mutex);
-                        if (s->flags & SLAB_DESTROY_BY_RCU)
-                                rcu_barrier();
-
-                        memcg_free_cache_params(s);
-                        kfree(s->name);
-                        kmem_cache_free(kmem_cache, s);
-                } else {
-                        list_add(&s->list, &slab_caches);
-                        mutex_unlock(&slab_mutex);
-                        printk(KERN_ERR "kmem_cache_destroy %s: Slab cache still has objects\n",
-                                s->name);
-                        dump_stack();
-                }
-        } else {
-                mutex_unlock(&slab_mutex);
+        if (s->refcount)
+                goto out_unlock;
+
+        if (kmem_cache_destroy_memcg_children(s) != 0)
+                goto out_unlock;
+
+        list_del(&s->list);
+        memcg_unregister_cache(s);
+
+        if (__kmem_cache_shutdown(s) != 0) {
+                list_add(&s->list, &slab_caches);
+                memcg_register_cache(s);
+                printk(KERN_ERR "kmem_cache_destroy %s: "
+                       "Slab cache still has objects\n", s->name);
+                dump_stack();
+                goto out_unlock;
         }
+
+        mutex_unlock(&slab_mutex);
+        if (s->flags & SLAB_DESTROY_BY_RCU)
+                rcu_barrier();
+
+        memcg_free_cache_params(s);
+        kfree(s->name);
+        kmem_cache_free(kmem_cache, s);
+        goto out_put_cpus;
+
+out_unlock:
+        mutex_unlock(&slab_mutex);
+out_put_cpus:
         put_online_cpus();
 }
 EXPORT_SYMBOL(kmem_cache_destroy);