1 files changed, 105 insertions, 13 deletions

diff --git a/mm/slab_common.c b/mm/slab_common.c
index a8e76d79ee65..3f3cd97d3fdf 100644
--- a/mm/slab_common.c
+++ b/mm/slab_common.c
@@ -18,6 +18,7 @@
 #include <asm/cacheflush.h>
 #include <asm/tlbflush.h>
 #include <asm/page.h>
+#include <linux/memcontrol.h>
 
 #include "slab.h"
 
@@ -27,7 +28,8 @@ DEFINE_MUTEX(slab_mutex);
 struct kmem_cache *kmem_cache;
 
 #ifdef CONFIG_DEBUG_VM
-static int kmem_cache_sanity_check(const char *name, size_t size)
+static int kmem_cache_sanity_check(struct mem_cgroup *memcg, const char *name,
+                                   size_t size)
 {
         struct kmem_cache *s = NULL;
 
@@ -53,7 +55,13 @@ static int kmem_cache_sanity_check(const char *name, size_t size)
                         continue;
                 }
 
-                if (!strcmp(s->name, name)) {
+                /*
+                 * For simplicity, we won't check this in the list of memcg
+                 * caches. We have control over memcg naming, and if there
+                 * aren't duplicates in the global list, there won't be any
+                 * duplicates in the memcg lists as well.
+                 */
+                if (!memcg && !strcmp(s->name, name)) {
                         pr_err("%s (%s): Cache name already exists.\n",
                                __func__, name);
                         dump_stack();
@@ -66,12 +74,41 @@ static int kmem_cache_sanity_check(const char *name, size_t size)
         return 0;
 }
 #else
-static inline int kmem_cache_sanity_check(const char *name, size_t size)
+static inline int kmem_cache_sanity_check(struct mem_cgroup *memcg,
+                                          const char *name, size_t size)
 {
         return 0;
 }
 #endif
 
+#ifdef CONFIG_MEMCG_KMEM
+int memcg_update_all_caches(int num_memcgs)
+{
+        struct kmem_cache *s;
+        int ret = 0;
+        mutex_lock(&slab_mutex);
+
+        list_for_each_entry(s, &slab_caches, list) {
+                if (!is_root_cache(s))
+                        continue;
+
+                ret = memcg_update_cache_size(s, num_memcgs);
+                /*
+                 * See comment in memcontrol.c, memcg_update_cache_size:
+                 * Instead of freeing the memory, we'll just leave the caches
+                 * up to this point in an updated state.
+                 */
+                if (ret)
+                        goto out;
+        }
+
+        memcg_update_array_size(num_memcgs);
+out:
+        mutex_unlock(&slab_mutex);
+        return ret;
+}
+#endif
+
 /*
  * Figure out what the alignment of the objects will be given a set of
  * flags, a user specified alignment and the size of the objects.
@@ -125,8 +162,10 @@ unsigned long calculate_alignment(unsigned long flags,
  * as davem.
  */
 
-struct kmem_cache *kmem_cache_create(const char *name, size_t size, size_t align,
-                unsigned long flags, void (*ctor)(void *))
+struct kmem_cache *
+kmem_cache_create_memcg(struct mem_cgroup *memcg, const char *name, size_t size,
+                        size_t align, unsigned long flags, void (*ctor)(void *),
+                        struct kmem_cache *parent_cache)
 {
         struct kmem_cache *s = NULL;
         int err = 0;
@@ -134,7 +173,7 @@ struct kmem_cache *kmem_cache_create(const char *name, size_t size, size_t align
         get_online_cpus();
         mutex_lock(&slab_mutex);
 
-        if (!kmem_cache_sanity_check(name, size) == 0)
+        if (!kmem_cache_sanity_check(memcg, name, size) == 0)
                 goto out_locked;
 
         /*
@@ -145,7 +184,7 @@ struct kmem_cache *kmem_cache_create(const char *name, size_t size, size_t align
          */
         flags &= CACHE_CREATE_MASK;
 
-        s = __kmem_cache_alias(name, size, align, flags, ctor);
+        s = __kmem_cache_alias(memcg, name, size, align, flags, ctor);
         if (s)
                 goto out_locked;
 
@@ -154,6 +193,13 @@ struct kmem_cache *kmem_cache_create(const char *name, size_t size, size_t align
                 s->object_size = s->size = size;
                 s->align = calculate_alignment(flags, align, size);
                 s->ctor = ctor;
+
+                if (memcg_register_cache(memcg, s, parent_cache)) {
+                        kmem_cache_free(kmem_cache, s);
+                        err = -ENOMEM;
+                        goto out_locked;
+                }
+
                 s->name = kstrdup(name, GFP_KERNEL);
                 if (!s->name) {
                         kmem_cache_free(kmem_cache, s);
@@ -163,10 +209,9 @@ struct kmem_cache *kmem_cache_create(const char *name, size_t size, size_t align
 
                 err = __kmem_cache_create(s, flags);
                 if (!err) {
-
                         s->refcount = 1;
                         list_add(&s->list, &slab_caches);
-
+                        memcg_cache_list_add(memcg, s);
                 } else {
                         kfree(s->name);
                         kmem_cache_free(kmem_cache, s);
@@ -194,10 +239,20 @@ out_locked:
 
         return s;
 }
+
+struct kmem_cache *
+kmem_cache_create(const char *name, size_t size, size_t align,
+                  unsigned long flags, void (*ctor)(void *))
+{
+        return kmem_cache_create_memcg(NULL, name, size, align, flags, ctor, NULL);
+}
 EXPORT_SYMBOL(kmem_cache_create);
 
 void kmem_cache_destroy(struct kmem_cache *s)
 {
+        /* Destroy all the children caches if we aren't a memcg cache */
+        kmem_cache_destroy_memcg_children(s);
+
         get_online_cpus();
         mutex_lock(&slab_mutex);
         s->refcount--;
@@ -209,6 +264,7 @@ void kmem_cache_destroy(struct kmem_cache *s)
                         if (s->flags & SLAB_DESTROY_BY_RCU)
                                 rcu_barrier();
 
+                        memcg_release_cache(s);
                         kfree(s->name);
                         kmem_cache_free(kmem_cache, s);
                 } else {
@@ -267,7 +323,7 @@ struct kmem_cache *__init create_kmalloc_cache(const char *name, size_t size,
 
 
 #ifdef CONFIG_SLABINFO
-static void print_slabinfo_header(struct seq_file *m)
+void print_slabinfo_header(struct seq_file *m)
 {
         /*
          * Output format version, so at least we can change it
@@ -311,16 +367,43 @@ static void s_stop(struct seq_file *m, void *p)
         mutex_unlock(&slab_mutex);
 }
 
-static int s_show(struct seq_file *m, void *p)
+static void
+memcg_accumulate_slabinfo(struct kmem_cache *s, struct slabinfo *info)
+{
+        struct kmem_cache *c;
+        struct slabinfo sinfo;
+        int i;
+
+        if (!is_root_cache(s))
+                return;
+
+        for_each_memcg_cache_index(i) {
+                c = cache_from_memcg(s, i);
+                if (!c)
+                        continue;
+
+                memset(&sinfo, 0, sizeof(sinfo));
+                get_slabinfo(c, &sinfo);
+
+                info->active_slabs += sinfo.active_slabs;
+                info->num_slabs += sinfo.num_slabs;
+                info->shared_avail += sinfo.shared_avail;
+                info->active_objs += sinfo.active_objs;
+                info->num_objs += sinfo.num_objs;
+        }
+}
+
+int cache_show(struct kmem_cache *s, struct seq_file *m)
 {
-        struct kmem_cache *s = list_entry(p, struct kmem_cache, list);
         struct slabinfo sinfo;
 
         memset(&sinfo, 0, sizeof(sinfo));
         get_slabinfo(s, &sinfo);
 
+        memcg_accumulate_slabinfo(s, &sinfo);
+
         seq_printf(m, "%-17s %6lu %6lu %6u %4u %4d",
-                   s->name, sinfo.active_objs, sinfo.num_objs, s->size,
+                   cache_name(s), sinfo.active_objs, sinfo.num_objs, s->size,
                    sinfo.objects_per_slab, (1 << sinfo.cache_order));
 
         seq_printf(m, " : tunables %4u %4u %4u",
@@ -332,6 +415,15 @@ static int s_show(struct seq_file *m, void *p)
         return 0;
 }
 
+static int s_show(struct seq_file *m, void *p)
+{
+        struct kmem_cache *s = list_entry(p, struct kmem_cache, list);
+
+        if (!is_root_cache(s))
+                return 0;
+        return cache_show(s, m);
+}
+
 /*
  * slabinfo_op - iterator that generates /proc/slabinfo
  *