4 files changed, 53 insertions, 25 deletions

diff --git a/include/linux/slab.h b/include/linux/slab.h
index a0cc7a77cda2..af1a5bef80f4 100644
--- a/include/linux/slab.h
+++ b/include/linux/slab.h
@@ -556,6 +556,8 @@ struct memcg_cache_array {
  *              used to index child cachces during allocation and cleared
  *              early during shutdown.
  *
+ * @root_caches_node: List node for slab_root_caches list.
+ *
  * @children:   List of all child caches.  While the child caches are also
  *              reachable through @memcg_caches, a child cache remains on
  *              this list until it is actually destroyed.
@@ -573,6 +575,7 @@ struct memcg_cache_params {
         union {
                 struct {
                         struct memcg_cache_array __rcu *memcg_caches;
+                        struct list_head __root_caches_node;
                         struct list_head children;
                 };
                 struct {
diff --git a/mm/slab.h b/mm/slab.h
index a08f01016a3f..9631bb27c772 100644
--- a/mm/slab.h
+++ b/mm/slab.h
@@ -201,6 +201,11 @@ void __kmem_cache_free_bulk(struct kmem_cache *, size_t, void **);
 int __kmem_cache_alloc_bulk(struct kmem_cache *, gfp_t, size_t, void **);
 
 #if defined(CONFIG_MEMCG) && !defined(CONFIG_SLOB)
+
+/* List of all root caches. */
+extern struct list_head         slab_root_caches;
+#define root_caches_node        memcg_params.__root_caches_node
+
 /*
  * Iterate over all memcg caches of the given root cache. The caller must hold
  * slab_mutex.
@@ -300,9 +305,14 @@ static __always_inline void memcg_uncharge_slab(struct page *page, int order,
 }
 
 extern void slab_init_memcg_params(struct kmem_cache *);
+extern void memcg_link_cache(struct kmem_cache *s);
 
 #else /* CONFIG_MEMCG && !CONFIG_SLOB */
 
+/* If !memcg, all caches are root. */
+#define slab_root_caches        slab_caches
+#define root_caches_node        list
+
 #define for_each_memcg_cache(iter, root) \
         for ((void)(iter), (void)(root); 0; )
 
@@ -347,6 +357,11 @@ static inline void memcg_uncharge_slab(struct page *page, int order,
 static inline void slab_init_memcg_params(struct kmem_cache *s)
 {
 }
+
+static inline void memcg_link_cache(struct kmem_cache *s)
+{
+}
+
 #endif /* CONFIG_MEMCG && !CONFIG_SLOB */
 
 static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x)
diff --git a/mm/slab_common.c b/mm/slab_common.c
index c3bbeddaeaaf..274697e1a42a 100644
--- a/mm/slab_common.c
+++ b/mm/slab_common.c
@@ -138,6 +138,9 @@ int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t nr,
 }
 
 #if defined(CONFIG_MEMCG) && !defined(CONFIG_SLOB)
+
+LIST_HEAD(slab_root_caches);
+
 void slab_init_memcg_params(struct kmem_cache *s)
 {
         s->memcg_params.root_cache = NULL;
@@ -183,9 +186,6 @@ static int update_memcg_params(struct kmem_cache *s, int new_array_size)
 {
         struct memcg_cache_array *old, *new;
 
-        if (!is_root_cache(s))
-                return 0;
-
         new = kzalloc(sizeof(struct memcg_cache_array) +
                       new_array_size * sizeof(void *), GFP_KERNEL);
         if (!new)
@@ -209,7 +209,7 @@ int memcg_update_all_caches(int num_memcgs)
         int ret = 0;
 
         mutex_lock(&slab_mutex);
-        list_for_each_entry(s, &slab_caches, list) {
+        list_for_each_entry(s, &slab_root_caches, root_caches_node) {
                 ret = update_memcg_params(s, num_memcgs);
                 /*
                  * Instead of freeing the memory, we'll just leave the caches
@@ -222,10 +222,26 @@ int memcg_update_all_caches(int num_memcgs)
         return ret;
 }
 
-static void unlink_memcg_cache(struct kmem_cache *s)
+void memcg_link_cache(struct kmem_cache *s)
+{
+        if (is_root_cache(s)) {
+                list_add(&s->root_caches_node, &slab_root_caches);
+        } else {
+                list_add(&s->memcg_params.children_node,
+                         &s->memcg_params.root_cache->memcg_params.children);
+                list_add(&s->memcg_params.kmem_caches_node,
+                         &s->memcg_params.memcg->kmem_caches);
+        }
+}
+
+static void memcg_unlink_cache(struct kmem_cache *s)
 {
-        list_del(&s->memcg_params.children_node);
-        list_del(&s->memcg_params.kmem_caches_node);
+        if (is_root_cache(s)) {
+                list_del(&s->root_caches_node);
+        } else {
+                list_del(&s->memcg_params.children_node);
+                list_del(&s->memcg_params.kmem_caches_node);
+        }
 }
 #else
 static inline int init_memcg_params(struct kmem_cache *s,
@@ -238,7 +254,7 @@ static inline void destroy_memcg_params(struct kmem_cache *s)
 {
 }
 
-static inline void unlink_memcg_cache(struct kmem_cache *s)
+static inline void memcg_unlink_cache(struct kmem_cache *s)
 {
 }
 #endif /* CONFIG_MEMCG && !CONFIG_SLOB */
@@ -285,7 +301,7 @@ struct kmem_cache *find_mergeable(size_t size, size_t align,
         if (flags & SLAB_NEVER_MERGE)
                 return NULL;
 
-        list_for_each_entry_reverse(s, &slab_caches, list) {
+        list_for_each_entry_reverse(s, &slab_root_caches, root_caches_node) {
                 if (slab_unmergeable(s))
                         continue;
 
@@ -369,6 +385,7 @@ static struct kmem_cache *create_cache(const char *name,
 
         s->refcount = 1;
         list_add(&s->list, &slab_caches);
+        memcg_link_cache(s);
 out:
         if (err)
                 return ERR_PTR(err);
@@ -514,9 +531,8 @@ static int shutdown_cache(struct kmem_cache *s)
         if (__kmem_cache_shutdown(s) != 0)
                 return -EBUSY;
 
+        memcg_unlink_cache(s);
         list_del(&s->list);
-        if (!is_root_cache(s))
-                unlink_memcg_cache(s);
 
         if (s->flags & SLAB_DESTROY_BY_RCU) {
                 list_add_tail(&s->list, &slab_caches_to_rcu_destroy);
@@ -596,10 +612,6 @@ void memcg_create_kmem_cache(struct mem_cgroup *memcg,
                 goto out_unlock;
         }
 
-        list_add(&s->memcg_params.children_node,
-                 &root_cache->memcg_params.children);
-        list_add(&s->memcg_params.kmem_caches_node, &memcg->kmem_caches);
-
         /*
          * Since readers won't lock (see cache_from_memcg_idx()), we need a
          * barrier here to ensure nobody will see the kmem_cache partially
@@ -627,10 +639,7 @@ void memcg_deactivate_kmem_caches(struct mem_cgroup *memcg)
         get_online_mems();
 
         mutex_lock(&slab_mutex);
-        list_for_each_entry(s, &slab_caches, list) {
-                if (!is_root_cache(s))
-                        continue;
-
+        list_for_each_entry(s, &slab_root_caches, root_caches_node) {
                 arr = rcu_dereference_protected(s->memcg_params.memcg_caches,
                                                 lockdep_is_held(&slab_mutex));
                 c = arr->entries[idx];
@@ -829,6 +838,7 @@ struct kmem_cache *__init create_kmalloc_cache(const char *name, size_t size,
 
         create_boot_cache(s, name, size, flags);
         list_add(&s->list, &slab_caches);
+        memcg_link_cache(s);
         s->refcount = 1;
         return s;
 }
@@ -1136,12 +1146,12 @@ static void print_slabinfo_header(struct seq_file *m)
 void *slab_start(struct seq_file *m, loff_t *pos)
 {
         mutex_lock(&slab_mutex);
-        return seq_list_start(&slab_caches, *pos);
+        return seq_list_start(&slab_root_caches, *pos);
 }
 
 void *slab_next(struct seq_file *m, void *p, loff_t *pos)
 {
-        return seq_list_next(p, &slab_caches, pos);
+        return seq_list_next(p, &slab_root_caches, pos);
 }
 
 void slab_stop(struct seq_file *m, void *p)
@@ -1193,12 +1203,11 @@ static void cache_show(struct kmem_cache *s, struct seq_file *m)
 
 static int slab_show(struct seq_file *m, void *p)
 {
-        struct kmem_cache *s = list_entry(p, struct kmem_cache, list);
+        struct kmem_cache *s = list_entry(p, struct kmem_cache, root_caches_node);
 
-        if (p == slab_caches.next)
+        if (p == slab_root_caches.next)
                 print_slabinfo_header(m);
-        if (is_root_cache(s))
-                cache_show(s, m);
+        cache_show(s, m);
         return 0;
 }
 
diff --git a/mm/slub.c b/mm/slub.c
index caac5456f0ec..03b012bcb5fa 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -4127,6 +4127,7 @@ static struct kmem_cache * __init bootstrap(struct kmem_cache *static_cache)
         }
         slab_init_memcg_params(s);
         list_add(&s->list, &slab_caches);
+        memcg_link_cache(s);
         return s;
 }