memcg: free memcg_caches slot on css offline

We need to look up a kmem_cache in ->memcg_params.memcg_caches arrays only
on allocations, so there is no need to have the array entries set until
css free - we can clear them on css offline.  This will allow us to reuse
array entries more efficiently and avoid costly array relocations.

Signed-off-by: Vladimir Davydov <vdavydov@parallels.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Michal Hocko <mhocko@suse.cz>
Cc: Tejun Heo <tj@kernel.org>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Dave Chinner <david@fromorbit.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

1 files changed, 28 insertions, 11 deletions

diff --git a/mm/slab_common.c b/mm/slab_common.c
index 6087b1f9a385..0873bcc61c7a 100644
--- a/mm/slab_common.c
+++ b/mm/slab_common.c
@@ -440,18 +440,8 @@ static int do_kmem_cache_shutdown(struct kmem_cache *s,
                 *need_rcu_barrier = true;
 
 #ifdef CONFIG_MEMCG_KMEM
-        if (!is_root_cache(s)) {
-                int idx;
-                struct memcg_cache_array *arr;
-
-                idx = memcg_cache_id(s->memcg_params.memcg);
-                arr = rcu_dereference_protected(s->memcg_params.root_cache->
-                                                memcg_params.memcg_caches,
-                                                lockdep_is_held(&slab_mutex));
-                BUG_ON(arr->entries[idx] != s);
-                arr->entries[idx] = NULL;
+        if (!is_root_cache(s))
                 list_del(&s->memcg_params.list);
-        }
 #endif
         list_move(&s->list, release);
         return 0;
@@ -499,6 +489,13 @@ void memcg_create_kmem_cache(struct mem_cgroup *memcg,
 
         mutex_lock(&slab_mutex);
 
+        /*
+         * The memory cgroup could have been deactivated while the cache
+         * creation work was pending.
+         */
+        if (!memcg_kmem_is_active(memcg))
+                goto out_unlock;
+
         idx = memcg_cache_id(memcg);
         arr = rcu_dereference_protected(root_cache->memcg_params.memcg_caches,
                                         lockdep_is_held(&slab_mutex));
@@ -548,6 +545,26 @@ out_unlock:
         put_online_cpus();
 }
 
+void memcg_deactivate_kmem_caches(struct mem_cgroup *memcg)
+{
+        int idx;
+        struct memcg_cache_array *arr;
+        struct kmem_cache *s;
+
+        idx = memcg_cache_id(memcg);
+
+        mutex_lock(&slab_mutex);
+        list_for_each_entry(s, &slab_caches, list) {
+                if (!is_root_cache(s))
+                        continue;
+
+                arr = rcu_dereference_protected(s->memcg_params.memcg_caches,
+                                                lockdep_is_held(&slab_mutex));
+                arr->entries[idx] = NULL;
+        }
+        mutex_unlock(&slab_mutex);
+}
+
 void memcg_destroy_kmem_caches(struct mem_cgroup *memcg)
 {
         LIST_HEAD(release);