list_lru: introduce per-memcg lists

There are several FS shrinkers, including super_block::s_shrink, that
keep reclaimable objects in the list_lru structure.  Hence to turn them
to memcg-aware shrinkers, it is enough to make list_lru per-memcg.

This patch does the trick.  It adds an array of lru lists to the
list_lru_node structure (per-node part of the list_lru), one for each
kmem-active memcg, and dispatches every item addition or removal to the
list corresponding to the memcg which the item is accounted to.  So now
the list_lru structure is not just per node, but per node and per memcg.

Not all list_lrus need this feature, so this patch also adds a new
method, list_lru_init_memcg, which initializes a list_lru as memcg
aware.  Otherwise (i.e.  if initialized with old list_lru_init), the
list_lru won't have per memcg lists.

Just like per memcg caches arrays, the arrays of per-memcg lists are
indexed by memcg_cache_id, so we must grow them whenever
memcg_nr_cache_ids is increased.  So we introduce a callback,
memcg_update_all_list_lrus, invoked by memcg_alloc_cache_id if the id
space is full.

The locking is implemented in a manner similar to lruvecs, i.e.  we have
one lock per node that protects all lists (both global and per cgroup) on
the node.

Signed-off-by: Vladimir Davydov <vdavydov@parallels.com>
Cc: Dave Chinner <david@fromorbit.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Michal Hocko <mhocko@suse.cz>
Cc: Greg Thelen <gthelen@google.com>
Cc: Glauber Costa <glommer@gmail.com>
Cc: Alexander Viro <viro@zeniv.linux.org.uk>
Cc: Christoph Lameter <cl@linux.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Tejun Heo <tj@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

2 files changed, 370 insertions, 24 deletions

diff --git a/mm/list_lru.c b/mm/list_lru.c
index a9021cb3ccde..79aee70c3b9d 100644
--- a/mm/list_lru.c
+++ b/mm/list_lru.c
@@ -10,6 +10,7 @@
 #include <linux/list_lru.h>
 #include <linux/slab.h>
 #include <linux/mutex.h>
+#include <linux/memcontrol.h>
 
 #ifdef CONFIG_MEMCG_KMEM
 static LIST_HEAD(list_lrus);
@@ -38,16 +39,71 @@ static void list_lru_unregister(struct list_lru *lru)
 }
 #endif /* CONFIG_MEMCG_KMEM */
 
+#ifdef CONFIG_MEMCG_KMEM
+static inline bool list_lru_memcg_aware(struct list_lru *lru)
+{
+        return !!lru->node[0].memcg_lrus;
+}
+
+static inline struct list_lru_one *
+list_lru_from_memcg_idx(struct list_lru_node *nlru, int idx)
+{
+        /*
+         * The lock protects the array of per cgroup lists from relocation
+         * (see memcg_update_list_lru_node).
+         */
+        lockdep_assert_held(&nlru->lock);
+        if (nlru->memcg_lrus && idx >= 0)
+                return nlru->memcg_lrus->lru[idx];
+
+        return &nlru->lru;
+}
+
+static inline struct list_lru_one *
+list_lru_from_kmem(struct list_lru_node *nlru, void *ptr)
+{
+        struct mem_cgroup *memcg;
+
+        if (!nlru->memcg_lrus)
+                return &nlru->lru;
+
+        memcg = mem_cgroup_from_kmem(ptr);
+        if (!memcg)
+                return &nlru->lru;
+
+        return list_lru_from_memcg_idx(nlru, memcg_cache_id(memcg));
+}
+#else
+static inline bool list_lru_memcg_aware(struct list_lru *lru)
+{
+        return false;
+}
+
+static inline struct list_lru_one *
+list_lru_from_memcg_idx(struct list_lru_node *nlru, int idx)
+{
+        return &nlru->lru;
+}
+
+static inline struct list_lru_one *
+list_lru_from_kmem(struct list_lru_node *nlru, void *ptr)
+{
+        return &nlru->lru;
+}
+#endif /* CONFIG_MEMCG_KMEM */
+
 bool list_lru_add(struct list_lru *lru, struct list_head *item)
 {
         int nid = page_to_nid(virt_to_page(item));
         struct list_lru_node *nlru = &lru->node[nid];
+        struct list_lru_one *l;
 
         spin_lock(&nlru->lock);
-        WARN_ON_ONCE(nlru->nr_items < 0);
+        l = list_lru_from_kmem(nlru, item);
+        WARN_ON_ONCE(l->nr_items < 0);
         if (list_empty(item)) {
-                list_add_tail(item, &nlru->list);
-                nlru->nr_items++;
+                list_add_tail(item, &l->list);
+                l->nr_items++;
                 spin_unlock(&nlru->lock);
                 return true;
         }
@@ -60,12 +116,14 @@ bool list_lru_del(struct list_lru *lru, struct list_head *item)
 {
         int nid = page_to_nid(virt_to_page(item));
         struct list_lru_node *nlru = &lru->node[nid];
+        struct list_lru_one *l;
 
         spin_lock(&nlru->lock);
+        l = list_lru_from_kmem(nlru, item);
         if (!list_empty(item)) {
                 list_del_init(item);
-                nlru->nr_items--;
-                WARN_ON_ONCE(nlru->nr_items < 0);
+                l->nr_items--;
+                WARN_ON_ONCE(l->nr_items < 0);
                 spin_unlock(&nlru->lock);
                 return true;
         }
@@ -74,33 +132,58 @@ bool list_lru_del(struct list_lru *lru, struct list_head *item)
 }
 EXPORT_SYMBOL_GPL(list_lru_del);
 
-unsigned long
-list_lru_count_node(struct list_lru *lru, int nid)
+static unsigned long __list_lru_count_one(struct list_lru *lru,
+                                          int nid, int memcg_idx)
 {
-        unsigned long count = 0;
         struct list_lru_node *nlru = &lru->node[nid];
+        struct list_lru_one *l;
+        unsigned long count;
 
         spin_lock(&nlru->lock);
-        WARN_ON_ONCE(nlru->nr_items < 0);
-        count += nlru->nr_items;
+        l = list_lru_from_memcg_idx(nlru, memcg_idx);
+        WARN_ON_ONCE(l->nr_items < 0);
+        count = l->nr_items;
         spin_unlock(&nlru->lock);
 
         return count;
 }
+
+unsigned long list_lru_count_one(struct list_lru *lru,
+                                 int nid, struct mem_cgroup *memcg)
+{
+        return __list_lru_count_one(lru, nid, memcg_cache_id(memcg));
+}
+EXPORT_SYMBOL_GPL(list_lru_count_one);
+
+unsigned long list_lru_count_node(struct list_lru *lru, int nid)
+{
+        long count = 0;
+        int memcg_idx;
+
+        count += __list_lru_count_one(lru, nid, -1);
+        if (list_lru_memcg_aware(lru)) {
+                for_each_memcg_cache_index(memcg_idx)
+                        count += __list_lru_count_one(lru, nid, memcg_idx);
+        }
+        return count;
+}
 EXPORT_SYMBOL_GPL(list_lru_count_node);
 
-unsigned long
-list_lru_walk_node(struct list_lru *lru, int nid, list_lru_walk_cb isolate,
-                   void *cb_arg, unsigned long *nr_to_walk)
+static unsigned long
+__list_lru_walk_one(struct list_lru *lru, int nid, int memcg_idx,
+                    list_lru_walk_cb isolate, void *cb_arg,
+                    unsigned long *nr_to_walk)
 {
 
-        struct list_lru_node    *nlru = &lru->node[nid];
+        struct list_lru_node *nlru = &lru->node[nid];
+        struct list_lru_one *l;
         struct list_head *item, *n;
         unsigned long isolated = 0;
 
         spin_lock(&nlru->lock);
+        l = list_lru_from_memcg_idx(nlru, memcg_idx);
 restart:
-        list_for_each_safe(item, n, &nlru->list) {
+        list_for_each_safe(item, n, &l->list) {
                 enum lru_status ret;
 
                 /*
@@ -116,8 +199,8 @@ restart:
                 case LRU_REMOVED_RETRY:
                         assert_spin_locked(&nlru->lock);
                 case LRU_REMOVED:
-                        nlru->nr_items--;
-                        WARN_ON_ONCE(nlru->nr_items < 0);
+                        l->nr_items--;
+                        WARN_ON_ONCE(l->nr_items < 0);
                         isolated++;
                         /*
                          * If the lru lock has been dropped, our list
@@ -128,7 +211,7 @@ restart:
                                 goto restart;
                         break;
                 case LRU_ROTATE:
-                        list_move_tail(item, &nlru->list);
+                        list_move_tail(item, &l->list);
                         break;
                 case LRU_SKIP:
                         break;
@@ -147,36 +230,279 @@ restart:
         spin_unlock(&nlru->lock);
         return isolated;
 }
+
+unsigned long
+list_lru_walk_one(struct list_lru *lru, int nid, struct mem_cgroup *memcg,
+                  list_lru_walk_cb isolate, void *cb_arg,
+                  unsigned long *nr_to_walk)
+{
+        return __list_lru_walk_one(lru, nid, memcg_cache_id(memcg),
+                                   isolate, cb_arg, nr_to_walk);
+}
+EXPORT_SYMBOL_GPL(list_lru_walk_one);
+
+unsigned long list_lru_walk_node(struct list_lru *lru, int nid,
+                                 list_lru_walk_cb isolate, void *cb_arg,
+                                 unsigned long *nr_to_walk)
+{
+        long isolated = 0;
+        int memcg_idx;
+
+        isolated += __list_lru_walk_one(lru, nid, -1, isolate, cb_arg,
+                                        nr_to_walk);
+        if (*nr_to_walk > 0 && list_lru_memcg_aware(lru)) {
+                for_each_memcg_cache_index(memcg_idx) {
+                        isolated += __list_lru_walk_one(lru, nid, memcg_idx,
+                                                isolate, cb_arg, nr_to_walk);
+                        if (*nr_to_walk <= 0)
+                                break;
+                }
+        }
+        return isolated;
+}
 EXPORT_SYMBOL_GPL(list_lru_walk_node);
 
-int list_lru_init_key(struct list_lru *lru, struct lock_class_key *key)
+static void init_one_lru(struct list_lru_one *l)
+{
+        INIT_LIST_HEAD(&l->list);
+        l->nr_items = 0;
+}
+
+#ifdef CONFIG_MEMCG_KMEM
+static void __memcg_destroy_list_lru_node(struct list_lru_memcg *memcg_lrus,
+                                          int begin, int end)
+{
+        int i;
+
+        for (i = begin; i < end; i++)
+                kfree(memcg_lrus->lru[i]);
+}
+
+static int __memcg_init_list_lru_node(struct list_lru_memcg *memcg_lrus,
+                                      int begin, int end)
+{
+        int i;
+
+        for (i = begin; i < end; i++) {
+                struct list_lru_one *l;
+
+                l = kmalloc(sizeof(struct list_lru_one), GFP_KERNEL);
+                if (!l)
+                        goto fail;
+
+                init_one_lru(l);
+                memcg_lrus->lru[i] = l;
+        }
+        return 0;
+fail:
+        __memcg_destroy_list_lru_node(memcg_lrus, begin, i - 1);
+        return -ENOMEM;
+}
+
+static int memcg_init_list_lru_node(struct list_lru_node *nlru)
+{
+        int size = memcg_nr_cache_ids;
+
+        nlru->memcg_lrus = kmalloc(size * sizeof(void *), GFP_KERNEL);
+        if (!nlru->memcg_lrus)
+                return -ENOMEM;
+
+        if (__memcg_init_list_lru_node(nlru->memcg_lrus, 0, size)) {
+                kfree(nlru->memcg_lrus);
+                return -ENOMEM;
+        }
+
+        return 0;
+}
+
+static void memcg_destroy_list_lru_node(struct list_lru_node *nlru)
+{
+        __memcg_destroy_list_lru_node(nlru->memcg_lrus, 0, memcg_nr_cache_ids);
+        kfree(nlru->memcg_lrus);
+}
+
+static int memcg_update_list_lru_node(struct list_lru_node *nlru,
+                                      int old_size, int new_size)
+{
+        struct list_lru_memcg *old, *new;
+
+        BUG_ON(old_size > new_size);
+
+        old = nlru->memcg_lrus;
+        new = kmalloc(new_size * sizeof(void *), GFP_KERNEL);
+        if (!new)
+                return -ENOMEM;
+
+        if (__memcg_init_list_lru_node(new, old_size, new_size)) {
+                kfree(new);
+                return -ENOMEM;
+        }
+
+        memcpy(new, old, old_size * sizeof(void *));
+
+        /*
+         * The lock guarantees that we won't race with a reader
+         * (see list_lru_from_memcg_idx).
+         *
+         * Since list_lru_{add,del} may be called under an IRQ-safe lock,
+         * we have to use IRQ-safe primitives here to avoid deadlock.
+         */
+        spin_lock_irq(&nlru->lock);
+        nlru->memcg_lrus = new;
+        spin_unlock_irq(&nlru->lock);
+
+        kfree(old);
+        return 0;
+}
+
+static void memcg_cancel_update_list_lru_node(struct list_lru_node *nlru,
+                                              int old_size, int new_size)
+{
+        /* do not bother shrinking the array back to the old size, because we
+         * cannot handle allocation failures here */
+        __memcg_destroy_list_lru_node(nlru->memcg_lrus, old_size, new_size);
+}
+
+static int memcg_init_list_lru(struct list_lru *lru, bool memcg_aware)
+{
+        int i;
+
+        for (i = 0; i < nr_node_ids; i++) {
+                if (!memcg_aware)
+                        lru->node[i].memcg_lrus = NULL;
+                else if (memcg_init_list_lru_node(&lru->node[i]))
+                        goto fail;
+        }
+        return 0;
+fail:
+        for (i = i - 1; i >= 0; i--)
+                memcg_destroy_list_lru_node(&lru->node[i]);
+        return -ENOMEM;
+}
+
+static void memcg_destroy_list_lru(struct list_lru *lru)
+{
+        int i;
+
+        if (!list_lru_memcg_aware(lru))
+                return;
+
+        for (i = 0; i < nr_node_ids; i++)
+                memcg_destroy_list_lru_node(&lru->node[i]);
+}
+
+static int memcg_update_list_lru(struct list_lru *lru,
+                                 int old_size, int new_size)
+{
+        int i;
+
+        if (!list_lru_memcg_aware(lru))
+                return 0;
+
+        for (i = 0; i < nr_node_ids; i++) {
+                if (memcg_update_list_lru_node(&lru->node[i],
+                                               old_size, new_size))
+                        goto fail;
+        }
+        return 0;
+fail:
+        for (i = i - 1; i >= 0; i--)
+                memcg_cancel_update_list_lru_node(&lru->node[i],
+                                                  old_size, new_size);
+        return -ENOMEM;
+}
+
+static void memcg_cancel_update_list_lru(struct list_lru *lru,
+                                         int old_size, int new_size)
+{
+        int i;
+
+        if (!list_lru_memcg_aware(lru))
+                return;
+
+        for (i = 0; i < nr_node_ids; i++)
+                memcg_cancel_update_list_lru_node(&lru->node[i],
+                                                  old_size, new_size);
+}
+
+int memcg_update_all_list_lrus(int new_size)
+{
+        int ret = 0;
+        struct list_lru *lru;
+        int old_size = memcg_nr_cache_ids;
+
+        mutex_lock(&list_lrus_mutex);
+        list_for_each_entry(lru, &list_lrus, list) {
+                ret = memcg_update_list_lru(lru, old_size, new_size);
+                if (ret)
+                        goto fail;
+        }
+out:
+        mutex_unlock(&list_lrus_mutex);
+        return ret;
+fail:
+        list_for_each_entry_continue_reverse(lru, &list_lrus, list)
+                memcg_cancel_update_list_lru(lru, old_size, new_size);
+        goto out;
+}
+#else
+static int memcg_init_list_lru(struct list_lru *lru, bool memcg_aware)
+{
+        return 0;
+}
+
+static void memcg_destroy_list_lru(struct list_lru *lru)
+{
+}
+#endif /* CONFIG_MEMCG_KMEM */
+
+int __list_lru_init(struct list_lru *lru, bool memcg_aware,
+                    struct lock_class_key *key)
 {
         int i;
         size_t size = sizeof(*lru->node) * nr_node_ids;
+        int err = -ENOMEM;
+
+        memcg_get_cache_ids();
 
         lru->node = kzalloc(size, GFP_KERNEL);
         if (!lru->node)
-                return -ENOMEM;
+                goto out;
 
         for (i = 0; i < nr_node_ids; i++) {
                 spin_lock_init(&lru->node[i].lock);
                 if (key)
                         lockdep_set_class(&lru->node[i].lock, key);
-                INIT_LIST_HEAD(&lru->node[i].list);
-                lru->node[i].nr_items = 0;
+                init_one_lru(&lru->node[i].lru);
+        }
+
+        err = memcg_init_list_lru(lru, memcg_aware);
+        if (err) {
+                kfree(lru->node);
+                goto out;
         }
+
         list_lru_register(lru);
-        return 0;
+out:
+        memcg_put_cache_ids();
+        return err;
 }
-EXPORT_SYMBOL_GPL(list_lru_init_key);
+EXPORT_SYMBOL_GPL(__list_lru_init);
 
 void list_lru_destroy(struct list_lru *lru)
 {
         /* Already destroyed or not yet initialized? */
         if (!lru->node)
                 return;
+
+        memcg_get_cache_ids();
+
         list_lru_unregister(lru);
+
+        memcg_destroy_list_lru(lru);
         kfree(lru->node);
         lru->node = NULL;
+
+        memcg_put_cache_ids();
 }
 EXPORT_SYMBOL_GPL(list_lru_destroy);
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 6706e5fa5ac0..afa55bb38cbd 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -2572,6 +2572,8 @@ static int memcg_alloc_cache_id(void)
 
         err = memcg_update_all_caches(size);
         if (!err)
+                err = memcg_update_all_list_lrus(size);
+        if (!err)
                 memcg_nr_cache_ids = size;
 
         up_write(&memcg_cache_ids_sem);
@@ -2765,6 +2767,24 @@ void __memcg_kmem_uncharge_pages(struct page *page, int order)
         memcg_uncharge_kmem(memcg, 1 << order);
         page->mem_cgroup = NULL;
 }
+
+struct mem_cgroup *__mem_cgroup_from_kmem(void *ptr)
+{
+        struct mem_cgroup *memcg = NULL;
+        struct kmem_cache *cachep;
+        struct page *page;
+
+        page = virt_to_head_page(ptr);
+        if (PageSlab(page)) {
+                cachep = page->slab_cache;
+                if (!is_root_cache(cachep))
+                        memcg = cachep->memcg_params->memcg;
+        } else
+                /* page allocated by alloc_kmem_pages */
+                memcg = page->mem_cgroup;
+
+        return memcg;
+}
 #endif /* CONFIG_MEMCG_KMEM */
 
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE