1 files changed, 350 insertions, 24 deletions

diff --git a/mm/list_lru.c b/mm/list_lru.c
index a9021cb3ccde..79aee70c3b9d 100644
--- a/mm/list_lru.c
+++ b/mm/list_lru.c
@@ -10,6 +10,7 @@
 #include <linux/list_lru.h>
 #include <linux/slab.h>
 #include <linux/mutex.h>
+#include <linux/memcontrol.h>
 
 #ifdef CONFIG_MEMCG_KMEM
 static LIST_HEAD(list_lrus);
@@ -38,16 +39,71 @@ static void list_lru_unregister(struct list_lru *lru)
 }
 #endif /* CONFIG_MEMCG_KMEM */
 
+#ifdef CONFIG_MEMCG_KMEM
+static inline bool list_lru_memcg_aware(struct list_lru *lru)
+{
+        return !!lru->node[0].memcg_lrus;
+}
+
+static inline struct list_lru_one *
+list_lru_from_memcg_idx(struct list_lru_node *nlru, int idx)
+{
+        /*
+         * The lock protects the array of per cgroup lists from relocation
+         * (see memcg_update_list_lru_node).
+         */
+        lockdep_assert_held(&nlru->lock);
+        if (nlru->memcg_lrus && idx >= 0)
+                return nlru->memcg_lrus->lru[idx];
+
+        return &nlru->lru;
+}
+
+static inline struct list_lru_one *
+list_lru_from_kmem(struct list_lru_node *nlru, void *ptr)
+{
+        struct mem_cgroup *memcg;
+
+        if (!nlru->memcg_lrus)
+                return &nlru->lru;
+
+        memcg = mem_cgroup_from_kmem(ptr);
+        if (!memcg)
+                return &nlru->lru;
+
+        return list_lru_from_memcg_idx(nlru, memcg_cache_id(memcg));
+}
+#else
+static inline bool list_lru_memcg_aware(struct list_lru *lru)
+{
+        return false;
+}
+
+static inline struct list_lru_one *
+list_lru_from_memcg_idx(struct list_lru_node *nlru, int idx)
+{
+        return &nlru->lru;
+}
+
+static inline struct list_lru_one *
+list_lru_from_kmem(struct list_lru_node *nlru, void *ptr)
+{
+        return &nlru->lru;
+}
+#endif /* CONFIG_MEMCG_KMEM */
+
 bool list_lru_add(struct list_lru *lru, struct list_head *item)
 {
         int nid = page_to_nid(virt_to_page(item));
         struct list_lru_node *nlru = &lru->node[nid];
+        struct list_lru_one *l;
 
         spin_lock(&nlru->lock);
-        WARN_ON_ONCE(nlru->nr_items < 0);
+        l = list_lru_from_kmem(nlru, item);
+        WARN_ON_ONCE(l->nr_items < 0);
         if (list_empty(item)) {
-                list_add_tail(item, &nlru->list);
-                nlru->nr_items++;
+                list_add_tail(item, &l->list);
+                l->nr_items++;
                 spin_unlock(&nlru->lock);
                 return true;
         }
@@ -60,12 +116,14 @@ bool list_lru_del(struct list_lru *lru, struct list_head *item)
 {
         int nid = page_to_nid(virt_to_page(item));
         struct list_lru_node *nlru = &lru->node[nid];
+        struct list_lru_one *l;
 
         spin_lock(&nlru->lock);
+        l = list_lru_from_kmem(nlru, item);
         if (!list_empty(item)) {
                 list_del_init(item);
-                nlru->nr_items--;
-                WARN_ON_ONCE(nlru->nr_items < 0);
+                l->nr_items--;
+                WARN_ON_ONCE(l->nr_items < 0);
                 spin_unlock(&nlru->lock);
                 return true;
         }
@@ -74,33 +132,58 @@ bool list_lru_del(struct list_lru *lru, struct list_head *item)
 }
 EXPORT_SYMBOL_GPL(list_lru_del);
 
-unsigned long
-list_lru_count_node(struct list_lru *lru, int nid)
+static unsigned long __list_lru_count_one(struct list_lru *lru,
+                                          int nid, int memcg_idx)
 {
-        unsigned long count = 0;
         struct list_lru_node *nlru = &lru->node[nid];
+        struct list_lru_one *l;
+        unsigned long count;
 
         spin_lock(&nlru->lock);
-        WARN_ON_ONCE(nlru->nr_items < 0);
-        count += nlru->nr_items;
+        l = list_lru_from_memcg_idx(nlru, memcg_idx);
+        WARN_ON_ONCE(l->nr_items < 0);
+        count = l->nr_items;
         spin_unlock(&nlru->lock);
 
         return count;
 }
+
+unsigned long list_lru_count_one(struct list_lru *lru,
+                                 int nid, struct mem_cgroup *memcg)
+{
+        return __list_lru_count_one(lru, nid, memcg_cache_id(memcg));
+}
+EXPORT_SYMBOL_GPL(list_lru_count_one);
+
+unsigned long list_lru_count_node(struct list_lru *lru, int nid)
+{
+        long count = 0;
+        int memcg_idx;
+
+        count += __list_lru_count_one(lru, nid, -1);
+        if (list_lru_memcg_aware(lru)) {
+                for_each_memcg_cache_index(memcg_idx)
+                        count += __list_lru_count_one(lru, nid, memcg_idx);
+        }
+        return count;
+}
 EXPORT_SYMBOL_GPL(list_lru_count_node);
 
-unsigned long
-list_lru_walk_node(struct list_lru *lru, int nid, list_lru_walk_cb isolate,
-                   void *cb_arg, unsigned long *nr_to_walk)
+static unsigned long
+__list_lru_walk_one(struct list_lru *lru, int nid, int memcg_idx,
+                    list_lru_walk_cb isolate, void *cb_arg,
+                    unsigned long *nr_to_walk)
 {
 
-        struct list_lru_node    *nlru = &lru->node[nid];
+        struct list_lru_node *nlru = &lru->node[nid];
+        struct list_lru_one *l;
         struct list_head *item, *n;
         unsigned long isolated = 0;
 
         spin_lock(&nlru->lock);
+        l = list_lru_from_memcg_idx(nlru, memcg_idx);
 restart:
-        list_for_each_safe(item, n, &nlru->list) {
+        list_for_each_safe(item, n, &l->list) {
                 enum lru_status ret;
 
                 /*
@@ -116,8 +199,8 @@ restart:
                 case LRU_REMOVED_RETRY:
                         assert_spin_locked(&nlru->lock);
                 case LRU_REMOVED:
-                        nlru->nr_items--;
-                        WARN_ON_ONCE(nlru->nr_items < 0);
+                        l->nr_items--;
+                        WARN_ON_ONCE(l->nr_items < 0);
                         isolated++;
                         /*
                          * If the lru lock has been dropped, our list
@@ -128,7 +211,7 @@ restart:
                                 goto restart;
                         break;
                 case LRU_ROTATE:
-                        list_move_tail(item, &nlru->list);
+                        list_move_tail(item, &l->list);
                         break;
                 case LRU_SKIP:
                         break;
@@ -147,36 +230,279 @@ restart:
         spin_unlock(&nlru->lock);
         return isolated;
 }
+
+unsigned long
+list_lru_walk_one(struct list_lru *lru, int nid, struct mem_cgroup *memcg,
+                  list_lru_walk_cb isolate, void *cb_arg,
+                  unsigned long *nr_to_walk)
+{
+        return __list_lru_walk_one(lru, nid, memcg_cache_id(memcg),
+                                   isolate, cb_arg, nr_to_walk);
+}
+EXPORT_SYMBOL_GPL(list_lru_walk_one);
+
+unsigned long list_lru_walk_node(struct list_lru *lru, int nid,
+                                 list_lru_walk_cb isolate, void *cb_arg,
+                                 unsigned long *nr_to_walk)
+{
+        long isolated = 0;
+        int memcg_idx;
+
+        isolated += __list_lru_walk_one(lru, nid, -1, isolate, cb_arg,
+                                        nr_to_walk);
+        if (*nr_to_walk > 0 && list_lru_memcg_aware(lru)) {
+                for_each_memcg_cache_index(memcg_idx) {
+                        isolated += __list_lru_walk_one(lru, nid, memcg_idx,
+                                                isolate, cb_arg, nr_to_walk);
+                        if (*nr_to_walk <= 0)
+                                break;
+                }
+        }
+        return isolated;
+}
 EXPORT_SYMBOL_GPL(list_lru_walk_node);
 
-int list_lru_init_key(struct list_lru *lru, struct lock_class_key *key)
+static void init_one_lru(struct list_lru_one *l)
+{
+        INIT_LIST_HEAD(&l->list);
+        l->nr_items = 0;
+}
+
+#ifdef CONFIG_MEMCG_KMEM
+static void __memcg_destroy_list_lru_node(struct list_lru_memcg *memcg_lrus,
+                                          int begin, int end)
+{
+        int i;
+
+        for (i = begin; i < end; i++)
+                kfree(memcg_lrus->lru[i]);
+}
+
+static int __memcg_init_list_lru_node(struct list_lru_memcg *memcg_lrus,
+                                      int begin, int end)
+{
+        int i;
+
+        for (i = begin; i < end; i++) {
+                struct list_lru_one *l;
+
+                l = kmalloc(sizeof(struct list_lru_one), GFP_KERNEL);
+                if (!l)
+                        goto fail;
+
+                init_one_lru(l);
+                memcg_lrus->lru[i] = l;
+        }
+        return 0;
+fail:
+        __memcg_destroy_list_lru_node(memcg_lrus, begin, i - 1);
+        return -ENOMEM;
+}
+
+static int memcg_init_list_lru_node(struct list_lru_node *nlru)
+{
+        int size = memcg_nr_cache_ids;
+
+        nlru->memcg_lrus = kmalloc(size * sizeof(void *), GFP_KERNEL);
+        if (!nlru->memcg_lrus)
+                return -ENOMEM;
+
+        if (__memcg_init_list_lru_node(nlru->memcg_lrus, 0, size)) {
+                kfree(nlru->memcg_lrus);
+                return -ENOMEM;
+        }
+
+        return 0;
+}
+
+static void memcg_destroy_list_lru_node(struct list_lru_node *nlru)
+{
+        __memcg_destroy_list_lru_node(nlru->memcg_lrus, 0, memcg_nr_cache_ids);
+        kfree(nlru->memcg_lrus);
+}
+
+static int memcg_update_list_lru_node(struct list_lru_node *nlru,
+                                      int old_size, int new_size)
+{
+        struct list_lru_memcg *old, *new;
+
+        BUG_ON(old_size > new_size);
+
+        old = nlru->memcg_lrus;
+        new = kmalloc(new_size * sizeof(void *), GFP_KERNEL);
+        if (!new)
+                return -ENOMEM;
+
+        if (__memcg_init_list_lru_node(new, old_size, new_size)) {
+                kfree(new);
+                return -ENOMEM;
+        }
+
+        memcpy(new, old, old_size * sizeof(void *));
+
+        /*
+         * The lock guarantees that we won't race with a reader
+         * (see list_lru_from_memcg_idx).
+         *
+         * Since list_lru_{add,del} may be called under an IRQ-safe lock,
+         * we have to use IRQ-safe primitives here to avoid deadlock.
+         */
+        spin_lock_irq(&nlru->lock);
+        nlru->memcg_lrus = new;
+        spin_unlock_irq(&nlru->lock);
+
+        kfree(old);
+        return 0;
+}
+
+static void memcg_cancel_update_list_lru_node(struct list_lru_node *nlru,
+                                              int old_size, int new_size)
+{
+        /* do not bother shrinking the array back to the old size, because we
+         * cannot handle allocation failures here */
+        __memcg_destroy_list_lru_node(nlru->memcg_lrus, old_size, new_size);
+}
+
+static int memcg_init_list_lru(struct list_lru *lru, bool memcg_aware)
+{
+        int i;
+
+        for (i = 0; i < nr_node_ids; i++) {
+                if (!memcg_aware)
+                        lru->node[i].memcg_lrus = NULL;
+                else if (memcg_init_list_lru_node(&lru->node[i]))
+                        goto fail;
+        }
+        return 0;
+fail:
+        for (i = i - 1; i >= 0; i--)
+                memcg_destroy_list_lru_node(&lru->node[i]);
+        return -ENOMEM;
+}
+
+static void memcg_destroy_list_lru(struct list_lru *lru)
+{
+        int i;
+
+        if (!list_lru_memcg_aware(lru))
+                return;
+
+        for (i = 0; i < nr_node_ids; i++)
+                memcg_destroy_list_lru_node(&lru->node[i]);
+}
+
+static int memcg_update_list_lru(struct list_lru *lru,
+                                 int old_size, int new_size)
+{
+        int i;
+
+        if (!list_lru_memcg_aware(lru))
+                return 0;
+
+        for (i = 0; i < nr_node_ids; i++) {
+                if (memcg_update_list_lru_node(&lru->node[i],
+                                               old_size, new_size))
+                        goto fail;
+        }
+        return 0;
+fail:
+        for (i = i - 1; i >= 0; i--)
+                memcg_cancel_update_list_lru_node(&lru->node[i],
+                                                  old_size, new_size);
+        return -ENOMEM;
+}
+
+static void memcg_cancel_update_list_lru(struct list_lru *lru,
+                                         int old_size, int new_size)
+{
+        int i;
+
+        if (!list_lru_memcg_aware(lru))
+                return;
+
+        for (i = 0; i < nr_node_ids; i++)
+                memcg_cancel_update_list_lru_node(&lru->node[i],
+                                                  old_size, new_size);
+}
+
+int memcg_update_all_list_lrus(int new_size)
+{
+        int ret = 0;
+        struct list_lru *lru;
+        int old_size = memcg_nr_cache_ids;
+
+        mutex_lock(&list_lrus_mutex);
+        list_for_each_entry(lru, &list_lrus, list) {
+                ret = memcg_update_list_lru(lru, old_size, new_size);
+                if (ret)
+                        goto fail;
+        }
+out:
+        mutex_unlock(&list_lrus_mutex);
+        return ret;
+fail:
+        list_for_each_entry_continue_reverse(lru, &list_lrus, list)
+                memcg_cancel_update_list_lru(lru, old_size, new_size);
+        goto out;
+}
+#else
+static int memcg_init_list_lru(struct list_lru *lru, bool memcg_aware)
+{
+        return 0;
+}
+
+static void memcg_destroy_list_lru(struct list_lru *lru)
+{
+}
+#endif /* CONFIG_MEMCG_KMEM */
+
+int __list_lru_init(struct list_lru *lru, bool memcg_aware,
+                    struct lock_class_key *key)
 {
         int i;
         size_t size = sizeof(*lru->node) * nr_node_ids;
+        int err = -ENOMEM;
+
+        memcg_get_cache_ids();
 
         lru->node = kzalloc(size, GFP_KERNEL);
         if (!lru->node)
-                return -ENOMEM;
+                goto out;
 
         for (i = 0; i < nr_node_ids; i++) {
                 spin_lock_init(&lru->node[i].lock);
                 if (key)
                         lockdep_set_class(&lru->node[i].lock, key);
-                INIT_LIST_HEAD(&lru->node[i].list);
-                lru->node[i].nr_items = 0;
+                init_one_lru(&lru->node[i].lru);
+        }
+
+        err = memcg_init_list_lru(lru, memcg_aware);
+        if (err) {
+                kfree(lru->node);
+                goto out;
         }
+
         list_lru_register(lru);
-        return 0;
+out:
+        memcg_put_cache_ids();
+        return err;
 }
-EXPORT_SYMBOL_GPL(list_lru_init_key);
+EXPORT_SYMBOL_GPL(__list_lru_init);
 
 void list_lru_destroy(struct list_lru *lru)
 {
         /* Already destroyed or not yet initialized? */
         if (!lru->node)
                 return;
+
+        memcg_get_cache_ids();
+
         list_lru_unregister(lru);
+
+        memcg_destroy_list_lru(lru);
         kfree(lru->node);
         lru->node = NULL;
+
+        memcg_put_cache_ids();
 }
 EXPORT_SYMBOL_GPL(list_lru_destroy);