1 files changed, 67 insertions, 15 deletions

diff --git a/include/linux/list_lru.h b/include/linux/list_lru.h
index f3434533fbf8..2a6b9947aaa3 100644
--- a/include/linux/list_lru.h
+++ b/include/linux/list_lru.h
@@ -9,6 +9,9 @@
 
 #include <linux/list.h>
 #include <linux/nodemask.h>
+#include <linux/shrinker.h>
+
+struct mem_cgroup;
 
 /* list_lru_walk_cb has to always return one of those */
 enum lru_status {
@@ -21,24 +24,45 @@ enum lru_status {
                                    internally, but has to return locked. */
 };
 
-struct list_lru_node {
-        spinlock_t              lock;
+struct list_lru_one {
         struct list_head        list;
-        /* kept as signed so we can catch imbalance bugs */
+        /* may become negative during memcg reparenting */
         long                    nr_items;
+};
+
+struct list_lru_memcg {
+        /* array of per cgroup lists, indexed by memcg_cache_id */
+        struct list_lru_one     *lru[0];
+};
+
+struct list_lru_node {
+        /* protects all lists on the node, including per cgroup */
+        spinlock_t              lock;
+        /* global list, used for the root cgroup in cgroup aware lrus */
+        struct list_lru_one     lru;
+#ifdef CONFIG_MEMCG_KMEM
+        /* for cgroup aware lrus points to per cgroup lists, otherwise NULL */
+        struct list_lru_memcg   *memcg_lrus;
+#endif
 } ____cacheline_aligned_in_smp;
 
 struct list_lru {
         struct list_lru_node    *node;
-        nodemask_t              active_nodes;
+#ifdef CONFIG_MEMCG_KMEM
+        struct list_head        list;
+#endif
 };
 
 void list_lru_destroy(struct list_lru *lru);
-int list_lru_init_key(struct list_lru *lru, struct lock_class_key *key);
-static inline int list_lru_init(struct list_lru *lru)
-{
-        return list_lru_init_key(lru, NULL);
-}
+int __list_lru_init(struct list_lru *lru, bool memcg_aware,
+                    struct lock_class_key *key);
+
+#define list_lru_init(lru)              __list_lru_init((lru), false, NULL)
+#define list_lru_init_key(lru, key)     __list_lru_init((lru), false, (key))
+#define list_lru_init_memcg(lru)        __list_lru_init((lru), true, NULL)
+
+int memcg_update_all_list_lrus(int num_memcgs);
+void memcg_drain_all_list_lrus(int src_idx, int dst_idx);
 
 /**
  * list_lru_add: add an element to the lru list's tail
@@ -72,32 +96,48 @@ bool list_lru_add(struct list_lru *lru, struct list_head *item);
 bool list_lru_del(struct list_lru *lru, struct list_head *item);
 
 /**
- * list_lru_count_node: return the number of objects currently held by @lru
+ * list_lru_count_one: return the number of objects currently held by @lru
  * @lru: the lru pointer.
  * @nid: the node id to count from.
+ * @memcg: the cgroup to count from.
  *
  * Always return a non-negative number, 0 for empty lists. There is no
  * guarantee that the list is not updated while the count is being computed.
  * Callers that want such a guarantee need to provide an outer lock.
  */
+unsigned long list_lru_count_one(struct list_lru *lru,
+                                 int nid, struct mem_cgroup *memcg);
 unsigned long list_lru_count_node(struct list_lru *lru, int nid);
+
+static inline unsigned long list_lru_shrink_count(struct list_lru *lru,
+                                                  struct shrink_control *sc)
+{
+        return list_lru_count_one(lru, sc->nid, sc->memcg);
+}
+
 static inline unsigned long list_lru_count(struct list_lru *lru)
 {
         long count = 0;
         int nid;
 
-        for_each_node_mask(nid, lru->active_nodes)
+        for_each_node_state(nid, N_NORMAL_MEMORY)
                 count += list_lru_count_node(lru, nid);
 
         return count;
 }
 
-typedef enum lru_status
-(*list_lru_walk_cb)(struct list_head *item, spinlock_t *lock, void *cb_arg);
+void list_lru_isolate(struct list_lru_one *list, struct list_head *item);
+void list_lru_isolate_move(struct list_lru_one *list, struct list_head *item,
+                           struct list_head *head);
+
+typedef enum lru_status (*list_lru_walk_cb)(struct list_head *item,
+                struct list_lru_one *list, spinlock_t *lock, void *cb_arg);
+
 /**
- * list_lru_walk_node: walk a list_lru, isolating and disposing freeable items.
+ * list_lru_walk_one: walk a list_lru, isolating and disposing freeable items.
  * @lru: the lru pointer.
  * @nid: the node id to scan from.
+ * @memcg: the cgroup to scan from.
  * @isolate: callback function that is resposible for deciding what to do with
  *  the item currently being scanned
  * @cb_arg: opaque type that will be passed to @isolate
@@ -115,18 +155,30 @@ typedef enum lru_status
  *
  * Return value: the number of objects effectively removed from the LRU.
  */
+unsigned long list_lru_walk_one(struct list_lru *lru,
+                                int nid, struct mem_cgroup *memcg,
+                                list_lru_walk_cb isolate, void *cb_arg,
+                                unsigned long *nr_to_walk);
 unsigned long list_lru_walk_node(struct list_lru *lru, int nid,
                                  list_lru_walk_cb isolate, void *cb_arg,
                                  unsigned long *nr_to_walk);
 
 static inline unsigned long
+list_lru_shrink_walk(struct list_lru *lru, struct shrink_control *sc,
+                     list_lru_walk_cb isolate, void *cb_arg)
+{
+        return list_lru_walk_one(lru, sc->nid, sc->memcg, isolate, cb_arg,
+                                 &sc->nr_to_scan);
+}
+
+static inline unsigned long
 list_lru_walk(struct list_lru *lru, list_lru_walk_cb isolate,
               void *cb_arg, unsigned long nr_to_walk)
 {
         long isolated = 0;
         int nid;
 
-        for_each_node_mask(nid, lru->active_nodes) {
+        for_each_node_state(nid, N_NORMAL_MEMORY) {
                 isolated += list_lru_walk_node(lru, nid, isolate,
                                                cb_arg, &nr_to_walk);
                 if (nr_to_walk <= 0)