5 files changed, 85 insertions, 36 deletions

diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index 9724a38ee69d..50940da6adf3 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -84,6 +84,7 @@ int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *mem);
 
 extern struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page);
 extern struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p);
+extern struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm);
 
 static inline
 int mm_match_cgroup(const struct mm_struct *mm, const struct mem_cgroup *cgroup)
@@ -246,6 +247,11 @@ static inline struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page)
         return NULL;
 }
 
+static inline struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm)
+{
+        return NULL;
+}
+
 static inline int mm_match_cgroup(struct mm_struct *mm, struct mem_cgroup *mem)
 {
         return 1;
diff --git a/include/linux/swap.h b/include/linux/swap.h
index 384eb5fe530b..e70564647039 100644
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -358,6 +358,7 @@ struct backing_dev_info;
 extern struct mm_struct *swap_token_mm;
 extern void grab_swap_token(struct mm_struct *);
 extern void __put_swap_token(struct mm_struct *);
+extern void disable_swap_token(struct mem_cgroup *memcg);
 
 static inline int has_swap_token(struct mm_struct *mm)
 {
@@ -370,11 +371,6 @@ static inline void put_swap_token(struct mm_struct *mm)
                 __put_swap_token(mm);
 }
 
-static inline void disable_swap_token(void)
-{
-        put_swap_token(swap_token_mm);
-}
-
 #ifdef CONFIG_CGROUP_MEM_RES_CTLR
 extern void
 mem_cgroup_uncharge_swapcache(struct page *page, swp_entry_t ent, bool swapout);
@@ -500,7 +496,7 @@ static inline int has_swap_token(struct mm_struct *mm)
         return 0;
 }
 
-static inline void disable_swap_token(void)
+static inline void disable_swap_token(struct mem_cgroup *memcg)
 {
 }
 
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index bd9052a5d3ad..e37c44d88d46 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -735,7 +735,7 @@ struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
                                 struct mem_cgroup, css);
 }
 
-static struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm)
+struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm)
 {
         struct mem_cgroup *mem = NULL;
 
@@ -5414,18 +5414,16 @@ static void mem_cgroup_move_task(struct cgroup_subsys *ss,
                                 struct cgroup *old_cont,
                                 struct task_struct *p)
 {
-        struct mm_struct *mm;
+        struct mm_struct *mm = get_task_mm(p);
 
-        if (!mc.to)
-                /* no need to move charge */
-                return;
-
-        mm = get_task_mm(p);
         if (mm) {
-                mem_cgroup_move_charge(mm);
+                if (mc.to)
+                        mem_cgroup_move_charge(mm);
+                put_swap_token(mm);
                 mmput(mm);
         }
-        mem_cgroup_clear_mc();
+        if (mc.to)
+                mem_cgroup_clear_mc();
 }
 #else   /* !CONFIG_MMU */
 static int mem_cgroup_can_attach(struct cgroup_subsys *ss,
diff --git a/mm/thrash.c b/mm/thrash.c
index 2372d4ed5dd8..6cdf86511385 100644
--- a/mm/thrash.c
+++ b/mm/thrash.c
@@ -21,11 +21,31 @@
 #include <linux/mm.h>
 #include <linux/sched.h>
 #include <linux/swap.h>
+#include <linux/memcontrol.h>
 
 static DEFINE_SPINLOCK(swap_token_lock);
 struct mm_struct *swap_token_mm;
+struct mem_cgroup *swap_token_memcg;
 static unsigned int global_faults;
 
+#ifdef CONFIG_CGROUP_MEM_RES_CTLR
+static struct mem_cgroup *swap_token_memcg_from_mm(struct mm_struct *mm)
+{
+        struct mem_cgroup *memcg;
+
+        memcg = try_get_mem_cgroup_from_mm(mm);
+        if (memcg)
+                css_put(mem_cgroup_css(memcg));
+
+        return memcg;
+}
+#else
+static struct mem_cgroup *swap_token_memcg_from_mm(struct mm_struct *mm)
+{
+        return NULL;
+}
+#endif
+
 void grab_swap_token(struct mm_struct *mm)
 {
         int current_interval;
@@ -38,40 +58,69 @@ void grab_swap_token(struct mm_struct *mm)
                 return;
 
         /* First come first served */
-        if (swap_token_mm == NULL) {
-                mm->token_priority = mm->token_priority + 2;
-                swap_token_mm = mm;
+        if (!swap_token_mm)
+                goto replace_token;
+
+        if (mm == swap_token_mm) {
+                mm->token_priority += 2;
                 goto out;
         }
 
-        if (mm != swap_token_mm) {
-                if (current_interval < mm->last_interval)
-                        mm->token_priority++;
-                else {
-                        if (likely(mm->token_priority > 0))
-                                mm->token_priority--;
-                }
-                /* Check if we deserve the token */
-                if (mm->token_priority > swap_token_mm->token_priority) {
-                        mm->token_priority += 2;
-                        swap_token_mm = mm;
-                }
-        } else {
-                /* Token holder came in again! */
-                mm->token_priority += 2;
+        if (current_interval < mm->last_interval)
+                mm->token_priority++;
+        else {
+                if (likely(mm->token_priority > 0))
+                        mm->token_priority--;
         }
 
+        /* Check if we deserve the token */
+        if (mm->token_priority > swap_token_mm->token_priority)
+                goto replace_token;
+
 out:
         mm->faultstamp = global_faults;
         mm->last_interval = current_interval;
         spin_unlock(&swap_token_lock);
+        return;
+
+replace_token:
+        mm->token_priority += 2;
+        swap_token_mm = mm;
+        swap_token_memcg = swap_token_memcg_from_mm(mm);
+        goto out;
 }
 
 /* Called on process exit. */
 void __put_swap_token(struct mm_struct *mm)
 {
         spin_lock(&swap_token_lock);
-        if (likely(mm == swap_token_mm))
+        if (likely(mm == swap_token_mm)) {
                 swap_token_mm = NULL;
+                swap_token_memcg = NULL;
+        }
         spin_unlock(&swap_token_lock);
 }
+
+static bool match_memcg(struct mem_cgroup *a, struct mem_cgroup *b)
+{
+        if (!a)
+                return true;
+        if (!b)
+                return true;
+        if (a == b)
+                return true;
+        return false;
+}
+
+void disable_swap_token(struct mem_cgroup *memcg)
+{
+        /* memcg reclaim don't disable unrelated mm token. */
+        if (match_memcg(memcg, swap_token_memcg)) {
+                spin_lock(&swap_token_lock);
+                if (match_memcg(memcg, swap_token_memcg)) {
+                        swap_token_mm = NULL;
+                        swap_token_memcg = NULL;
+                }
+                spin_unlock(&swap_token_lock);
+        }
+}
diff --git a/mm/vmscan.c b/mm/vmscan.c
index faa0a088f9cc..dbe6ea321df4 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -2081,7 +2081,7 @@ static unsigned long do_try_to_free_pages(struct zonelist *zonelist,
         for (priority = DEF_PRIORITY; priority >= 0; priority--) {
                 sc->nr_scanned = 0;
                 if (!priority)
-                        disable_swap_token();
+                        disable_swap_token(sc->mem_cgroup);
                 total_scanned += shrink_zones(priority, zonelist, sc);
                 /*
                  * Don't shrink slabs when reclaiming memory from
@@ -2407,7 +2407,7 @@ loop_again:
 
                 /* The swap token gets in the way of swapout... */
                 if (!priority)
-                        disable_swap_token();
+                        disable_swap_token(NULL);
 
                 all_zones_ok = 1;
                 balanced = 0;