1 files changed, 86 insertions, 19 deletions

diff --git a/mm/thrash.c b/mm/thrash.c
index 2372d4ed5dd8..fabf2d0f5169 100644
--- a/mm/thrash.c
+++ b/mm/thrash.c
@@ -21,14 +21,40 @@
 #include <linux/mm.h>
 #include <linux/sched.h>
 #include <linux/swap.h>
+#include <linux/memcontrol.h>
+
+#include <trace/events/vmscan.h>
+
+#define TOKEN_AGING_INTERVAL    (0xFF)
 
 static DEFINE_SPINLOCK(swap_token_lock);
 struct mm_struct *swap_token_mm;
+struct mem_cgroup *swap_token_memcg;
 static unsigned int global_faults;
+static unsigned int last_aging;
+
+#ifdef CONFIG_CGROUP_MEM_RES_CTLR
+static struct mem_cgroup *swap_token_memcg_from_mm(struct mm_struct *mm)
+{
+        struct mem_cgroup *memcg;
+
+        memcg = try_get_mem_cgroup_from_mm(mm);
+        if (memcg)
+                css_put(mem_cgroup_css(memcg));
+
+        return memcg;
+}
+#else
+static struct mem_cgroup *swap_token_memcg_from_mm(struct mm_struct *mm)
+{
+        return NULL;
+}
+#endif
 
 void grab_swap_token(struct mm_struct *mm)
 {
         int current_interval;
+        unsigned int old_prio = mm->token_priority;
 
         global_faults++;
 
@@ -38,40 +64,81 @@ void grab_swap_token(struct mm_struct *mm)
                 return;
 
         /* First come first served */
-        if (swap_token_mm == NULL) {
-                mm->token_priority = mm->token_priority + 2;
-                swap_token_mm = mm;
-                goto out;
+        if (!swap_token_mm)
+                goto replace_token;
+
+        if ((global_faults - last_aging) > TOKEN_AGING_INTERVAL) {
+                swap_token_mm->token_priority /= 2;
+                last_aging = global_faults;
         }
 
-        if (mm != swap_token_mm) {
-                if (current_interval < mm->last_interval)
-                        mm->token_priority++;
-                else {
-                        if (likely(mm->token_priority > 0))
-                                mm->token_priority--;
-                }
-                /* Check if we deserve the token */
-                if (mm->token_priority > swap_token_mm->token_priority) {
-                        mm->token_priority += 2;
-                        swap_token_mm = mm;
-                }
-        } else {
-                /* Token holder came in again! */
+        if (mm == swap_token_mm) {
                 mm->token_priority += 2;
+                goto update_priority;
+        }
+
+        if (current_interval < mm->last_interval)
+                mm->token_priority++;
+        else {
+                if (likely(mm->token_priority > 0))
+                        mm->token_priority--;
         }
 
+        /* Check if we deserve the token */
+        if (mm->token_priority > swap_token_mm->token_priority)
+                goto replace_token;
+
+update_priority:
+        trace_update_swap_token_priority(mm, old_prio, swap_token_mm);
+
 out:
         mm->faultstamp = global_faults;
         mm->last_interval = current_interval;
         spin_unlock(&swap_token_lock);
+        return;
+
+replace_token:
+        mm->token_priority += 2;
+        trace_replace_swap_token(swap_token_mm, mm);
+        swap_token_mm = mm;
+        swap_token_memcg = swap_token_memcg_from_mm(mm);
+        last_aging = global_faults;
+        goto out;
 }
 
 /* Called on process exit. */
 void __put_swap_token(struct mm_struct *mm)
 {
         spin_lock(&swap_token_lock);
-        if (likely(mm == swap_token_mm))
+        if (likely(mm == swap_token_mm)) {
+                trace_put_swap_token(swap_token_mm);
                 swap_token_mm = NULL;
+                swap_token_memcg = NULL;
+        }
         spin_unlock(&swap_token_lock);
 }
+
+static bool match_memcg(struct mem_cgroup *a, struct mem_cgroup *b)
+{
+        if (!a)
+                return true;
+        if (!b)
+                return true;
+        if (a == b)
+                return true;
+        return false;
+}
+
+void disable_swap_token(struct mem_cgroup *memcg)
+{
+        /* memcg reclaim don't disable unrelated mm token. */
+        if (match_memcg(memcg, swap_token_memcg)) {
+                spin_lock(&swap_token_lock);
+                if (match_memcg(memcg, swap_token_memcg)) {
+                        trace_disable_swap_token(swap_token_mm);
+                        swap_token_mm = NULL;
+                        swap_token_memcg = NULL;
+                }
+                spin_unlock(&swap_token_lock);
+        }
+}