1 files changed, 214 insertions, 251 deletions

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 36896f3eb7f5..866dcc7eeb0c 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -32,11 +32,12 @@
 #include <linux/fs.h>
 #include <linux/seq_file.h>
 #include <linux/vmalloc.h>
+#include <linux/mm_inline.h>
+#include <linux/page_cgroup.h>
 
 #include <asm/uaccess.h>
 
 struct cgroup_subsys mem_cgroup_subsys __read_mostly;
-static struct kmem_cache *page_cgroup_cache __read_mostly;
 #define MEM_CGROUP_RECLAIM_RETRIES      5
 
 /*
@@ -65,11 +66,10 @@ struct mem_cgroup_stat {
 /*
  * For accounting under irq disable, no need for increment preempt count.
  */
-static void __mem_cgroup_stat_add_safe(struct mem_cgroup_stat *stat,
+static inline void __mem_cgroup_stat_add_safe(struct mem_cgroup_stat_cpu *stat,
                 enum mem_cgroup_stat_index idx, int val)
 {
-        int cpu = smp_processor_id();
-        stat->cpustat[cpu].count[idx] += val;
+        stat->count[idx] += val;
 }
 
 static s64 mem_cgroup_read_stat(struct mem_cgroup_stat *stat,
@@ -85,22 +85,13 @@ static s64 mem_cgroup_read_stat(struct mem_cgroup_stat *stat,
 /*
  * per-zone information in memory controller.
  */
-
-enum mem_cgroup_zstat_index {
-        MEM_CGROUP_ZSTAT_ACTIVE,
-        MEM_CGROUP_ZSTAT_INACTIVE,
-
-        NR_MEM_CGROUP_ZSTAT,
-};
-
 struct mem_cgroup_per_zone {
         /*
          * spin_lock to protect the per cgroup LRU
          */
         spinlock_t              lru_lock;
-        struct list_head        active_list;
-        struct list_head        inactive_list;
-        unsigned long count[NR_MEM_CGROUP_ZSTAT];
+        struct list_head        lists[NR_LRU_LISTS];
+        unsigned long           count[NR_LRU_LISTS];
 };
 /* Macro for accessing counter */
 #define MEM_CGROUP_ZSTAT(mz, idx)       ((mz)->count[(idx)])
@@ -144,69 +135,52 @@ struct mem_cgroup {
 };
 static struct mem_cgroup init_mem_cgroup;
 
-/*
- * We use the lower bit of the page->page_cgroup pointer as a bit spin
- * lock.  We need to ensure that page->page_cgroup is at least two
- * byte aligned (based on comments from Nick Piggin).  But since
- * bit_spin_lock doesn't actually set that lock bit in a non-debug
- * uniprocessor kernel, we should avoid setting it here too.
- */
-#define PAGE_CGROUP_LOCK_BIT    0x0
-#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
-#define PAGE_CGROUP_LOCK        (1 << PAGE_CGROUP_LOCK_BIT)
-#else
-#define PAGE_CGROUP_LOCK        0x0
-#endif
-
-/*
- * A page_cgroup page is associated with every page descriptor. The
- * page_cgroup helps us identify information about the cgroup
- */
-struct page_cgroup {
-        struct list_head lru;           /* per cgroup LRU list */
-        struct page *page;
-        struct mem_cgroup *mem_cgroup;
-        int flags;
-};
-#define PAGE_CGROUP_FLAG_CACHE  (0x1)   /* charged as cache */
-#define PAGE_CGROUP_FLAG_ACTIVE (0x2)   /* page is active in this cgroup */
-
-static int page_cgroup_nid(struct page_cgroup *pc)
-{
-        return page_to_nid(pc->page);
-}
-
-static enum zone_type page_cgroup_zid(struct page_cgroup *pc)
-{
-        return page_zonenum(pc->page);
-}
-
 enum charge_type {
         MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
         MEM_CGROUP_CHARGE_TYPE_MAPPED,
+        MEM_CGROUP_CHARGE_TYPE_SHMEM,   /* used by page migration of shmem */
         MEM_CGROUP_CHARGE_TYPE_FORCE,   /* used by force_empty */
+        NR_CHARGE_TYPE,
+};
+
+/* only for here (for easy reading.) */
+#define PCGF_CACHE      (1UL << PCG_CACHE)
+#define PCGF_USED       (1UL << PCG_USED)
+#define PCGF_ACTIVE     (1UL << PCG_ACTIVE)
+#define PCGF_LOCK       (1UL << PCG_LOCK)
+#define PCGF_FILE       (1UL << PCG_FILE)
+static const unsigned long
+pcg_default_flags[NR_CHARGE_TYPE] = {
+        PCGF_CACHE | PCGF_FILE | PCGF_USED | PCGF_LOCK, /* File Cache */
+        PCGF_ACTIVE | PCGF_USED | PCGF_LOCK, /* Anon */
+        PCGF_ACTIVE | PCGF_CACHE | PCGF_USED | PCGF_LOCK, /* Shmem */
+        0, /* FORCE */
 };
 
 /*
  * Always modified under lru lock. Then, not necessary to preempt_disable()
  */
-static void mem_cgroup_charge_statistics(struct mem_cgroup *mem, int flags,
-                                        bool charge)
+static void mem_cgroup_charge_statistics(struct mem_cgroup *mem,
+                                         struct page_cgroup *pc,
+                                         bool charge)
 {
         int val = (charge)? 1 : -1;
         struct mem_cgroup_stat *stat = &mem->stat;
+        struct mem_cgroup_stat_cpu *cpustat;
 
         VM_BUG_ON(!irqs_disabled());
-        if (flags & PAGE_CGROUP_FLAG_CACHE)
-                __mem_cgroup_stat_add_safe(stat, MEM_CGROUP_STAT_CACHE, val);
+
+        cpustat = &stat->cpustat[smp_processor_id()];
+        if (PageCgroupCache(pc))
+                __mem_cgroup_stat_add_safe(cpustat, MEM_CGROUP_STAT_CACHE, val);
         else
-                __mem_cgroup_stat_add_safe(stat, MEM_CGROUP_STAT_RSS, val);
+                __mem_cgroup_stat_add_safe(cpustat, MEM_CGROUP_STAT_RSS, val);
 
         if (charge)
-                __mem_cgroup_stat_add_safe(stat,
+                __mem_cgroup_stat_add_safe(cpustat,
                                 MEM_CGROUP_STAT_PGPGIN_COUNT, 1);
         else
-                __mem_cgroup_stat_add_safe(stat,
+                __mem_cgroup_stat_add_safe(cpustat,
                                 MEM_CGROUP_STAT_PGPGOUT_COUNT, 1);
 }
 
@@ -227,7 +201,7 @@ page_cgroup_zoneinfo(struct page_cgroup *pc)
 }
 
 static unsigned long mem_cgroup_get_all_zonestat(struct mem_cgroup *mem,
-                                        enum mem_cgroup_zstat_index idx)
+                                        enum lru_list idx)
 {
         int nid, zid;
         struct mem_cgroup_per_zone *mz;
@@ -262,85 +236,77 @@ struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
                                 struct mem_cgroup, css);
 }
 
-static inline int page_cgroup_locked(struct page *page)
-{
-        return bit_spin_is_locked(PAGE_CGROUP_LOCK_BIT, &page->page_cgroup);
-}
-
-static void page_assign_page_cgroup(struct page *page, struct page_cgroup *pc)
-{
-        VM_BUG_ON(!page_cgroup_locked(page));
-        page->page_cgroup = ((unsigned long)pc | PAGE_CGROUP_LOCK);
-}
-
-struct page_cgroup *page_get_page_cgroup(struct page *page)
-{
-        return (struct page_cgroup *) (page->page_cgroup & ~PAGE_CGROUP_LOCK);
-}
-
-static void lock_page_cgroup(struct page *page)
-{
-        bit_spin_lock(PAGE_CGROUP_LOCK_BIT, &page->page_cgroup);
-}
-
-static int try_lock_page_cgroup(struct page *page)
-{
-        return bit_spin_trylock(PAGE_CGROUP_LOCK_BIT, &page->page_cgroup);
-}
-
-static void unlock_page_cgroup(struct page *page)
-{
-        bit_spin_unlock(PAGE_CGROUP_LOCK_BIT, &page->page_cgroup);
-}
-
 static void __mem_cgroup_remove_list(struct mem_cgroup_per_zone *mz,
                         struct page_cgroup *pc)
 {
-        int from = pc->flags & PAGE_CGROUP_FLAG_ACTIVE;
+        int lru = LRU_BASE;
+
+        if (PageCgroupUnevictable(pc))
+                lru = LRU_UNEVICTABLE;
+        else {
+                if (PageCgroupActive(pc))
+                        lru += LRU_ACTIVE;
+                if (PageCgroupFile(pc))
+                        lru += LRU_FILE;
+        }
 
-        if (from)
-                MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_ACTIVE) -= 1;
-        else
-                MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_INACTIVE) -= 1;
+        MEM_CGROUP_ZSTAT(mz, lru) -= 1;
 
-        mem_cgroup_charge_statistics(pc->mem_cgroup, pc->flags, false);
+        mem_cgroup_charge_statistics(pc->mem_cgroup, pc, false);
         list_del(&pc->lru);
 }
 
 static void __mem_cgroup_add_list(struct mem_cgroup_per_zone *mz,
                                 struct page_cgroup *pc)
 {
-        int to = pc->flags & PAGE_CGROUP_FLAG_ACTIVE;
-
-        if (!to) {
-                MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_INACTIVE) += 1;
-                list_add(&pc->lru, &mz->inactive_list);
-        } else {
-                MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_ACTIVE) += 1;
-                list_add(&pc->lru, &mz->active_list);
+        int lru = LRU_BASE;
+
+        if (PageCgroupUnevictable(pc))
+                lru = LRU_UNEVICTABLE;
+        else {
+                if (PageCgroupActive(pc))
+                        lru += LRU_ACTIVE;
+                if (PageCgroupFile(pc))
+                        lru += LRU_FILE;
         }
-        mem_cgroup_charge_statistics(pc->mem_cgroup, pc->flags, true);
+
+        MEM_CGROUP_ZSTAT(mz, lru) += 1;
+        list_add(&pc->lru, &mz->lists[lru]);
+
+        mem_cgroup_charge_statistics(pc->mem_cgroup, pc, true);
 }
 
-static void __mem_cgroup_move_lists(struct page_cgroup *pc, bool active)
+static void __mem_cgroup_move_lists(struct page_cgroup *pc, enum lru_list lru)
 {
-        int from = pc->flags & PAGE_CGROUP_FLAG_ACTIVE;
         struct mem_cgroup_per_zone *mz = page_cgroup_zoneinfo(pc);
+        int active    = PageCgroupActive(pc);
+        int file      = PageCgroupFile(pc);
+        int unevictable = PageCgroupUnevictable(pc);
+        enum lru_list from = unevictable ? LRU_UNEVICTABLE :
+                                (LRU_FILE * !!file + !!active);
 
-        if (from)
-                MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_ACTIVE) -= 1;
-        else
-                MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_INACTIVE) -= 1;
+        if (lru == from)
+                return;
 
-        if (active) {
-                MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_ACTIVE) += 1;
-                pc->flags |= PAGE_CGROUP_FLAG_ACTIVE;
-                list_move(&pc->lru, &mz->active_list);
+        MEM_CGROUP_ZSTAT(mz, from) -= 1;
+        /*
+         * However this is done under mz->lru_lock, another flags, which
+         * are not related to LRU, will be modified from out-of-lock.
+         * We have to use atomic set/clear flags.
+         */
+        if (is_unevictable_lru(lru)) {
+                ClearPageCgroupActive(pc);
+                SetPageCgroupUnevictable(pc);
         } else {
-                MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_INACTIVE) += 1;
-                pc->flags &= ~PAGE_CGROUP_FLAG_ACTIVE;
-                list_move(&pc->lru, &mz->inactive_list);
+                if (is_active_lru(lru))
+                        SetPageCgroupActive(pc);
+                else
+                        ClearPageCgroupActive(pc);
+                ClearPageCgroupUnevictable(pc);
         }
+
+        MEM_CGROUP_ZSTAT(mz, lru) += 1;
+        list_move(&pc->lru, &mz->lists[lru]);
 }
 
 int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *mem)
@@ -356,7 +322,7 @@ int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *mem)
 /*
  * This routine assumes that the appropriate zone's lru lock is already held
  */
-void mem_cgroup_move_lists(struct page *page, bool active)
+void mem_cgroup_move_lists(struct page *page, enum lru_list lru)
 {
         struct page_cgroup *pc;
         struct mem_cgroup_per_zone *mz;
@@ -372,17 +338,16 @@ void mem_cgroup_move_lists(struct page *page, bool active)
          * safely get to page_cgroup without it, so just try_lock it:
          * mem_cgroup_isolate_pages allows for page left on wrong list.
          */
-        if (!try_lock_page_cgroup(page))
+        pc = lookup_page_cgroup(page);
+        if (!trylock_page_cgroup(pc))
                 return;
-
-        pc = page_get_page_cgroup(page);
-        if (pc) {
+        if (pc && PageCgroupUsed(pc)) {
                 mz = page_cgroup_zoneinfo(pc);
                 spin_lock_irqsave(&mz->lru_lock, flags);
-                __mem_cgroup_move_lists(pc, active);
+                __mem_cgroup_move_lists(pc, lru);
                 spin_unlock_irqrestore(&mz->lru_lock, flags);
         }
-        unlock_page_cgroup(page);
+        unlock_page_cgroup(pc);
 }
 
 /*
@@ -403,21 +368,6 @@ int mem_cgroup_calc_mapped_ratio(struct mem_cgroup *mem)
 }
 
 /*
- * This function is called from vmscan.c. In page reclaiming loop. balance
- * between active and inactive list is calculated. For memory controller
- * page reclaiming, we should use using mem_cgroup's imbalance rather than
- * zone's global lru imbalance.
- */
-long mem_cgroup_reclaim_imbalance(struct mem_cgroup *mem)
-{
-        unsigned long active, inactive;
-        /* active and inactive are the number of pages. 'long' is ok.*/
-        active = mem_cgroup_get_all_zonestat(mem, MEM_CGROUP_ZSTAT_ACTIVE);
-        inactive = mem_cgroup_get_all_zonestat(mem, MEM_CGROUP_ZSTAT_INACTIVE);
-        return (long) (active / (inactive + 1));
-}
-
-/*
  * prev_priority control...this will be used in memory reclaim path.
  */
 int mem_cgroup_get_reclaim_priority(struct mem_cgroup *mem)
@@ -444,28 +394,17 @@ void mem_cgroup_record_reclaim_priority(struct mem_cgroup *mem, int priority)
  * (see include/linux/mmzone.h)
  */
 
-long mem_cgroup_calc_reclaim_active(struct mem_cgroup *mem,
-                                   struct zone *zone, int priority)
+long mem_cgroup_calc_reclaim(struct mem_cgroup *mem, struct zone *zone,
+                                        int priority, enum lru_list lru)
 {
-        long nr_active;
+        long nr_pages;
         int nid = zone->zone_pgdat->node_id;
         int zid = zone_idx(zone);
         struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(mem, nid, zid);
 
-        nr_active = MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_ACTIVE);
-        return (nr_active >> priority);
-}
+        nr_pages = MEM_CGROUP_ZSTAT(mz, lru);
 
-long mem_cgroup_calc_reclaim_inactive(struct mem_cgroup *mem,
-                                        struct zone *zone, int priority)
-{
-        long nr_inactive;
-        int nid = zone->zone_pgdat->node_id;
-        int zid = zone_idx(zone);
-        struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(mem, nid, zid);
-
-        nr_inactive = MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_INACTIVE);
-        return (nr_inactive >> priority);
+        return (nr_pages >> priority);
 }
 
 unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan,
@@ -473,7 +412,7 @@ unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan,
                                         unsigned long *scanned, int order,
                                         int mode, struct zone *z,
                                         struct mem_cgroup *mem_cont,
-                                        int active)
+                                        int active, int file)
 {
         unsigned long nr_taken = 0;
         struct page *page;
@@ -484,38 +423,38 @@ unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan,
         int nid = z->zone_pgdat->node_id;
         int zid = zone_idx(z);
         struct mem_cgroup_per_zone *mz;
+        int lru = LRU_FILE * !!file + !!active;
 
         BUG_ON(!mem_cont);
         mz = mem_cgroup_zoneinfo(mem_cont, nid, zid);
-        if (active)
-                src = &mz->active_list;
-        else
-                src = &mz->inactive_list;
-
+        src = &mz->lists[lru];
 
         spin_lock(&mz->lru_lock);
         scan = 0;
         list_for_each_entry_safe_reverse(pc, tmp, src, lru) {
                 if (scan >= nr_to_scan)
                         break;
+                if (unlikely(!PageCgroupUsed(pc)))
+                        continue;
                 page = pc->page;
 
                 if (unlikely(!PageLRU(page)))
                         continue;
 
-                if (PageActive(page) && !active) {
-                        __mem_cgroup_move_lists(pc, true);
-                        continue;
-                }
-                if (!PageActive(page) && active) {
-                        __mem_cgroup_move_lists(pc, false);
+                /*
+                 * TODO: play better with lumpy reclaim, grabbing anything.
+                 */
+                if (PageUnevictable(page) ||
+                    (PageActive(page) && !active) ||
+                    (!PageActive(page) && active)) {
+                        __mem_cgroup_move_lists(pc, page_lru(page));
                         continue;
                 }
 
                 scan++;
                 list_move(&pc->lru, &pc_list);
 
-                if (__isolate_lru_page(page, mode) == 0) {
+                if (__isolate_lru_page(page, mode, file) == 0) {
                         list_move(&page->lru, dst);
                         nr_taken++;
                 }
@@ -540,26 +479,27 @@ static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm,
 {
         struct mem_cgroup *mem;
         struct page_cgroup *pc;
-        unsigned long flags;
         unsigned long nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
         struct mem_cgroup_per_zone *mz;
+        unsigned long flags;
 
-        pc = kmem_cache_alloc(page_cgroup_cache, gfp_mask);
-        if (unlikely(pc == NULL))
-                goto err;
-
+        pc = lookup_page_cgroup(page);
+        /* can happen at boot */
+        if (unlikely(!pc))
+                return 0;
+        prefetchw(pc);
         /*
          * We always charge the cgroup the mm_struct belongs to.
          * The mm_struct's mem_cgroup changes on task migration if the
          * thread group leader migrates. It's possible that mm is not
          * set, if so charge the init_mm (happens for pagecache usage).
          */
+
         if (likely(!memcg)) {
                 rcu_read_lock();
                 mem = mem_cgroup_from_task(rcu_dereference(mm->owner));
                 if (unlikely(!mem)) {
                         rcu_read_unlock();
-                        kmem_cache_free(page_cgroup_cache, pc);
                         return 0;
                 }
                 /*
@@ -572,7 +512,7 @@ static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm,
                 css_get(&memcg->css);
         }
 
-        while (res_counter_charge(&mem->res, PAGE_SIZE)) {
+        while (unlikely(res_counter_charge(&mem->res, PAGE_SIZE))) {
                 if (!(gfp_mask & __GFP_WAIT))
                         goto out;
 
@@ -595,39 +535,33 @@ static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm,
                 }
         }
 
-        pc->mem_cgroup = mem;
-        pc->page = page;
-        /*
-         * If a page is accounted as a page cache, insert to inactive list.
-         * If anon, insert to active list.
-         */
-        if (ctype == MEM_CGROUP_CHARGE_TYPE_CACHE)
-                pc->flags = PAGE_CGROUP_FLAG_CACHE;
-        else
-                pc->flags = PAGE_CGROUP_FLAG_ACTIVE;
 
-        lock_page_cgroup(page);
-        if (unlikely(page_get_page_cgroup(page))) {
-                unlock_page_cgroup(page);
+        lock_page_cgroup(pc);
+        if (unlikely(PageCgroupUsed(pc))) {
+                unlock_page_cgroup(pc);
                 res_counter_uncharge(&mem->res, PAGE_SIZE);
                 css_put(&mem->css);
-                kmem_cache_free(page_cgroup_cache, pc);
+
                 goto done;
         }
-        page_assign_page_cgroup(page, pc);
+        pc->mem_cgroup = mem;
+        /*
+         * If a page is accounted as a page cache, insert to inactive list.
+         * If anon, insert to active list.
+         */
+        pc->flags = pcg_default_flags[ctype];
 
         mz = page_cgroup_zoneinfo(pc);
+
         spin_lock_irqsave(&mz->lru_lock, flags);
         __mem_cgroup_add_list(mz, pc);
         spin_unlock_irqrestore(&mz->lru_lock, flags);
+        unlock_page_cgroup(pc);
 
-        unlock_page_cgroup(page);
 done:
         return 0;
 out:
         css_put(&mem->css);
-        kmem_cache_free(page_cgroup_cache, pc);
-err:
         return -ENOMEM;
 }
 
@@ -635,7 +569,8 @@ int mem_cgroup_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask)
 {
         if (mem_cgroup_subsys.disabled)
                 return 0;
-
+        if (PageCompound(page))
+                return 0;
         /*
          * If already mapped, we don't have to account.
          * If page cache, page->mapping has address_space.
@@ -656,7 +591,8 @@ int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm,
 {
         if (mem_cgroup_subsys.disabled)
                 return 0;
-
+        if (PageCompound(page))
+                return 0;
         /*
          * Corner case handling. This is called from add_to_page_cache()
          * in usual. But some FS (shmem) precharges this page before calling it
@@ -669,22 +605,27 @@ int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm,
         if (!(gfp_mask & __GFP_WAIT)) {
                 struct page_cgroup *pc;
 
-                lock_page_cgroup(page);
-                pc = page_get_page_cgroup(page);
-                if (pc) {
-                        VM_BUG_ON(pc->page != page);
-                        VM_BUG_ON(!pc->mem_cgroup);
-                        unlock_page_cgroup(page);
+
+                pc = lookup_page_cgroup(page);
+                if (!pc)
+                        return 0;
+                lock_page_cgroup(pc);
+                if (PageCgroupUsed(pc)) {
+                        unlock_page_cgroup(pc);
                         return 0;
                 }
-                unlock_page_cgroup(page);
+                unlock_page_cgroup(pc);
         }
 
         if (unlikely(!mm))
                 mm = &init_mm;
 
-        return mem_cgroup_charge_common(page, mm, gfp_mask,
+        if (page_is_file_cache(page))
+                return mem_cgroup_charge_common(page, mm, gfp_mask,
                                 MEM_CGROUP_CHARGE_TYPE_CACHE, NULL);
+        else
+                return mem_cgroup_charge_common(page, mm, gfp_mask,
+                                MEM_CGROUP_CHARGE_TYPE_SHMEM, NULL);
 }
 
 /*
@@ -704,44 +645,46 @@ __mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype)
         /*
          * Check if our page_cgroup is valid
          */
-        lock_page_cgroup(page);
-        pc = page_get_page_cgroup(page);
-        if (unlikely(!pc))
-                goto unlock;
-
-        VM_BUG_ON(pc->page != page);
+        pc = lookup_page_cgroup(page);
+        if (unlikely(!pc || !PageCgroupUsed(pc)))
+                return;
 
-        if ((ctype == MEM_CGROUP_CHARGE_TYPE_MAPPED)
-            && ((pc->flags & PAGE_CGROUP_FLAG_CACHE)
-                || page_mapped(page)))
-                goto unlock;
+        lock_page_cgroup(pc);
+        if ((ctype == MEM_CGROUP_CHARGE_TYPE_MAPPED && page_mapped(page))
+             || !PageCgroupUsed(pc)) {
+                /* This happens at race in zap_pte_range() and do_swap_page()*/
+                unlock_page_cgroup(pc);
+                return;
+        }
+        ClearPageCgroupUsed(pc);
+        mem = pc->mem_cgroup;
 
         mz = page_cgroup_zoneinfo(pc);
         spin_lock_irqsave(&mz->lru_lock, flags);
         __mem_cgroup_remove_list(mz, pc);
         spin_unlock_irqrestore(&mz->lru_lock, flags);
+        unlock_page_cgroup(pc);
 
-        page_assign_page_cgroup(page, NULL);
-        unlock_page_cgroup(page);
-
-        mem = pc->mem_cgroup;
         res_counter_uncharge(&mem->res, PAGE_SIZE);
         css_put(&mem->css);
 
-        kmem_cache_free(page_cgroup_cache, pc);
         return;
-unlock:
-        unlock_page_cgroup(page);
 }
 
 void mem_cgroup_uncharge_page(struct page *page)
 {
+        /* early check. */
+        if (page_mapped(page))
+                return;
+        if (page->mapping && !PageAnon(page))
+                return;
         __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_MAPPED);
 }
 
 void mem_cgroup_uncharge_cache_page(struct page *page)
 {
         VM_BUG_ON(page_mapped(page));
+        VM_BUG_ON(page->mapping);
         __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_CACHE);
 }
 
@@ -758,15 +701,19 @@ int mem_cgroup_prepare_migration(struct page *page, struct page *newpage)
         if (mem_cgroup_subsys.disabled)
                 return 0;
 
-        lock_page_cgroup(page);
-        pc = page_get_page_cgroup(page);
-        if (pc) {
+        pc = lookup_page_cgroup(page);
+        lock_page_cgroup(pc);
+        if (PageCgroupUsed(pc)) {
                 mem = pc->mem_cgroup;
                 css_get(&mem->css);
-                if (pc->flags & PAGE_CGROUP_FLAG_CACHE)
-                        ctype = MEM_CGROUP_CHARGE_TYPE_CACHE;
+                if (PageCgroupCache(pc)) {
+                        if (page_is_file_cache(page))
+                                ctype = MEM_CGROUP_CHARGE_TYPE_CACHE;
+                        else
+                                ctype = MEM_CGROUP_CHARGE_TYPE_SHMEM;
+                }
         }
-        unlock_page_cgroup(page);
+        unlock_page_cgroup(pc);
         if (mem) {
                 ret = mem_cgroup_charge_common(newpage, NULL, GFP_KERNEL,
                         ctype, mem);
@@ -791,7 +738,7 @@ void mem_cgroup_end_migration(struct page *newpage)
          */
         if (!newpage->mapping)
                 __mem_cgroup_uncharge_common(newpage,
-                                         MEM_CGROUP_CHARGE_TYPE_FORCE);
+                                MEM_CGROUP_CHARGE_TYPE_FORCE);
         else if (PageAnon(newpage))
                 mem_cgroup_uncharge_page(newpage);
 }
@@ -863,7 +810,7 @@ int mem_cgroup_resize_limit(struct mem_cgroup *memcg, unsigned long long val)
 #define FORCE_UNCHARGE_BATCH    (128)
 static void mem_cgroup_force_empty_list(struct mem_cgroup *mem,
                             struct mem_cgroup_per_zone *mz,
-                            int active)
+                            enum lru_list lru)
 {
         struct page_cgroup *pc;
         struct page *page;
@@ -871,15 +818,14 @@ static void mem_cgroup_force_empty_list(struct mem_cgroup *mem,
         unsigned long flags;
         struct list_head *list;
 
-        if (active)
-                list = &mz->active_list;
-        else
-                list = &mz->inactive_list;
+        list = &mz->lists[lru];
 
         spin_lock_irqsave(&mz->lru_lock, flags);
         while (!list_empty(list)) {
                 pc = list_entry(list->prev, struct page_cgroup, lru);
                 page = pc->page;
+                if (!PageCgroupUsed(pc))
+                        break;
                 get_page(page);
                 spin_unlock_irqrestore(&mz->lru_lock, flags);
                 /*
@@ -894,8 +840,10 @@ static void mem_cgroup_force_empty_list(struct mem_cgroup *mem,
                                 count = FORCE_UNCHARGE_BATCH;
                                 cond_resched();
                         }
-                } else
-                        cond_resched();
+                } else {
+                        spin_lock_irqsave(&mz->lru_lock, flags);
+                        break;
+                }
                 spin_lock_irqsave(&mz->lru_lock, flags);
         }
         spin_unlock_irqrestore(&mz->lru_lock, flags);
@@ -919,15 +867,17 @@ static int mem_cgroup_force_empty(struct mem_cgroup *mem)
         while (mem->res.usage > 0) {
                 if (atomic_read(&mem->css.cgroup->count) > 0)
                         goto out;
+                /* This is for making all *used* pages to be on LRU. */
+                lru_add_drain_all();
                 for_each_node_state(node, N_POSSIBLE)
                         for (zid = 0; zid < MAX_NR_ZONES; zid++) {
                                 struct mem_cgroup_per_zone *mz;
+                                enum lru_list l;
                                 mz = mem_cgroup_zoneinfo(mem, node, zid);
-                                /* drop all page_cgroup in active_list */
-                                mem_cgroup_force_empty_list(mem, mz, 1);
-                                /* drop all page_cgroup in inactive_list */
-                                mem_cgroup_force_empty_list(mem, mz, 0);
+                                for_each_lru(l)
+                                        mem_cgroup_force_empty_list(mem, mz, l);
                         }
+                cond_resched();
         }
         ret = 0;
 out:
@@ -1012,14 +962,27 @@ static int mem_control_stat_show(struct cgroup *cont, struct cftype *cft,
         }
         /* showing # of active pages */
         {
-                unsigned long active, inactive;
-
-                inactive = mem_cgroup_get_all_zonestat(mem_cont,
-                                                MEM_CGROUP_ZSTAT_INACTIVE);
-                active = mem_cgroup_get_all_zonestat(mem_cont,
-                                                MEM_CGROUP_ZSTAT_ACTIVE);
-                cb->fill(cb, "active", (active) * PAGE_SIZE);
-                cb->fill(cb, "inactive", (inactive) * PAGE_SIZE);
+                unsigned long active_anon, inactive_anon;
+                unsigned long active_file, inactive_file;
+                unsigned long unevictable;
+
+                inactive_anon = mem_cgroup_get_all_zonestat(mem_cont,
+                                                LRU_INACTIVE_ANON);
+                active_anon = mem_cgroup_get_all_zonestat(mem_cont,
+                                                LRU_ACTIVE_ANON);
+                inactive_file = mem_cgroup_get_all_zonestat(mem_cont,
+                                                LRU_INACTIVE_FILE);
+                active_file = mem_cgroup_get_all_zonestat(mem_cont,
+                                                LRU_ACTIVE_FILE);
+                unevictable = mem_cgroup_get_all_zonestat(mem_cont,
+                                                        LRU_UNEVICTABLE);
+
+                cb->fill(cb, "active_anon", (active_anon) * PAGE_SIZE);
+                cb->fill(cb, "inactive_anon", (inactive_anon) * PAGE_SIZE);
+                cb->fill(cb, "active_file", (active_file) * PAGE_SIZE);
+                cb->fill(cb, "inactive_file", (inactive_file) * PAGE_SIZE);
+                cb->fill(cb, "unevictable", unevictable * PAGE_SIZE);
+
         }
         return 0;
 }
@@ -1062,6 +1025,7 @@ static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node)
 {
         struct mem_cgroup_per_node *pn;
         struct mem_cgroup_per_zone *mz;
+        enum lru_list l;
         int zone, tmp = node;
         /*
          * This routine is called against possible nodes.
@@ -1082,9 +1046,9 @@ static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node)
 
         for (zone = 0; zone < MAX_NR_ZONES; zone++) {
                 mz = &pn->zoneinfo[zone];
-                INIT_LIST_HEAD(&mz->active_list);
-                INIT_LIST_HEAD(&mz->inactive_list);
                 spin_lock_init(&mz->lru_lock);
+                for_each_lru(l)
+                        INIT_LIST_HEAD(&mz->lists[l]);
         }
         return 0;
 }
@@ -1125,7 +1089,6 @@ mem_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont)
 
         if (unlikely((cont->parent) == NULL)) {
                 mem = &init_mem_cgroup;
-                page_cgroup_cache = KMEM_CACHE(page_cgroup, SLAB_PANIC);
         } else {
                 mem = mem_cgroup_alloc();
                 if (!mem)