1 files changed, 63 insertions, 46 deletions

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index da5912b84551..a61706193c31 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -166,7 +166,6 @@ struct page_cgroup {
         struct list_head lru;           /* per cgroup LRU list */
         struct page *page;
         struct mem_cgroup *mem_cgroup;
-        int ref_cnt;                    /* cached, mapped, migrating */
         int flags;
 };
 #define PAGE_CGROUP_FLAG_CACHE  (0x1)   /* charged as cache */
@@ -185,6 +184,7 @@ static enum zone_type page_cgroup_zid(struct page_cgroup *pc)
 enum charge_type {
         MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
         MEM_CGROUP_CHARGE_TYPE_MAPPED,
+        MEM_CGROUP_CHARGE_TYPE_FORCE,   /* used by force_empty */
 };
 
 /*
@@ -552,9 +552,7 @@ retry:
          */
         if (pc) {
                 VM_BUG_ON(pc->page != page);
-                VM_BUG_ON(pc->ref_cnt <= 0);
-
-                pc->ref_cnt++;
+                VM_BUG_ON(!pc->mem_cgroup);
                 unlock_page_cgroup(page);
                 goto done;
         }
@@ -570,10 +568,7 @@ retry:
          * thread group leader migrates. It's possible that mm is not
          * set, if so charge the init_mm (happens for pagecache usage).
          */
-        if (!memcg) {
-                if (!mm)
-                        mm = &init_mm;
-
+        if (likely(!memcg)) {
                 rcu_read_lock();
                 mem = mem_cgroup_from_task(rcu_dereference(mm->owner));
                 /*
@@ -609,7 +604,6 @@ retry:
                 }
         }
 
-        pc->ref_cnt = 1;
         pc->mem_cgroup = mem;
         pc->page = page;
         /*
@@ -653,6 +647,17 @@ err:
 
 int mem_cgroup_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask)
 {
+        /*
+         * If already mapped, we don't have to account.
+         * If page cache, page->mapping has address_space.
+         * But page->mapping may have out-of-use anon_vma pointer,
+         * detecit it by PageAnon() check. newly-mapped-anon's page->mapping
+         * is NULL.
+         */
+        if (page_mapped(page) || (page->mapping && !PageAnon(page)))
+                return 0;
+        if (unlikely(!mm))
+                mm = &init_mm;
         return mem_cgroup_charge_common(page, mm, gfp_mask,
                                 MEM_CGROUP_CHARGE_TYPE_MAPPED, NULL);
 }
@@ -660,32 +665,17 @@ int mem_cgroup_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask)
 int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm,
                                 gfp_t gfp_mask)
 {
-        if (!mm)
+        if (unlikely(!mm))
                 mm = &init_mm;
         return mem_cgroup_charge_common(page, mm, gfp_mask,
                                 MEM_CGROUP_CHARGE_TYPE_CACHE, NULL);
 }
 
-int mem_cgroup_getref(struct page *page)
-{
-        struct page_cgroup *pc;
-
-        if (mem_cgroup_subsys.disabled)
-                return 0;
-
-        lock_page_cgroup(page);
-        pc = page_get_page_cgroup(page);
-        VM_BUG_ON(!pc);
-        pc->ref_cnt++;
-        unlock_page_cgroup(page);
-        return 0;
-}
-
 /*
- * Uncharging is always a welcome operation, we never complain, simply
- * uncharge.
+ * uncharge if !page_mapped(page)
  */
-void mem_cgroup_uncharge_page(struct page *page)
+static void
+__mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype)
 {
         struct page_cgroup *pc;
         struct mem_cgroup *mem;
@@ -704,29 +694,41 @@ void mem_cgroup_uncharge_page(struct page *page)
                 goto unlock;
 
         VM_BUG_ON(pc->page != page);
-        VM_BUG_ON(pc->ref_cnt <= 0);
 
-        if (--(pc->ref_cnt) == 0) {
-                mz = page_cgroup_zoneinfo(pc);
-                spin_lock_irqsave(&mz->lru_lock, flags);
-                __mem_cgroup_remove_list(mz, pc);
-                spin_unlock_irqrestore(&mz->lru_lock, flags);
+        if ((ctype == MEM_CGROUP_CHARGE_TYPE_MAPPED)
+            && ((pc->flags & PAGE_CGROUP_FLAG_CACHE)
+                || page_mapped(page)))
+                goto unlock;
 
-                page_assign_page_cgroup(page, NULL);
-                unlock_page_cgroup(page);
+        mz = page_cgroup_zoneinfo(pc);
+        spin_lock_irqsave(&mz->lru_lock, flags);
+        __mem_cgroup_remove_list(mz, pc);
+        spin_unlock_irqrestore(&mz->lru_lock, flags);
 
-                mem = pc->mem_cgroup;
-                res_counter_uncharge(&mem->res, PAGE_SIZE);
-                css_put(&mem->css);
+        page_assign_page_cgroup(page, NULL);
+        unlock_page_cgroup(page);
 
-                kmem_cache_free(page_cgroup_cache, pc);
-                return;
-        }
+        mem = pc->mem_cgroup;
+        res_counter_uncharge(&mem->res, PAGE_SIZE);
+        css_put(&mem->css);
 
+        kmem_cache_free(page_cgroup_cache, pc);
+        return;
 unlock:
         unlock_page_cgroup(page);
 }
 
+void mem_cgroup_uncharge_page(struct page *page)
+{
+        __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_MAPPED);
+}
+
+void mem_cgroup_uncharge_cache_page(struct page *page)
+{
+        VM_BUG_ON(page_mapped(page));
+        __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_CACHE);
+}
+
 /*
  * Before starting migration, account against new page.
  */
@@ -757,15 +759,29 @@ int mem_cgroup_prepare_migration(struct page *page, struct page *newpage)
         return ret;
 }
 
-/* remove redundant charge */
+/* remove redundant charge if migration failed*/
 void mem_cgroup_end_migration(struct page *newpage)
 {
-        mem_cgroup_uncharge_page(newpage);
+        /*
+         * At success, page->mapping is not NULL.
+         * special rollback care is necessary when
+         * 1. at migration failure. (newpage->mapping is cleared in this case)
+         * 2. the newpage was moved but not remapped again because the task
+         *    exits and the newpage is obsolete. In this case, the new page
+         *    may be a swapcache. So, we just call mem_cgroup_uncharge_page()
+         *    always for avoiding mess. The  page_cgroup will be removed if
+         *    unnecessary. File cache pages is still on radix-tree. Don't
+         *    care it.
+         */
+        if (!newpage->mapping)
+                __mem_cgroup_uncharge_common(newpage,
+                                         MEM_CGROUP_CHARGE_TYPE_FORCE);
+        else if (PageAnon(newpage))
+                mem_cgroup_uncharge_page(newpage);
 }
 
 /*
  * This routine traverse page_cgroup in given list and drop them all.
- * This routine ignores page_cgroup->ref_cnt.
  * *And* this routine doesn't reclaim page itself, just removes page_cgroup.
  */
 #define FORCE_UNCHARGE_BATCH    (128)
@@ -795,7 +811,8 @@ static void mem_cgroup_force_empty_list(struct mem_cgroup *mem,
                  * if it's under page migration.
                  */
                 if (PageLRU(page)) {
-                        mem_cgroup_uncharge_page(page);
+                        __mem_cgroup_uncharge_common(page,
+                                        MEM_CGROUP_CHARGE_TYPE_FORCE);
                         put_page(page);
                         if (--count <= 0) {
                                 count = FORCE_UNCHARGE_BATCH;