mm: memcontrol: use page lists for uncharge batching

Pages are now uncharged at release time, and all sources of batched
uncharges operate on lists of pages.  Directly use those lists, and
get rid of the per-task batching state.

This also batches statistics accounting, in addition to the res
counter charges, to reduce IRQ-disabling and re-enabling.

Signed-off-by: Johannes Weiner <hannes@cmpxchg.org>
Acked-by: Michal Hocko <mhocko@suse.cz>
Cc: Hugh Dickins <hughd@google.com>
Cc: Tejun Heo <tj@kernel.org>
Cc: Vladimir Davydov <vdavydov@parallels.com>
Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: Vladimir Davydov <vdavydov@parallels.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

1 files changed, 5 insertions, 7 deletions

diff --git a/mm/vmscan.c b/mm/vmscan.c
index 7068e838d22b..2836b5373b2e 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -822,7 +822,6 @@ static unsigned long shrink_page_list(struct list_head *page_list,
 
         cond_resched();
 
-        mem_cgroup_uncharge_start();
         while (!list_empty(page_list)) {
                 struct address_space *mapping;
                 struct page *page;
@@ -1103,7 +1102,6 @@ static unsigned long shrink_page_list(struct list_head *page_list,
                  */
                 __clear_page_locked(page);
 free_it:
-                mem_cgroup_uncharge(page);
                 nr_reclaimed++;
 
                 /*
@@ -1133,8 +1131,8 @@ keep:
                 list_add(&page->lru, &ret_pages);
                 VM_BUG_ON_PAGE(PageLRU(page) || PageUnevictable(page), page);
         }
-        mem_cgroup_uncharge_end();
 
+        mem_cgroup_uncharge_list(&free_pages);
         free_hot_cold_page_list(&free_pages, true);
 
         list_splice(&ret_pages, page_list);
@@ -1437,10 +1435,9 @@ putback_inactive_pages(struct lruvec *lruvec, struct list_head *page_list)
                         __ClearPageActive(page);
                         del_page_from_lru_list(page, lruvec, lru);
 
-                        mem_cgroup_uncharge(page);
-
                         if (unlikely(PageCompound(page))) {
                                 spin_unlock_irq(&zone->lru_lock);
+                                mem_cgroup_uncharge(page);
                                 (*get_compound_page_dtor(page))(page);
                                 spin_lock_irq(&zone->lru_lock);
                         } else
@@ -1548,6 +1545,7 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec,
 
         spin_unlock_irq(&zone->lru_lock);
 
+        mem_cgroup_uncharge_list(&page_list);
         free_hot_cold_page_list(&page_list, true);
 
         /*
@@ -1660,10 +1658,9 @@ static void move_active_pages_to_lru(struct lruvec *lruvec,
                         __ClearPageActive(page);
                         del_page_from_lru_list(page, lruvec, lru);
 
-                        mem_cgroup_uncharge(page);
-
                         if (unlikely(PageCompound(page))) {
                                 spin_unlock_irq(&zone->lru_lock);
+                                mem_cgroup_uncharge(page);
                                 (*get_compound_page_dtor(page))(page);
                                 spin_lock_irq(&zone->lru_lock);
                         } else
@@ -1771,6 +1768,7 @@ static void shrink_active_list(unsigned long nr_to_scan,
         __mod_zone_page_state(zone, NR_ISOLATED_ANON + file, -nr_taken);
         spin_unlock_irq(&zone->lru_lock);
 
+        mem_cgroup_uncharge_list(&l_hold);
         free_hot_cold_page_list(&l_hold, true);
 }