mm: make per-memcg LRU lists exclusive

Now that all code that operated on global per-zone LRU lists is
converted to operate on per-memory cgroup LRU lists instead, there is no
reason to keep the double-LRU scheme around any longer.

The pc->lru member is removed and page->lru is linked directly to the
per-memory cgroup LRU lists, which removes two pointers from a
descriptor that exists for every page frame in the system.

Signed-off-by: Johannes Weiner <jweiner@redhat.com>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Ying Han <yinghan@google.com>
Reviewed-by: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Reviewed-by: Michal Hocko <mhocko@suse.cz>
Reviewed-by: Kirill A. Shutemov <kirill@shutemov.name>
Cc: Daisuke Nishimura <nishimura@mxp.nes.nec.co.jp>
Cc: Balbir Singh <bsingharora@gmail.com>
Cc: Greg Thelen <gthelen@google.com>
Cc: Michel Lespinasse <walken@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Christoph Hellwig <hch@infradead.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

1 files changed, 29 insertions, 35 deletions

diff --git a/mm/vmscan.c b/mm/vmscan.c
index 93cdc44a1693..813aae820a27 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -1139,15 +1139,14 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
 
                 switch (__isolate_lru_page(page, mode, file)) {
                 case 0:
+                        mem_cgroup_lru_del(page);
                         list_move(&page->lru, dst);
-                        mem_cgroup_del_lru(page);
                         nr_taken += hpage_nr_pages(page);
                         break;
 
                 case -EBUSY:
                         /* else it is being freed elsewhere */
                         list_move(&page->lru, src);
-                        mem_cgroup_rotate_lru_list(page, page_lru(page));
                         continue;
 
                 default:
@@ -1197,8 +1196,8 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
                                 break;
 
                         if (__isolate_lru_page(cursor_page, mode, file) == 0) {
+                                mem_cgroup_lru_del(cursor_page);
                                 list_move(&cursor_page->lru, dst);
-                                mem_cgroup_del_lru(cursor_page);
                                 nr_taken += hpage_nr_pages(cursor_page);
                                 nr_lumpy_taken++;
                                 if (PageDirty(cursor_page))
@@ -1239,18 +1238,20 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
         return nr_taken;
 }
 
-static unsigned long isolate_pages_global(unsigned long nr,
-                                        struct list_head *dst,
-                                        unsigned long *scanned, int order,
-                                        isolate_mode_t mode,
-                                        struct zone *z, int active, int file)
+static unsigned long isolate_pages(unsigned long nr, struct mem_cgroup_zone *mz,
+                                   struct list_head *dst,
+                                   unsigned long *scanned, int order,
+                                   isolate_mode_t mode, int active, int file)
 {
+        struct lruvec *lruvec;
         int lru = LRU_BASE;
+
+        lruvec = mem_cgroup_zone_lruvec(mz->zone, mz->mem_cgroup);
         if (active)
                 lru += LRU_ACTIVE;
         if (file)
                 lru += LRU_FILE;
-        return isolate_lru_pages(nr, &z->lruvec.lists[lru], dst,
+        return isolate_lru_pages(nr, &lruvec->lists[lru], dst,
                                  scanned, order, mode, file);
 }
 
@@ -1518,14 +1519,9 @@ shrink_inactive_list(unsigned long nr_to_scan, struct mem_cgroup_zone *mz,
 
         spin_lock_irq(&zone->lru_lock);
 
-        if (scanning_global_lru(mz)) {
-                nr_taken = isolate_pages_global(nr_to_scan, &page_list,
-                        &nr_scanned, sc->order, reclaim_mode, zone, 0, file);
-        } else {
-                nr_taken = mem_cgroup_isolate_pages(nr_to_scan, &page_list,
-                        &nr_scanned, sc->order, reclaim_mode, zone,
-                        mz->mem_cgroup, 0, file);
-        }
+        nr_taken = isolate_pages(nr_to_scan, mz, &page_list,
+                                 &nr_scanned, sc->order,
+                                 reclaim_mode, 0, file);
         if (global_reclaim(sc)) {
                 zone->pages_scanned += nr_scanned;
                 if (current_is_kswapd())
@@ -1625,13 +1621,15 @@ static void move_active_pages_to_lru(struct zone *zone,
         pagevec_init(&pvec, 1);
 
         while (!list_empty(list)) {
+                struct lruvec *lruvec;
+
                 page = lru_to_page(list);
 
                 VM_BUG_ON(PageLRU(page));
                 SetPageLRU(page);
 
-                list_move(&page->lru, &zone->lruvec.lists[lru]);
-                mem_cgroup_add_lru_list(page, lru);
+                lruvec = mem_cgroup_lru_add_list(zone, page, lru);
+                list_move(&page->lru, &lruvec->lists[lru]);
                 pgmoved += hpage_nr_pages(page);
 
                 if (!pagevec_add(&pvec, page) || list_empty(list)) {
@@ -1672,17 +1670,10 @@ static void shrink_active_list(unsigned long nr_pages,
                 reclaim_mode |= ISOLATE_CLEAN;
 
         spin_lock_irq(&zone->lru_lock);
-        if (scanning_global_lru(mz)) {
-                nr_taken = isolate_pages_global(nr_pages, &l_hold,
-                                                &pgscanned, sc->order,
-                                                reclaim_mode, zone,
-                                                1, file);
-        } else {
-                nr_taken = mem_cgroup_isolate_pages(nr_pages, &l_hold,
-                                                &pgscanned, sc->order,
-                                                reclaim_mode, zone,
-                                                mz->mem_cgroup, 1, file);
-        }
+
+        nr_taken = isolate_pages(nr_pages, mz, &l_hold,
+                                 &pgscanned, sc->order,
+                                 reclaim_mode, 1, file);
 
         if (global_reclaim(sc))
                 zone->pages_scanned += pgscanned;
@@ -3440,16 +3431,18 @@ int page_evictable(struct page *page, struct vm_area_struct *vma)
  */
 static void check_move_unevictable_page(struct page *page, struct zone *zone)
 {
-        VM_BUG_ON(PageActive(page));
+        struct lruvec *lruvec;
 
+        VM_BUG_ON(PageActive(page));
 retry:
         ClearPageUnevictable(page);
         if (page_evictable(page, NULL)) {
                 enum lru_list l = page_lru_base_type(page);
 
                 __dec_zone_state(zone, NR_UNEVICTABLE);
-                list_move(&page->lru, &zone->lruvec.lists[l]);
-                mem_cgroup_move_lists(page, LRU_UNEVICTABLE, l);
+                lruvec = mem_cgroup_lru_move_lists(zone, page,
+                                                   LRU_UNEVICTABLE, l);
+                list_move(&page->lru, &lruvec->lists[l]);
                 __inc_zone_state(zone, NR_INACTIVE_ANON + l);
                 __count_vm_event(UNEVICTABLE_PGRESCUED);
         } else {
@@ -3457,8 +3450,9 @@ retry:
                  * rotate unevictable list
                  */
                 SetPageUnevictable(page);
-                list_move(&page->lru, &zone->lruvec.lists[LRU_UNEVICTABLE]);
-                mem_cgroup_rotate_lru_list(page, LRU_UNEVICTABLE);
+                lruvec = mem_cgroup_lru_move_lists(zone, page, LRU_UNEVICTABLE,
+                                                   LRU_UNEVICTABLE);
+                list_move(&page->lru, &lruvec->lists[LRU_UNEVICTABLE]);
                 if (page_evictable(page, NULL))
                         goto retry;
         }