mm, vmscan: move LRU lists to node

This moves the LRU lists from the zone to the node and related data such
as counters, tracing, congestion tracking and writeback tracking.

Unfortunately, due to reclaim and compaction retry logic, it is
necessary to account for the number of LRU pages on both zone and node
logic.  Most reclaim logic is based on the node counters but the retry
logic uses the zone counters which do not distinguish inactive and
active sizes.  It would be possible to leave the LRU counters on a
per-zone basis but it's a heavier calculation across multiple cache
lines that is much more frequent than the retry checks.

Other than the LRU counters, this is mostly a mechanical patch but note
that it introduces a number of anomalies.  For example, the scans are
per-zone but using per-node counters.  We also mark a node as congested
when a zone is congested.  This causes weird problems that are fixed
later but is easier to review.

In the event that there is excessive overhead on 32-bit systems due to
the nodes being on LRU then there are two potential solutions

1. Long-term isolation of highmem pages when reclaim is lowmem

   When pages are skipped, they are immediately added back onto the LRU
   list. If lowmem reclaim persisted for long periods of time, the same
   highmem pages get continually scanned. The idea would be that lowmem
   keeps those pages on a separate list until a reclaim for highmem pages
   arrives that splices the highmem pages back onto the LRU. It potentially
   could be implemented similar to the UNEVICTABLE list.

   That would reduce the skip rate with the potential corner case is that
   highmem pages have to be scanned and reclaimed to free lowmem slab pages.

2. Linear scan lowmem pages if the initial LRU shrink fails

   This will break LRU ordering but may be preferable and faster during
   memory pressure than skipping LRU pages.

Link: http://lkml.kernel.org/r/1467970510-21195-4-git-send-email-mgorman@techsingularity.net
Signed-off-by: Mel Gorman <mgorman@techsingularity.net>
Acked-by: Johannes Weiner <hannes@cmpxchg.org>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Cc: Hillf Danton <hillf.zj@alibaba-inc.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Rik van Riel <riel@surriel.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

1 files changed, 37 insertions, 31 deletions

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 5760c626c309..35e2d0f9d44f 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -1078,9 +1078,9 @@ static void free_pcppages_bulk(struct zone *zone, int count,
 
         spin_lock(&zone->lock);
         isolated_pageblocks = has_isolate_pageblock(zone);
-        nr_scanned = zone_page_state(zone, NR_PAGES_SCANNED);
+        nr_scanned = node_page_state(zone->zone_pgdat, NR_PAGES_SCANNED);
         if (nr_scanned)
-                __mod_zone_page_state(zone, NR_PAGES_SCANNED, -nr_scanned);
+                __mod_node_page_state(zone->zone_pgdat, NR_PAGES_SCANNED, -nr_scanned);
 
         while (count) {
                 struct page *page;
@@ -1135,9 +1135,9 @@ static void free_one_page(struct zone *zone,
 {
         unsigned long nr_scanned;
         spin_lock(&zone->lock);
-        nr_scanned = zone_page_state(zone, NR_PAGES_SCANNED);
+        nr_scanned = node_page_state(zone->zone_pgdat, NR_PAGES_SCANNED);
         if (nr_scanned)
-                __mod_zone_page_state(zone, NR_PAGES_SCANNED, -nr_scanned);
+                __mod_node_page_state(zone->zone_pgdat, NR_PAGES_SCANNED, -nr_scanned);
 
         if (unlikely(has_isolate_pageblock(zone) ||
                 is_migrate_isolate(migratetype))) {
@@ -4288,6 +4288,7 @@ void show_free_areas(unsigned int filter)
         unsigned long free_pcp = 0;
         int cpu;
         struct zone *zone;
+        pg_data_t *pgdat;
 
         for_each_populated_zone(zone) {
                 if (skip_free_areas_node(filter, zone_to_nid(zone)))
@@ -4306,13 +4307,13 @@ void show_free_areas(unsigned int filter)
                 " anon_thp: %lu shmem_thp: %lu shmem_pmdmapped: %lu\n"
 #endif
                 " free:%lu free_pcp:%lu free_cma:%lu\n",
-                global_page_state(NR_ACTIVE_ANON),
-                global_page_state(NR_INACTIVE_ANON),
-                global_page_state(NR_ISOLATED_ANON),
-                global_page_state(NR_ACTIVE_FILE),
-                global_page_state(NR_INACTIVE_FILE),
-                global_page_state(NR_ISOLATED_FILE),
-                global_page_state(NR_UNEVICTABLE),
+                global_node_page_state(NR_ACTIVE_ANON),
+                global_node_page_state(NR_INACTIVE_ANON),
+                global_node_page_state(NR_ISOLATED_ANON),
+                global_node_page_state(NR_ACTIVE_FILE),
+                global_node_page_state(NR_INACTIVE_FILE),
+                global_node_page_state(NR_ISOLATED_FILE),
+                global_node_page_state(NR_UNEVICTABLE),
                 global_page_state(NR_FILE_DIRTY),
                 global_page_state(NR_WRITEBACK),
                 global_page_state(NR_UNSTABLE_NFS),
@@ -4331,6 +4332,28 @@ void show_free_areas(unsigned int filter)
                 free_pcp,
                 global_page_state(NR_FREE_CMA_PAGES));
 
+        for_each_online_pgdat(pgdat) {
+                printk("Node %d"
+                        " active_anon:%lukB"
+                        " inactive_anon:%lukB"
+                        " active_file:%lukB"
+                        " inactive_file:%lukB"
+                        " unevictable:%lukB"
+                        " isolated(anon):%lukB"
+                        " isolated(file):%lukB"
+                        " all_unreclaimable? %s"
+                        "\n",
+                        pgdat->node_id,
+                        K(node_page_state(pgdat, NR_ACTIVE_ANON)),
+                        K(node_page_state(pgdat, NR_INACTIVE_ANON)),
+                        K(node_page_state(pgdat, NR_ACTIVE_FILE)),
+                        K(node_page_state(pgdat, NR_INACTIVE_FILE)),
+                        K(node_page_state(pgdat, NR_UNEVICTABLE)),
+                        K(node_page_state(pgdat, NR_ISOLATED_ANON)),
+                        K(node_page_state(pgdat, NR_ISOLATED_FILE)),
+                        !pgdat_reclaimable(pgdat) ? "yes" : "no");
+        }
+
         for_each_populated_zone(zone) {
                 int i;
 
@@ -4347,13 +4370,6 @@ void show_free_areas(unsigned int filter)
                         " min:%lukB"
                         " low:%lukB"
                         " high:%lukB"
-                        " active_anon:%lukB"
-                        " inactive_anon:%lukB"
-                        " active_file:%lukB"
-                        " inactive_file:%lukB"
-                        " unevictable:%lukB"
-                        " isolated(anon):%lukB"
-                        " isolated(file):%lukB"
                         " present:%lukB"
                         " managed:%lukB"
                         " mlocked:%lukB"
@@ -4376,21 +4392,13 @@ void show_free_areas(unsigned int filter)
                         " local_pcp:%ukB"
                         " free_cma:%lukB"
                         " writeback_tmp:%lukB"
-                        " pages_scanned:%lu"
-                        " all_unreclaimable? %s"
+                        " node_pages_scanned:%lu"
                         "\n",
                         zone->name,
                         K(zone_page_state(zone, NR_FREE_PAGES)),
                         K(min_wmark_pages(zone)),
                         K(low_wmark_pages(zone)),
                         K(high_wmark_pages(zone)),
-                        K(zone_page_state(zone, NR_ACTIVE_ANON)),
-                        K(zone_page_state(zone, NR_INACTIVE_ANON)),
-                        K(zone_page_state(zone, NR_ACTIVE_FILE)),
-                        K(zone_page_state(zone, NR_INACTIVE_FILE)),
-                        K(zone_page_state(zone, NR_UNEVICTABLE)),
-                        K(zone_page_state(zone, NR_ISOLATED_ANON)),
-                        K(zone_page_state(zone, NR_ISOLATED_FILE)),
                         K(zone->present_pages),
                         K(zone->managed_pages),
                         K(zone_page_state(zone, NR_MLOCK)),
@@ -4415,9 +4423,7 @@ void show_free_areas(unsigned int filter)
                         K(this_cpu_read(zone->pageset->pcp.count)),
                         K(zone_page_state(zone, NR_FREE_CMA_PAGES)),
                         K(zone_page_state(zone, NR_WRITEBACK_TEMP)),
-                        K(zone_page_state(zone, NR_PAGES_SCANNED)),
-                        (!zone_reclaimable(zone) ? "yes" : "no")
-                        );
+                        K(node_page_state(zone->zone_pgdat, NR_PAGES_SCANNED)));
                 printk("lowmem_reserve[]:");
                 for (i = 0; i < MAX_NR_ZONES; i++)
                         printk(" %ld", zone->lowmem_reserve[i]);
@@ -5967,7 +5973,7 @@ static void __paginginit free_area_init_core(struct pglist_data *pgdat)
                 /* For bootup, initialized properly in watermark setup */
                 mod_zone_page_state(zone, NR_ALLOC_BATCH, zone->managed_pages);
 
-                lruvec_init(&zone->lruvec);
+                lruvec_init(zone_lruvec(zone));
                 if (!size)
                         continue;