5 files changed, 43 insertions, 23 deletions

diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 8e02b3750fe0..d2c50ab6ae40 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -630,6 +630,8 @@ typedef struct pglist_data {
         int kswapd_order;
         enum zone_type kswapd_classzone_idx;
 
+        int kswapd_failures;            /* Number of 'reclaimed == 0' runs */
+
 #ifdef CONFIG_COMPACTION
         int kcompactd_max_order;
         enum zone_type kcompactd_classzone_idx;
diff --git a/mm/internal.h b/mm/internal.h
index 266efaeaa370..e5a0e0ec2177 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -81,6 +81,12 @@ static inline void set_page_refcounted(struct page *page)
 extern unsigned long highest_memmap_pfn;
 
 /*
+ * Maximum number of reclaim retries without progress before the OOM
+ * killer is consider the only way forward.
+ */
+#define MAX_RECLAIM_RETRIES 16
+
+/*
  * in mm/vmscan.c:
  */
 extern int isolate_lru_page(struct page *page);
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index bd01501efab9..42c0543e46c3 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -3522,12 +3522,6 @@ bool gfp_pfmemalloc_allowed(gfp_t gfp_mask)
 }
 
 /*
- * Maximum number of reclaim retries without any progress before OOM killer
- * is consider as the only way to move forward.
- */
-#define MAX_RECLAIM_RETRIES 16
-
-/*
  * Checks whether it makes sense to retry the reclaim to make a forward progress
  * for the given allocation request.
  * The reclaim feedback represented by did_some_progress (any progress during
@@ -4534,7 +4528,8 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask)
                         K(node_page_state(pgdat, NR_WRITEBACK_TEMP)),
                         K(node_page_state(pgdat, NR_UNSTABLE_NFS)),
                         node_page_state(pgdat, NR_PAGES_SCANNED),
-                        !pgdat_reclaimable(pgdat) ? "yes" : "no");
+                        pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES ?
+                                "yes" : "no");
         }
 
         for_each_populated_zone(zone) {
diff --git a/mm/vmscan.c b/mm/vmscan.c
index bc8031ef994d..667644e53b5c 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -2620,6 +2620,15 @@ static bool shrink_node(pg_data_t *pgdat, struct scan_control *sc)
         } while (should_continue_reclaim(pgdat, sc->nr_reclaimed - nr_reclaimed,
                                          sc->nr_scanned - nr_scanned, sc));
 
+        /*
+         * Kswapd gives up on balancing particular nodes after too
+         * many failures to reclaim anything from them and goes to
+         * sleep. On reclaim progress, reset the failure counter. A
+         * successful direct reclaim run will revive a dormant kswapd.
+         */
+        if (reclaimable)
+                pgdat->kswapd_failures = 0;
+
         return reclaimable;
 }
 
@@ -2694,10 +2703,6 @@ static void shrink_zones(struct zonelist *zonelist, struct scan_control *sc)
                                                  GFP_KERNEL | __GFP_HARDWALL))
                                 continue;
 
-                        if (sc->priority != DEF_PRIORITY &&
-                            !pgdat_reclaimable(zone->zone_pgdat))
-                                continue;       /* Let kswapd poll it */
-
                         /*
                          * If we already have plenty of memory free for
                          * compaction in this zone, don't free any more.
@@ -2817,7 +2822,7 @@ retry:
         return 0;
 }
 
-static bool pfmemalloc_watermark_ok(pg_data_t *pgdat)
+static bool allow_direct_reclaim(pg_data_t *pgdat)
 {
         struct zone *zone;
         unsigned long pfmemalloc_reserve = 0;
@@ -2825,6 +2830,9 @@ static bool pfmemalloc_watermark_ok(pg_data_t *pgdat)
         int i;
         bool wmark_ok;
 
+        if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES)
+                return true;
+
         for (i = 0; i <= ZONE_NORMAL; i++) {
                 zone = &pgdat->node_zones[i];
                 if (!managed_zone(zone) ||
@@ -2905,7 +2913,7 @@ static bool throttle_direct_reclaim(gfp_t gfp_mask, struct zonelist *zonelist,
 
                 /* Throttle based on the first usable node */
                 pgdat = zone->zone_pgdat;
-                if (pfmemalloc_watermark_ok(pgdat))
+                if (allow_direct_reclaim(pgdat))
                         goto out;
                 break;
         }
@@ -2927,14 +2935,14 @@ static bool throttle_direct_reclaim(gfp_t gfp_mask, struct zonelist *zonelist,
          */
         if (!(gfp_mask & __GFP_FS)) {
                 wait_event_interruptible_timeout(pgdat->pfmemalloc_wait,
-                        pfmemalloc_watermark_ok(pgdat), HZ);
+                        allow_direct_reclaim(pgdat), HZ);
 
                 goto check_pending;
         }
 
         /* Throttle until kswapd wakes the process */
         wait_event_killable(zone->zone_pgdat->pfmemalloc_wait,
-                pfmemalloc_watermark_ok(pgdat));
+                allow_direct_reclaim(pgdat));
 
 check_pending:
         if (fatal_signal_pending(current))
@@ -3114,7 +3122,7 @@ static bool prepare_kswapd_sleep(pg_data_t *pgdat, int order, int classzone_idx)
 
         /*
          * The throttled processes are normally woken up in balance_pgdat() as
-         * soon as pfmemalloc_watermark_ok() is true. But there is a potential
+         * soon as allow_direct_reclaim() is true. But there is a potential
          * race between when kswapd checks the watermarks and a process gets
          * throttled. There is also a potential race if processes get
          * throttled, kswapd wakes, a large process exits thereby balancing the
@@ -3128,6 +3136,10 @@ static bool prepare_kswapd_sleep(pg_data_t *pgdat, int order, int classzone_idx)
         if (waitqueue_active(&pgdat->pfmemalloc_wait))
                 wake_up_all(&pgdat->pfmemalloc_wait);
 
+        /* Hopeless node, leave it to direct reclaim */
+        if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES)
+                return true;
+
         for (i = 0; i <= classzone_idx; i++) {
                 struct zone *zone = pgdat->node_zones + i;
 
@@ -3214,9 +3226,9 @@ static int balance_pgdat(pg_data_t *pgdat, int order, int classzone_idx)
         count_vm_event(PAGEOUTRUN);
 
         do {
+                unsigned long nr_reclaimed = sc.nr_reclaimed;
                 bool raise_priority = true;
 
-                sc.nr_reclaimed = 0;
                 sc.reclaim_idx = classzone_idx;
 
                 /*
@@ -3295,7 +3307,7 @@ static int balance_pgdat(pg_data_t *pgdat, int order, int classzone_idx)
                  * able to safely make forward progress. Wake them
                  */
                 if (waitqueue_active(&pgdat->pfmemalloc_wait) &&
-                                pfmemalloc_watermark_ok(pgdat))
+                                allow_direct_reclaim(pgdat))
                         wake_up_all(&pgdat->pfmemalloc_wait);
 
                 /* Check if kswapd should be suspending */
@@ -3306,10 +3318,14 @@ static int balance_pgdat(pg_data_t *pgdat, int order, int classzone_idx)
                  * Raise priority if scanning rate is too low or there was no
                  * progress in reclaiming pages
                  */
-                if (raise_priority || !sc.nr_reclaimed)
+                nr_reclaimed = sc.nr_reclaimed - nr_reclaimed;
+                if (raise_priority || !nr_reclaimed)
                         sc.priority--;
         } while (sc.priority >= 1);
 
+        if (!sc.nr_reclaimed)
+                pgdat->kswapd_failures++;
+
 out:
         /*
          * Return the order kswapd stopped reclaiming at as
@@ -3509,6 +3525,10 @@ void wakeup_kswapd(struct zone *zone, int order, enum zone_type classzone_idx)
         if (!waitqueue_active(&pgdat->kswapd_wait))
                 return;
 
+        /* Hopeless node, leave it to direct reclaim */
+        if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES)
+                return;
+
         /* Only wake kswapd if all zones are unbalanced */
         for (z = 0; z <= classzone_idx; z++) {
                 zone = pgdat->node_zones + z;
@@ -3779,9 +3799,6 @@ int node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order)
             sum_zone_node_page_state(pgdat->node_id, NR_SLAB_RECLAIMABLE) <= pgdat->min_slab_pages)
                 return NODE_RECLAIM_FULL;
 
-        if (!pgdat_reclaimable(pgdat))
-                return NODE_RECLAIM_FULL;
-
         /*
          * Do not scan if the allocation should not be delayed.
          */
diff --git a/mm/vmstat.c b/mm/vmstat.c
index 5a4f5c5a31e8..baee70dafba8 100644
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -1425,7 +1425,7 @@ static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat,
                    "\n  node_unreclaimable:  %u"
                    "\n  start_pfn:           %lu"
                    "\n  node_inactive_ratio: %u",
-                   !pgdat_reclaimable(zone->zone_pgdat),
+                   pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES,
                    zone->zone_start_pfn,
                    zone->zone_pgdat->inactive_ratio);
         seq_putc(m, '\n');