1 files changed, 80 insertions, 9 deletions

diff --git a/mm/compaction.c b/mm/compaction.c
index 329973a1ae45..7487067b4613 100644
--- a/mm/compaction.c
+++ b/mm/compaction.c
@@ -638,12 +638,13 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
 {
         struct zone *zone = cc->zone;
         unsigned long nr_scanned = 0, nr_isolated = 0;
-        struct list_head *migratelist = &cc->migratepages;
         struct lruvec *lruvec;
         unsigned long flags = 0;
         bool locked = false;
         struct page *page = NULL, *valid_page = NULL;
         unsigned long start_pfn = low_pfn;
+        bool skip_on_failure = false;
+        unsigned long next_skip_pfn = 0;
 
         /*
          * Ensure that there are not too many pages isolated from the LRU
@@ -664,10 +665,37 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
         if (compact_should_abort(cc))
                 return 0;
 
+        if (cc->direct_compaction && (cc->mode == MIGRATE_ASYNC)) {
+                skip_on_failure = true;
+                next_skip_pfn = block_end_pfn(low_pfn, cc->order);
+        }
+
         /* Time to isolate some pages for migration */
         for (; low_pfn < end_pfn; low_pfn++) {
                 bool is_lru;
 
+                if (skip_on_failure && low_pfn >= next_skip_pfn) {
+                        /*
+                         * We have isolated all migration candidates in the
+                         * previous order-aligned block, and did not skip it due
+                         * to failure. We should migrate the pages now and
+                         * hopefully succeed compaction.
+                         */
+                        if (nr_isolated)
+                                break;
+
+                        /*
+                         * We failed to isolate in the previous order-aligned
+                         * block. Set the new boundary to the end of the
+                         * current block. Note we can't simply increase
+                         * next_skip_pfn by 1 << order, as low_pfn might have
+                         * been incremented by a higher number due to skipping
+                         * a compound or a high-order buddy page in the
+                         * previous loop iteration.
+                         */
+                        next_skip_pfn = block_end_pfn(low_pfn, cc->order);
+                }
+
                 /*
                  * Periodically drop the lock (if held) regardless of its
                  * contention, to give chance to IRQs. Abort async compaction
@@ -679,7 +707,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
                         break;
 
                 if (!pfn_valid_within(low_pfn))
-                        continue;
+                        goto isolate_fail;
                 nr_scanned++;
 
                 page = pfn_to_page(low_pfn);
@@ -734,11 +762,11 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
                         if (likely(comp_order < MAX_ORDER))
                                 low_pfn += (1UL << comp_order) - 1;
 
-                        continue;
+                        goto isolate_fail;
                 }
 
                 if (!is_lru)
-                        continue;
+                        goto isolate_fail;
 
                 /*
                  * Migration will fail if an anonymous page is pinned in memory,
@@ -747,7 +775,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
                  */
                 if (!page_mapping(page) &&
                     page_count(page) > page_mapcount(page))
-                        continue;
+                        goto isolate_fail;
 
                 /* If we already hold the lock, we can skip some rechecking */
                 if (!locked) {
@@ -758,7 +786,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
 
                         /* Recheck PageLRU and PageCompound under lock */
                         if (!PageLRU(page))
-                                continue;
+                                goto isolate_fail;
 
                         /*
                          * Page become compound since the non-locked check,
@@ -767,7 +795,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
                          */
                         if (unlikely(PageCompound(page))) {
                                 low_pfn += (1UL << compound_order(page)) - 1;
-                                continue;
+                                goto isolate_fail;
                         }
                 }
 
@@ -775,7 +803,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
 
                 /* Try isolate the page */
                 if (__isolate_lru_page(page, isolate_mode) != 0)
-                        continue;
+                        goto isolate_fail;
 
                 VM_BUG_ON_PAGE(PageCompound(page), page);
 
@@ -783,7 +811,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
                 del_page_from_lru_list(page, lruvec, page_lru(page));
 
 isolate_success:
-                list_add(&page->lru, migratelist);
+                list_add(&page->lru, &cc->migratepages);
                 cc->nr_migratepages++;
                 nr_isolated++;
 
@@ -801,6 +829,37 @@ isolate_success:
                         ++low_pfn;
                         break;
                 }
+
+                continue;
+isolate_fail:
+                if (!skip_on_failure)
+                        continue;
+
+                /*
+                 * We have isolated some pages, but then failed. Release them
+                 * instead of migrating, as we cannot form the cc->order buddy
+                 * page anyway.
+                 */
+                if (nr_isolated) {
+                        if (locked) {
+                                spin_unlock_irqrestore(&zone->lru_lock, flags);
+                                locked = false;
+                        }
+                        acct_isolated(zone, cc);
+                        putback_movable_pages(&cc->migratepages);
+                        cc->nr_migratepages = 0;
+                        cc->last_migrated_pfn = 0;
+                        nr_isolated = 0;
+                }
+
+                if (low_pfn < next_skip_pfn) {
+                        low_pfn = next_skip_pfn - 1;
+                        /*
+                         * The check near the loop beginning would have updated
+                         * next_skip_pfn too, but this is a bit simpler.
+                         */
+                        next_skip_pfn += 1UL << cc->order;
+                }
         }
 
         /*
@@ -1401,6 +1460,18 @@ static int compact_zone(struct zone *zone, struct compact_control *cc)
                                 ret = COMPACT_CONTENDED;
                                 goto out;
                         }
+                        /*
+                         * We failed to migrate at least one page in the current
+                         * order-aligned block, so skip the rest of it.
+                         */
+                        if (cc->direct_compaction &&
+                                                (cc->mode == MIGRATE_ASYNC)) {
+                                cc->migrate_pfn = block_end_pfn(
+                                                cc->migrate_pfn - 1, cc->order);
+                                /* Draining pcplists is useless in this case */
+                                cc->last_migrated_pfn = 0;
+
+                        }
                 }
 
 check_drain: