1 files changed, 61 insertions, 28 deletions

diff --git a/mm/compaction.c b/mm/compaction.c
index 81e1eaa2a2cf..613c59e928cb 100644
--- a/mm/compaction.c
+++ b/mm/compaction.c
@@ -89,11 +89,6 @@ static void map_pages(struct list_head *list)
         list_splice(&tmp_list, list);
 }
 
-static inline bool migrate_async_suitable(int migratetype)
-{
-        return is_migrate_cma(migratetype) || migratetype == MIGRATE_MOVABLE;
-}
-
 #ifdef CONFIG_COMPACTION
 
 int PageMovable(struct page *page)
@@ -988,13 +983,26 @@ isolate_migratepages_range(struct compact_control *cc, unsigned long start_pfn,
 #endif /* CONFIG_COMPACTION || CONFIG_CMA */
 #ifdef CONFIG_COMPACTION
 
-/* Returns true if the page is within a block suitable for migration to */
-static bool suitable_migration_target(struct compact_control *cc,
+static bool suitable_migration_source(struct compact_control *cc,
                                                         struct page *page)
 {
-        if (cc->ignore_block_suitable)
+        int block_mt;
+
+        if ((cc->mode != MIGRATE_ASYNC) || !cc->direct_compaction)
                 return true;
 
+        block_mt = get_pageblock_migratetype(page);
+
+        if (cc->migratetype == MIGRATE_MOVABLE)
+                return is_migrate_movable(block_mt);
+        else
+                return block_mt == cc->migratetype;
+}
+
+/* Returns true if the page is within a block suitable for migration to */
+static bool suitable_migration_target(struct compact_control *cc,
+                                                        struct page *page)
+{
         /* If the page is a large free page, then disallow migration */
         if (PageBuddy(page)) {
                 /*
@@ -1006,8 +1014,11 @@ static bool suitable_migration_target(struct compact_control *cc,
                         return false;
         }
 
+        if (cc->ignore_block_suitable)
+                return true;
+
         /* If the block is MIGRATE_MOVABLE or MIGRATE_CMA, allow migration */
-        if (migrate_async_suitable(get_pageblock_migratetype(page)))
+        if (is_migrate_movable(get_pageblock_migratetype(page)))
                 return true;
 
         /* Otherwise skip the block */
@@ -1242,8 +1253,7 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,
                  * Async compaction is optimistic to see if the minimum amount
                  * of work satisfies the allocation.
                  */
-                if (cc->mode == MIGRATE_ASYNC &&
-                    !migrate_async_suitable(get_pageblock_migratetype(page)))
+                if (!suitable_migration_source(cc, page))
                         continue;
 
                 /* Perform the isolation */
@@ -1276,11 +1286,11 @@ static inline bool is_via_compact_memory(int order)
         return order == -1;
 }
 
-static enum compact_result __compact_finished(struct zone *zone, struct compact_control *cc,
-                            const int migratetype)
+static enum compact_result __compact_finished(struct zone *zone,
+                                                struct compact_control *cc)
 {
         unsigned int order;
-        unsigned long watermark;
+        const int migratetype = cc->migratetype;
 
         if (cc->contended || fatal_signal_pending(current))
                 return COMPACT_CONTENDED;
@@ -1308,12 +1318,16 @@ static enum compact_result __compact_finished(struct zone *zone, struct compact_
         if (is_via_compact_memory(cc->order))
                 return COMPACT_CONTINUE;
 
-        /* Compaction run is not finished if the watermark is not met */
-        watermark = zone->watermark[cc->alloc_flags & ALLOC_WMARK_MASK];
-
-        if (!zone_watermark_ok(zone, cc->order, watermark, cc->classzone_idx,
-                                                        cc->alloc_flags))
-                return COMPACT_CONTINUE;
+        if (cc->finishing_block) {
+                /*
+                 * We have finished the pageblock, but better check again that
+                 * we really succeeded.
+                 */
+                if (IS_ALIGNED(cc->migrate_pfn, pageblock_nr_pages))
+                        cc->finishing_block = false;
+                else
+                        return COMPACT_CONTINUE;
+        }
 
         /* Direct compactor: Is a suitable page free? */
         for (order = cc->order; order < MAX_ORDER; order++) {
@@ -1335,20 +1349,40 @@ static enum compact_result __compact_finished(struct zone *zone, struct compact_
                  * other migratetype buddy lists.
                  */
                 if (find_suitable_fallback(area, order, migratetype,
-                                                true, &can_steal) != -1)
-                        return COMPACT_SUCCESS;
+                                                true, &can_steal) != -1) {
+
+                        /* movable pages are OK in any pageblock */
+                        if (migratetype == MIGRATE_MOVABLE)
+                                return COMPACT_SUCCESS;
+
+                        /*
+                         * We are stealing for a non-movable allocation. Make
+                         * sure we finish compacting the current pageblock
+                         * first so it is as free as possible and we won't
+                         * have to steal another one soon. This only applies
+                         * to sync compaction, as async compaction operates
+                         * on pageblocks of the same migratetype.
+                         */
+                        if (cc->mode == MIGRATE_ASYNC ||
+                                        IS_ALIGNED(cc->migrate_pfn,
+                                                        pageblock_nr_pages)) {
+                                return COMPACT_SUCCESS;
+                        }
+
+                        cc->finishing_block = true;
+                        return COMPACT_CONTINUE;
+                }
         }
 
         return COMPACT_NO_SUITABLE_PAGE;
 }
 
 static enum compact_result compact_finished(struct zone *zone,
-                        struct compact_control *cc,
-                        const int migratetype)
+                        struct compact_control *cc)
 {
         int ret;
 
-        ret = __compact_finished(zone, cc, migratetype);
+        ret = __compact_finished(zone, cc);
         trace_mm_compaction_finished(zone, cc->order, ret);
         if (ret == COMPACT_NO_SUITABLE_PAGE)
                 ret = COMPACT_CONTINUE;
@@ -1481,9 +1515,9 @@ static enum compact_result compact_zone(struct zone *zone, struct compact_contro
         enum compact_result ret;
         unsigned long start_pfn = zone->zone_start_pfn;
         unsigned long end_pfn = zone_end_pfn(zone);
-        const int migratetype = gfpflags_to_migratetype(cc->gfp_mask);
         const bool sync = cc->mode != MIGRATE_ASYNC;
 
+        cc->migratetype = gfpflags_to_migratetype(cc->gfp_mask);
         ret = compaction_suitable(zone, cc->order, cc->alloc_flags,
                                                         cc->classzone_idx);
         /* Compaction is likely to fail */
@@ -1533,8 +1567,7 @@ static enum compact_result compact_zone(struct zone *zone, struct compact_contro
 
         migrate_prep_local();
 
-        while ((ret = compact_finished(zone, cc, migratetype)) ==
-                                                COMPACT_CONTINUE) {
+        while ((ret = compact_finished(zone, cc)) == COMPACT_CONTINUE) {
                 int err;
 
                 switch (isolate_migratepages(zone, cc)) {