mm, compaction: capture a page under direct compaction

Compaction is inherently race-prone as a suitable page freed during
compaction can be allocated by any parallel task.  This patch uses a
capture_control structure to isolate a page immediately when it is freed
by a direct compactor in the slow path of the page allocator.  The
intent is to avoid redundant scanning.

                                     5.0.0-rc1              5.0.0-rc1
                               selective-v3r17          capture-v3r19
Amean     fault-both-1         0.00 (   0.00%)        0.00 *   0.00%*
Amean     fault-both-3      2582.11 (   0.00%)     2563.68 (   0.71%)
Amean     fault-both-5      4500.26 (   0.00%)     4233.52 (   5.93%)
Amean     fault-both-7      5819.53 (   0.00%)     6333.65 (  -8.83%)
Amean     fault-both-12     9321.18 (   0.00%)     9759.38 (  -4.70%)
Amean     fault-both-18     9782.76 (   0.00%)    10338.76 (  -5.68%)
Amean     fault-both-24    15272.81 (   0.00%)    13379.55 *  12.40%*
Amean     fault-both-30    15121.34 (   0.00%)    16158.25 (  -6.86%)
Amean     fault-both-32    18466.67 (   0.00%)    18971.21 (  -2.73%)

Latency is only moderately affected but the devil is in the details.  A
closer examination indicates that base page fault latency is reduced but
latency of huge pages is increased as it takes creater care to succeed.
Part of the "problem" is that allocation success rates are close to 100%
even when under pressure and compaction gets harder

                                5.0.0-rc1              5.0.0-rc1
                          selective-v3r17          capture-v3r19
Percentage huge-3        96.70 (   0.00%)       98.23 (   1.58%)
Percentage huge-5        96.99 (   0.00%)       95.30 (  -1.75%)
Percentage huge-7        94.19 (   0.00%)       97.24 (   3.24%)
Percentage huge-12       94.95 (   0.00%)       97.35 (   2.53%)
Percentage huge-18       96.74 (   0.00%)       97.30 (   0.58%)
Percentage huge-24       97.07 (   0.00%)       97.55 (   0.50%)
Percentage huge-30       95.69 (   0.00%)       98.50 (   2.95%)
Percentage huge-32       96.70 (   0.00%)       99.27 (   2.65%)

And scan rates are reduced as expected by 6% for the migration scanner
and 29% for the free scanner indicating that there is less redundant
work.

Compaction migrate scanned    20815362    19573286
Compaction free scanned       16352612    11510663

[mgorman@techsingularity.net: remove redundant check]
  Link: http://lkml.kernel.org/r/20190201143853.GH9565@techsingularity.net
Link: http://lkml.kernel.org/r/20190118175136.31341-23-mgorman@techsingularity.net
Signed-off-by: Mel Gorman <mgorman@techsingularity.net>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Dan Carpenter <dan.carpenter@oracle.com>
Cc: David Rientjes <rientjes@google.com>
Cc: YueHaibing <yuehaibing@huawei.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

1 files changed, 69 insertions, 4 deletions

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 2e132b9e7a93..09bf2c5f8b4b 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -789,6 +789,57 @@ static inline int page_is_buddy(struct page *page, struct page *buddy,
         return 0;
 }
 
+#ifdef CONFIG_COMPACTION
+static inline struct capture_control *task_capc(struct zone *zone)
+{
+        struct capture_control *capc = current->capture_control;
+
+        return capc &&
+                !(current->flags & PF_KTHREAD) &&
+                !capc->page &&
+                capc->cc->zone == zone &&
+                capc->cc->direct_compaction ? capc : NULL;
+}
+
+static inline bool
+compaction_capture(struct capture_control *capc, struct page *page,
+                   int order, int migratetype)
+{
+        if (!capc || order != capc->cc->order)
+                return false;
+
+        /* Do not accidentally pollute CMA or isolated regions*/
+        if (is_migrate_cma(migratetype) ||
+            is_migrate_isolate(migratetype))
+                return false;
+
+        /*
+         * Do not let lower order allocations polluate a movable pageblock.
+         * This might let an unmovable request use a reclaimable pageblock
+         * and vice-versa but no more than normal fallback logic which can
+         * have trouble finding a high-order free page.
+         */
+        if (order < pageblock_order && migratetype == MIGRATE_MOVABLE)
+                return false;
+
+        capc->page = page;
+        return true;
+}
+
+#else
+static inline struct capture_control *task_capc(struct zone *zone)
+{
+        return NULL;
+}
+
+static inline bool
+compaction_capture(struct capture_control *capc, struct page *page,
+                   int order, int migratetype)
+{
+        return false;
+}
+#endif /* CONFIG_COMPACTION */
+
 /*
  * Freeing function for a buddy system allocator.
  *
@@ -822,6 +873,7 @@ static inline void __free_one_page(struct page *page,
         unsigned long uninitialized_var(buddy_pfn);
         struct page *buddy;
         unsigned int max_order;
+        struct capture_control *capc = task_capc(zone);
 
         max_order = min_t(unsigned int, MAX_ORDER, pageblock_order + 1);
 
@@ -837,6 +889,11 @@ static inline void __free_one_page(struct page *page,
 
 continue_merging:
         while (order < max_order - 1) {
+                if (compaction_capture(capc, page, order, migratetype)) {
+                        __mod_zone_freepage_state(zone, -(1 << order),
+                                                                migratetype);
+                        return;
+                }
                 buddy_pfn = __find_buddy_pfn(pfn, order);
                 buddy = page + (buddy_pfn - pfn);
 
@@ -3710,7 +3767,7 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
                 unsigned int alloc_flags, const struct alloc_context *ac,
                 enum compact_priority prio, enum compact_result *compact_result)
 {
-        struct page *page;
+        struct page *page = NULL;
         unsigned long pflags;
         unsigned int noreclaim_flag;
 
@@ -3721,13 +3778,15 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
         noreclaim_flag = memalloc_noreclaim_save();
 
         *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac,
-                                                                        prio);
+                                                                prio, &page);
 
         memalloc_noreclaim_restore(noreclaim_flag);
         psi_memstall_leave(&pflags);
 
-        if (*compact_result <= COMPACT_INACTIVE)
+        if (*compact_result <= COMPACT_INACTIVE) {
+                WARN_ON_ONCE(page);
                 return NULL;
+        }
 
         /*
          * At least in one zone compaction wasn't deferred or skipped, so let's
@@ -3735,7 +3794,13 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
          */
         count_vm_event(COMPACTSTALL);
 
-        page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
+        /* Prep a captured page if available */
+        if (page)
+                prep_new_page(page, order, gfp_mask, alloc_flags);
+
+        /* Try get a page from the freelist if available */
+        if (!page)
+                page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
 
         if (page) {
                 struct zone *zone = page_zone(page);