1 files changed, 272 insertions, 160 deletions

diff --git a/mm/compaction.c b/mm/compaction.c
index 74a8c825ff28..2f42d9528539 100644
--- a/mm/compaction.c
+++ b/mm/compaction.c
@@ -16,30 +16,11 @@
 #include <linux/sysfs.h>
 #include "internal.h"
 
+#if defined CONFIG_COMPACTION || defined CONFIG_CMA
+
 #define CREATE_TRACE_POINTS
 #include <trace/events/compaction.h>
 
-/*
- * compact_control is used to track pages being migrated and the free pages
- * they are being migrated to during memory compaction. The free_pfn starts
- * at the end of a zone and migrate_pfn begins at the start. Movable pages
- * are moved to the end of a zone during a compaction run and the run
- * completes when free_pfn <= migrate_pfn
- */
-struct compact_control {
-        struct list_head freepages;     /* List of free pages to migrate to */
-        struct list_head migratepages;  /* List of pages being migrated */
-        unsigned long nr_freepages;     /* Number of isolated free pages */
-        unsigned long nr_migratepages;  /* Number of pages to migrate */
-        unsigned long free_pfn;         /* isolate_freepages search base */
-        unsigned long migrate_pfn;      /* isolate_migratepages search base */
-        bool sync;                      /* Synchronous migration */
-
-        int order;                      /* order a direct compactor needs */
-        int migratetype;                /* MOVABLE, RECLAIMABLE etc */
-        struct zone *zone;
-};
-
 static unsigned long release_freepages(struct list_head *freelist)
 {
         struct page *page, *next;
@@ -54,24 +35,35 @@ static unsigned long release_freepages(struct list_head *freelist)
         return count;
 }
 
-/* Isolate free pages onto a private freelist. Must hold zone->lock */
-static unsigned long isolate_freepages_block(struct zone *zone,
-                                unsigned long blockpfn,
-                                struct list_head *freelist)
+static void map_pages(struct list_head *list)
+{
+        struct page *page;
+
+        list_for_each_entry(page, list, lru) {
+                arch_alloc_page(page, 0);
+                kernel_map_pages(page, 1, 1);
+        }
+}
+
+static inline bool migrate_async_suitable(int migratetype)
+{
+        return is_migrate_cma(migratetype) || migratetype == MIGRATE_MOVABLE;
+}
+
+/*
+ * Isolate free pages onto a private freelist. Caller must hold zone->lock.
+ * If @strict is true, will abort returning 0 on any invalid PFNs or non-free
+ * pages inside of the pageblock (even though it may still end up isolating
+ * some pages).
+ */
+static unsigned long isolate_freepages_block(unsigned long blockpfn,
+                                unsigned long end_pfn,
+                                struct list_head *freelist,
+                                bool strict)
 {
-        unsigned long zone_end_pfn, end_pfn;
         int nr_scanned = 0, total_isolated = 0;
         struct page *cursor;
 
-        /* Get the last PFN we should scan for free pages at */
-        zone_end_pfn = zone->zone_start_pfn + zone->spanned_pages;
-        end_pfn = min(blockpfn + pageblock_nr_pages, zone_end_pfn);
-
-        /* Find the first usable PFN in the block to initialse page cursor */
-        for (; blockpfn < end_pfn; blockpfn++) {
-                if (pfn_valid_within(blockpfn))
-                        break;
-        }
         cursor = pfn_to_page(blockpfn);
 
         /* Isolate free pages. This assumes the block is valid */
@@ -79,15 +71,23 @@ static unsigned long isolate_freepages_block(struct zone *zone,
                 int isolated, i;
                 struct page *page = cursor;
 
-                if (!pfn_valid_within(blockpfn))
+                if (!pfn_valid_within(blockpfn)) {
+                        if (strict)
+                                return 0;
                         continue;
+                }
                 nr_scanned++;
 
-                if (!PageBuddy(page))
+                if (!PageBuddy(page)) {
+                        if (strict)
+                                return 0;
                         continue;
+                }
 
                 /* Found a free page, break it into order-0 pages */
                 isolated = split_free_page(page);
+                if (!isolated && strict)
+                        return 0;
                 total_isolated += isolated;
                 for (i = 0; i < isolated; i++) {
                         list_add(&page->lru, freelist);
@@ -105,114 +105,71 @@ static unsigned long isolate_freepages_block(struct zone *zone,
         return total_isolated;
 }
 
-/* Returns true if the page is within a block suitable for migration to */
-static bool suitable_migration_target(struct page *page)
-{
-
-        int migratetype = get_pageblock_migratetype(page);
-
-        /* Don't interfere with memory hot-remove or the min_free_kbytes blocks */
-        if (migratetype == MIGRATE_ISOLATE || migratetype == MIGRATE_RESERVE)
-                return false;
-
-        /* If the page is a large free page, then allow migration */
-        if (PageBuddy(page) && page_order(page) >= pageblock_order)
-                return true;
-
-        /* If the block is MIGRATE_MOVABLE, allow migration */
-        if (migratetype == MIGRATE_MOVABLE)
-                return true;
-
-        /* Otherwise skip the block */
-        return false;
-}
-
-/*
- * Based on information in the current compact_control, find blocks
- * suitable for isolating free pages from and then isolate them.
+/**
+ * isolate_freepages_range() - isolate free pages.
+ * @start_pfn: The first PFN to start isolating.
+ * @end_pfn:   The one-past-last PFN.
+ *
+ * Non-free pages, invalid PFNs, or zone boundaries within the
+ * [start_pfn, end_pfn) range are considered errors, cause function to
+ * undo its actions and return zero.
+ *
+ * Otherwise, function returns one-past-the-last PFN of isolated page
+ * (which may be greater then end_pfn if end fell in a middle of
+ * a free page).
  */
-static void isolate_freepages(struct zone *zone,
-                                struct compact_control *cc)
+unsigned long
+isolate_freepages_range(unsigned long start_pfn, unsigned long end_pfn)
 {
-        struct page *page;
-        unsigned long high_pfn, low_pfn, pfn;
-        unsigned long flags;
-        int nr_freepages = cc->nr_freepages;
-        struct list_head *freelist = &cc->freepages;
-
-        /*
-         * Initialise the free scanner. The starting point is where we last
-         * scanned from (or the end of the zone if starting). The low point
-         * is the end of the pageblock the migration scanner is using.
-         */
-        pfn = cc->free_pfn;
-        low_pfn = cc->migrate_pfn + pageblock_nr_pages;
+        unsigned long isolated, pfn, block_end_pfn, flags;
+        struct zone *zone = NULL;
+        LIST_HEAD(freelist);
 
-        /*
-         * Take care that if the migration scanner is at the end of the zone
-         * that the free scanner does not accidentally move to the next zone
-         * in the next isolation cycle.
-         */
-        high_pfn = min(low_pfn, pfn);
+        if (pfn_valid(start_pfn))
+                zone = page_zone(pfn_to_page(start_pfn));
 
-        /*
-         * Isolate free pages until enough are available to migrate the
-         * pages on cc->migratepages. We stop searching if the migrate
-         * and free page scanners meet or enough free pages are isolated.
-         */
-        for (; pfn > low_pfn && cc->nr_migratepages > nr_freepages;
-                                        pfn -= pageblock_nr_pages) {
-                unsigned long isolated;
-
-                if (!pfn_valid(pfn))
-                        continue;
+        for (pfn = start_pfn; pfn < end_pfn; pfn += isolated) {
+                if (!pfn_valid(pfn) || zone != page_zone(pfn_to_page(pfn)))
+                        break;
 
                 /*
-                 * Check for overlapping nodes/zones. It's possible on some
-                 * configurations to have a setup like
-                 * node0 node1 node0
-                 * i.e. it's possible that all pages within a zones range of
-                 * pages do not belong to a single zone.
+                 * On subsequent iterations ALIGN() is actually not needed,
+                 * but we keep it that we not to complicate the code.
                  */
-                page = pfn_to_page(pfn);
-                if (page_zone(page) != zone)
-                        continue;
+                block_end_pfn = ALIGN(pfn + 1, pageblock_nr_pages);
+                block_end_pfn = min(block_end_pfn, end_pfn);
 
-                /* Check the block is suitable for migration */
-                if (!suitable_migration_target(page))
-                        continue;
+                spin_lock_irqsave(&zone->lock, flags);
+                isolated = isolate_freepages_block(pfn, block_end_pfn,
+                                                   &freelist, true);
+                spin_unlock_irqrestore(&zone->lock, flags);
 
                 /*
-                 * Found a block suitable for isolating free pages from. Now
-                 * we disabled interrupts, double check things are ok and
-                 * isolate the pages. This is to minimise the time IRQs
-                 * are disabled
+                 * In strict mode, isolate_freepages_block() returns 0 if
+                 * there are any holes in the block (ie. invalid PFNs or
+                 * non-free pages).
                  */
-                isolated = 0;
-                spin_lock_irqsave(&zone->lock, flags);
-                if (suitable_migration_target(page)) {
-                        isolated = isolate_freepages_block(zone, pfn, freelist);
-                        nr_freepages += isolated;
-                }
-                spin_unlock_irqrestore(&zone->lock, flags);
+                if (!isolated)
+                        break;
 
                 /*
-                 * Record the highest PFN we isolated pages from. When next
-                 * looking for free pages, the search will restart here as
-                 * page migration may have returned some pages to the allocator
+                 * If we managed to isolate pages, it is always (1 << n) *
+                 * pageblock_nr_pages for some non-negative n.  (Max order
+                 * page may span two pageblocks).
                  */
-                if (isolated)
-                        high_pfn = max(high_pfn, pfn);
         }
 
         /* split_free_page does not map the pages */
-        list_for_each_entry(page, freelist, lru) {
-                arch_alloc_page(page, 0);
-                kernel_map_pages(page, 1, 1);
+        map_pages(&freelist);
+
+        if (pfn < end_pfn) {
+                /* Loop terminated early, cleanup. */
+                release_freepages(&freelist);
+                return 0;
         }
 
-        cc->free_pfn = high_pfn;
-        cc->nr_freepages = nr_freepages;
+        /* We don't use freelists for anything. */
+        return pfn;
 }
 
 /* Update the number of anon and file isolated pages in the zone */
@@ -243,37 +200,34 @@ static bool too_many_isolated(struct zone *zone)
         return isolated > (inactive + active) / 2;
 }
 
-/* possible outcome of isolate_migratepages */
-typedef enum {
-        ISOLATE_ABORT,          /* Abort compaction now */
-        ISOLATE_NONE,           /* No pages isolated, continue scanning */
-        ISOLATE_SUCCESS,        /* Pages isolated, migrate */
-} isolate_migrate_t;
-
-/*
- * Isolate all pages that can be migrated from the block pointed to by
- * the migrate scanner within compact_control.
+/**
+ * isolate_migratepages_range() - isolate all migrate-able pages in range.
+ * @zone:       Zone pages are in.
+ * @cc:         Compaction control structure.
+ * @low_pfn:    The first PFN of the range.
+ * @end_pfn:    The one-past-the-last PFN of the range.
+ *
+ * Isolate all pages that can be migrated from the range specified by
+ * [low_pfn, end_pfn).  Returns zero if there is a fatal signal
+ * pending), otherwise PFN of the first page that was not scanned
+ * (which may be both less, equal to or more then end_pfn).
+ *
+ * Assumes that cc->migratepages is empty and cc->nr_migratepages is
+ * zero.
+ *
+ * Apart from cc->migratepages and cc->nr_migratetypes this function
+ * does not modify any cc's fields, in particular it does not modify
+ * (or read for that matter) cc->migrate_pfn.
  */
-static isolate_migrate_t isolate_migratepages(struct zone *zone,
-                                        struct compact_control *cc)
+unsigned long
+isolate_migratepages_range(struct zone *zone, struct compact_control *cc,
+                           unsigned long low_pfn, unsigned long end_pfn)
 {
-        unsigned long low_pfn, end_pfn;
         unsigned long last_pageblock_nr = 0, pageblock_nr;
         unsigned long nr_scanned = 0, nr_isolated = 0;
         struct list_head *migratelist = &cc->migratepages;
-        isolate_mode_t mode = ISOLATE_ACTIVE|ISOLATE_INACTIVE;
-
-        /* Do not scan outside zone boundaries */
-        low_pfn = max(cc->migrate_pfn, zone->zone_start_pfn);
-
-        /* Only scan within a pageblock boundary */
-        end_pfn = ALIGN(low_pfn + pageblock_nr_pages, pageblock_nr_pages);
-
-        /* Do not cross the free scanner or scan within a memory hole */
-        if (end_pfn > cc->free_pfn || !pfn_valid(low_pfn)) {
-                cc->migrate_pfn = end_pfn;
-                return ISOLATE_NONE;
-        }
+        isolate_mode_t mode = 0;
+        struct lruvec *lruvec;
 
         /*
          * Ensure that there are not too many pages isolated from the LRU
@@ -283,12 +237,12 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,
         while (unlikely(too_many_isolated(zone))) {
                 /* async migration should just abort */
                 if (!cc->sync)
-                        return ISOLATE_ABORT;
+                        return 0;
 
                 congestion_wait(BLK_RW_ASYNC, HZ/10);
 
                 if (fatal_signal_pending(current))
-                        return ISOLATE_ABORT;
+                        return 0;
         }
 
         /* Time to isolate some pages for migration */
@@ -351,7 +305,7 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,
                  */
                 pageblock_nr = low_pfn >> pageblock_order;
                 if (!cc->sync && last_pageblock_nr != pageblock_nr &&
-                                get_pageblock_migratetype(page) != MIGRATE_MOVABLE) {
+                    !migrate_async_suitable(get_pageblock_migratetype(page))) {
                         low_pfn += pageblock_nr_pages;
                         low_pfn = ALIGN(low_pfn, pageblock_nr_pages) - 1;
                         last_pageblock_nr = pageblock_nr;
@@ -374,14 +328,16 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,
                 if (!cc->sync)
                         mode |= ISOLATE_ASYNC_MIGRATE;
 
+                lruvec = mem_cgroup_page_lruvec(page, zone);
+
                 /* Try isolate the page */
-                if (__isolate_lru_page(page, mode, 0) != 0)
+                if (__isolate_lru_page(page, mode) != 0)
                         continue;
 
                 VM_BUG_ON(PageTransCompound(page));
 
                 /* Successfully isolated */
-                del_page_from_lru_list(zone, page, page_lru(page));
+                del_page_from_lru_list(page, lruvec, page_lru(page));
                 list_add(&page->lru, migratelist);
                 cc->nr_migratepages++;
                 nr_isolated++;
@@ -396,11 +352,124 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,
         acct_isolated(zone, cc);
 
         spin_unlock_irq(&zone->lru_lock);
-        cc->migrate_pfn = low_pfn;
 
         trace_mm_compaction_isolate_migratepages(nr_scanned, nr_isolated);
 
-        return ISOLATE_SUCCESS;
+        return low_pfn;
+}
+
+#endif /* CONFIG_COMPACTION || CONFIG_CMA */
+#ifdef CONFIG_COMPACTION
+
+/* Returns true if the page is within a block suitable for migration to */
+static bool suitable_migration_target(struct page *page)
+{
+
+        int migratetype = get_pageblock_migratetype(page);
+
+        /* Don't interfere with memory hot-remove or the min_free_kbytes blocks */
+        if (migratetype == MIGRATE_ISOLATE || migratetype == MIGRATE_RESERVE)
+                return false;
+
+        /* If the page is a large free page, then allow migration */
+        if (PageBuddy(page) && page_order(page) >= pageblock_order)
+                return true;
+
+        /* If the block is MIGRATE_MOVABLE or MIGRATE_CMA, allow migration */
+        if (migrate_async_suitable(migratetype))
+                return true;
+
+        /* Otherwise skip the block */
+        return false;
+}
+
+/*
+ * Based on information in the current compact_control, find blocks
+ * suitable for isolating free pages from and then isolate them.
+ */
+static void isolate_freepages(struct zone *zone,
+                                struct compact_control *cc)
+{
+        struct page *page;
+        unsigned long high_pfn, low_pfn, pfn, zone_end_pfn, end_pfn;
+        unsigned long flags;
+        int nr_freepages = cc->nr_freepages;
+        struct list_head *freelist = &cc->freepages;
+
+        /*
+         * Initialise the free scanner. The starting point is where we last
+         * scanned from (or the end of the zone if starting). The low point
+         * is the end of the pageblock the migration scanner is using.
+         */
+        pfn = cc->free_pfn;
+        low_pfn = cc->migrate_pfn + pageblock_nr_pages;
+
+        /*
+         * Take care that if the migration scanner is at the end of the zone
+         * that the free scanner does not accidentally move to the next zone
+         * in the next isolation cycle.
+         */
+        high_pfn = min(low_pfn, pfn);
+
+        zone_end_pfn = zone->zone_start_pfn + zone->spanned_pages;
+
+        /*
+         * Isolate free pages until enough are available to migrate the
+         * pages on cc->migratepages. We stop searching if the migrate
+         * and free page scanners meet or enough free pages are isolated.
+         */
+        for (; pfn > low_pfn && cc->nr_migratepages > nr_freepages;
+                                        pfn -= pageblock_nr_pages) {
+                unsigned long isolated;
+
+                if (!pfn_valid(pfn))
+                        continue;
+
+                /*
+                 * Check for overlapping nodes/zones. It's possible on some
+                 * configurations to have a setup like
+                 * node0 node1 node0
+                 * i.e. it's possible that all pages within a zones range of
+                 * pages do not belong to a single zone.
+                 */
+                page = pfn_to_page(pfn);
+                if (page_zone(page) != zone)
+                        continue;
+
+                /* Check the block is suitable for migration */
+                if (!suitable_migration_target(page))
+                        continue;
+
+                /*
+                 * Found a block suitable for isolating free pages from. Now
+                 * we disabled interrupts, double check things are ok and
+                 * isolate the pages. This is to minimise the time IRQs
+                 * are disabled
+                 */
+                isolated = 0;
+                spin_lock_irqsave(&zone->lock, flags);
+                if (suitable_migration_target(page)) {
+                        end_pfn = min(pfn + pageblock_nr_pages, zone_end_pfn);
+                        isolated = isolate_freepages_block(pfn, end_pfn,
+                                                           freelist, false);
+                        nr_freepages += isolated;
+                }
+                spin_unlock_irqrestore(&zone->lock, flags);
+
+                /*
+                 * Record the highest PFN we isolated pages from. When next
+                 * looking for free pages, the search will restart here as
+                 * page migration may have returned some pages to the allocator
+                 */
+                if (isolated)
+                        high_pfn = max(high_pfn, pfn);
+        }
+
+        /* split_free_page does not map the pages */
+        map_pages(freelist);
+
+        cc->free_pfn = high_pfn;
+        cc->nr_freepages = nr_freepages;
 }
 
 /*
@@ -449,6 +518,44 @@ static void update_nr_listpages(struct compact_control *cc)
         cc->nr_freepages = nr_freepages;
 }
 
+/* possible outcome of isolate_migratepages */
+typedef enum {
+        ISOLATE_ABORT,          /* Abort compaction now */
+        ISOLATE_NONE,           /* No pages isolated, continue scanning */
+        ISOLATE_SUCCESS,        /* Pages isolated, migrate */
+} isolate_migrate_t;
+
+/*
+ * Isolate all pages that can be migrated from the block pointed to by
+ * the migrate scanner within compact_control.
+ */
+static isolate_migrate_t isolate_migratepages(struct zone *zone,
+                                        struct compact_control *cc)
+{
+        unsigned long low_pfn, end_pfn;
+
+        /* Do not scan outside zone boundaries */
+        low_pfn = max(cc->migrate_pfn, zone->zone_start_pfn);
+
+        /* Only scan within a pageblock boundary */
+        end_pfn = ALIGN(low_pfn + pageblock_nr_pages, pageblock_nr_pages);
+
+        /* Do not cross the free scanner or scan within a memory hole */
+        if (end_pfn > cc->free_pfn || !pfn_valid(low_pfn)) {
+                cc->migrate_pfn = end_pfn;
+                return ISOLATE_NONE;
+        }
+
+        /* Perform the isolation */
+        low_pfn = isolate_migratepages_range(zone, cc, low_pfn, end_pfn);
+        if (!low_pfn)
+                return ISOLATE_ABORT;
+
+        cc->migrate_pfn = low_pfn;
+
+        return ISOLATE_SUCCESS;
+}
+
 static int compact_finished(struct zone *zone,
                             struct compact_control *cc)
 {
@@ -594,8 +701,11 @@ static int compact_zone(struct zone *zone, struct compact_control *cc)
                 if (err) {
                         putback_lru_pages(&cc->migratepages);
                         cc->nr_migratepages = 0;
+                        if (err == -ENOMEM) {
+                                ret = COMPACT_PARTIAL;
+                                goto out;
+                        }
                 }
-
         }
 
 out:
@@ -795,3 +905,5 @@ void compaction_unregister_node(struct node *node)
         return device_remove_file(&node->dev, &dev_attr_compact);
 }
 #endif /* CONFIG_SYSFS && CONFIG_NUMA */
+
+#endif /* CONFIG_COMPACTION */