1 files changed, 45 insertions, 44 deletions

diff --git a/mm/migrate.c b/mm/migrate.c
index 9194375b2307..a8025befc323 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -72,28 +72,12 @@ int migrate_prep_local(void)
 }
 
 /*
- * Add isolated pages on the list back to the LRU under page lock
- * to avoid leaking evictable pages back onto unevictable list.
- */
-void putback_lru_pages(struct list_head *l)
-{
-        struct page *page;
-        struct page *page2;
-
-        list_for_each_entry_safe(page, page2, l, lru) {
-                list_del(&page->lru);
-                dec_zone_page_state(page, NR_ISOLATED_ANON +
-                                page_is_file_cache(page));
-                        putback_lru_page(page);
-        }
-}
-
-/*
  * Put previously isolated pages back onto the appropriate lists
  * from where they were once taken off for compaction/migration.
  *
- * This function shall be used instead of putback_lru_pages(),
- * whenever the isolated pageset has been built by isolate_migratepages_range()
+ * This function shall be used whenever the isolated pageset has been
+ * built from lru, balloon, hugetlbfs page. See isolate_migratepages_range()
+ * and isolate_huge_page().
  */
 void putback_movable_pages(struct list_head *l)
 {
@@ -199,7 +183,12 @@ out:
  */
 static void remove_migration_ptes(struct page *old, struct page *new)
 {
-        rmap_walk(new, remove_migration_pte, old);
+        struct rmap_walk_control rwc = {
+                .rmap_one = remove_migration_pte,
+                .arg = old,
+        };
+
+        rmap_walk(new, &rwc);
 }
 
 /*
@@ -563,14 +552,6 @@ void migrate_page_copy(struct page *newpage, struct page *page)
  *                    Migration functions
  ***********************************************************/
 
-/* Always fail migration. Used for mappings that are not movable */
-int fail_migrate_page(struct address_space *mapping,
-                        struct page *newpage, struct page *page)
-{
-        return -EIO;
-}
-EXPORT_SYMBOL(fail_migrate_page);
-
 /*
  * Common logic to directly migrate a single page suitable for
  * pages that do not use PagePrivate/PagePrivate2.
@@ -1008,7 +989,7 @@ static int unmap_and_move_huge_page(new_page_t get_new_page,
 {
         int rc = 0;
         int *result = NULL;
-        struct page *new_hpage = get_new_page(hpage, private, &result);
+        struct page *new_hpage;
         struct anon_vma *anon_vma = NULL;
 
         /*
@@ -1018,9 +999,12 @@ static int unmap_and_move_huge_page(new_page_t get_new_page,
          * tables or check whether the hugepage is pmd-based or not before
          * kicking migration.
          */
-        if (!hugepage_migration_support(page_hstate(hpage)))
+        if (!hugepage_migration_support(page_hstate(hpage))) {
+                putback_active_hugepage(hpage);
                 return -ENOSYS;
+        }
 
+        new_hpage = get_new_page(hpage, private, &result);
         if (!new_hpage)
                 return -ENOMEM;
 
@@ -1120,7 +1104,12 @@ int migrate_pages(struct list_head *from, new_page_t get_new_page,
                                 nr_succeeded++;
                                 break;
                         default:
-                                /* Permanent failure */
+                                /*
+                                 * Permanent failure (-EBUSY, -ENOSYS, etc.):
+                                 * unlike -EAGAIN case, the failed page is
+                                 * removed from migration page list and not
+                                 * retried in the next outer loop.
+                                 */
                                 nr_failed++;
                                 break;
                         }
@@ -1594,31 +1583,38 @@ bool migrate_ratelimited(int node)
 }
 
 /* Returns true if the node is migrate rate-limited after the update */
-bool numamigrate_update_ratelimit(pg_data_t *pgdat, unsigned long nr_pages)
+static bool numamigrate_update_ratelimit(pg_data_t *pgdat,
+                                        unsigned long nr_pages)
 {
-        bool rate_limited = false;
-
         /*
          * Rate-limit the amount of data that is being migrated to a node.
          * Optimal placement is no good if the memory bus is saturated and
          * all the time is being spent migrating!
          */
-        spin_lock(&pgdat->numabalancing_migrate_lock);
         if (time_after(jiffies, pgdat->numabalancing_migrate_next_window)) {
+                spin_lock(&pgdat->numabalancing_migrate_lock);
                 pgdat->numabalancing_migrate_nr_pages = 0;
                 pgdat->numabalancing_migrate_next_window = jiffies +
                         msecs_to_jiffies(migrate_interval_millisecs);
+                spin_unlock(&pgdat->numabalancing_migrate_lock);
         }
-        if (pgdat->numabalancing_migrate_nr_pages > ratelimit_pages)
-                rate_limited = true;
-        else
-                pgdat->numabalancing_migrate_nr_pages += nr_pages;
-        spin_unlock(&pgdat->numabalancing_migrate_lock);
-        
-        return rate_limited;
+        if (pgdat->numabalancing_migrate_nr_pages > ratelimit_pages) {
+                trace_mm_numa_migrate_ratelimit(current, pgdat->node_id,
+                                                                nr_pages);
+                return true;
+        }
+
+        /*
+         * This is an unlocked non-atomic update so errors are possible.
+         * The consequences are failing to migrate when we potentiall should
+         * have which is not severe enough to warrant locking. If it is ever
+         * a problem, it can be converted to a per-cpu counter.
+         */
+        pgdat->numabalancing_migrate_nr_pages += nr_pages;
+        return false;
 }
 
-int numamigrate_isolate_page(pg_data_t *pgdat, struct page *page)
+static int numamigrate_isolate_page(pg_data_t *pgdat, struct page *page)
 {
         int page_lru;
 
@@ -1705,7 +1701,12 @@ int migrate_misplaced_page(struct page *page, struct vm_area_struct *vma,
         nr_remaining = migrate_pages(&migratepages, alloc_misplaced_dst_page,
                                      node, MIGRATE_ASYNC, MR_NUMA_MISPLACED);
         if (nr_remaining) {
-                putback_lru_pages(&migratepages);
+                if (!list_empty(&migratepages)) {
+                        list_del(&page->lru);
+                        dec_zone_page_state(page, NR_ISOLATED_ANON +
+                                        page_is_file_cache(page));
+                        putback_lru_page(page);
+                }
                 isolated = 0;
         } else
                 count_vm_numa_event(NUMA_PAGE_MIGRATE);