1 files changed, 64 insertions, 59 deletions

diff --git a/mm/migrate.c b/mm/migrate.c
index 6ae8a66a7045..46fe8cc13d67 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -113,6 +113,8 @@ static int remove_migration_pte(struct page *new, struct vm_area_struct *vma,
                         goto out;
 
                 pmd = pmd_offset(pud, addr);
+                if (pmd_trans_huge(*pmd))
+                        goto out;
                 if (!pmd_present(*pmd))
                         goto out;
 
@@ -246,7 +248,7 @@ static int migrate_page_move_mapping(struct address_space *mapping,
 
         expected_count = 2 + page_has_private(page);
         if (page_count(page) != expected_count ||
-                        (struct page *)radix_tree_deref_slot(pslot) != page) {
+                radix_tree_deref_slot_protected(pslot, &mapping->tree_lock) != page) {
                 spin_unlock_irq(&mapping->tree_lock);
                 return -EAGAIN;
         }
@@ -318,7 +320,7 @@ int migrate_huge_page_move_mapping(struct address_space *mapping,
 
         expected_count = 2 + page_has_private(page);
         if (page_count(page) != expected_count ||
-            (struct page *)radix_tree_deref_slot(pslot) != page) {
+                radix_tree_deref_slot_protected(pslot, &mapping->tree_lock) != page) {
                 spin_unlock_irq(&mapping->tree_lock);
                 return -EAGAIN;
         }
@@ -614,13 +616,12 @@ static int move_to_new_page(struct page *newpage, struct page *page,
  * to the newly allocated page in newpage.
  */
 static int unmap_and_move(new_page_t get_new_page, unsigned long private,
-                        struct page *page, int force, int offlining)
+                        struct page *page, int force, bool offlining, bool sync)
 {
         int rc = 0;
         int *result = NULL;
         struct page *newpage = get_new_page(page, private, &result);
         int remap_swapcache = 1;
-        int rcu_locked = 0;
         int charge = 0;
         struct mem_cgroup *mem = NULL;
         struct anon_vma *anon_vma = NULL;
@@ -632,6 +633,9 @@ static int unmap_and_move(new_page_t get_new_page, unsigned long private,
                 /* page was freed from under us. So we are done. */
                 goto move_newpage;
         }
+        if (unlikely(PageTransHuge(page)))
+                if (unlikely(split_huge_page(page)))
+                        goto move_newpage;
 
         /* prepare cgroup just returns 0 or -ENOMEM */
         rc = -EAGAIN;
@@ -639,6 +643,23 @@ static int unmap_and_move(new_page_t get_new_page, unsigned long private,
         if (!trylock_page(page)) {
                 if (!force)
                         goto move_newpage;
+
+                /*
+                 * It's not safe for direct compaction to call lock_page.
+                 * For example, during page readahead pages are added locked
+                 * to the LRU. Later, when the IO completes the pages are
+                 * marked uptodate and unlocked. However, the queueing
+                 * could be merging multiple pages for one bio (e.g.
+                 * mpage_readpages). If an allocation happens for the
+                 * second or third page, the process can end up locking
+                 * the same page twice and deadlocking. Rather than
+                 * trying to be clever about what pages can be locked,
+                 * avoid the use of lock_page for direct compaction
+                 * altogether.
+                 */
+                if (current->flags & PF_MEMALLOC)
+                        goto move_newpage;
+
                 lock_page(page);
         }
 
@@ -665,27 +686,33 @@ static int unmap_and_move(new_page_t get_new_page, unsigned long private,
         BUG_ON(charge);
 
         if (PageWriteback(page)) {
-                if (!force)
+                if (!force || !sync)
                         goto uncharge;
                 wait_on_page_writeback(page);
         }
         /*
          * By try_to_unmap(), page->mapcount goes down to 0 here. In this case,
          * we cannot notice that anon_vma is freed while we migrates a page.
-         * This rcu_read_lock() delays freeing anon_vma pointer until the end
+         * This get_anon_vma() delays freeing anon_vma pointer until the end
          * of migration. File cache pages are no problem because of page_lock()
          * File Caches may use write_page() or lock_page() in migration, then,
          * just care Anon page here.
          */
         if (PageAnon(page)) {
-                rcu_read_lock();
-                rcu_locked = 1;
-
-                /* Determine how to safely use anon_vma */
-                if (!page_mapped(page)) {
-                        if (!PageSwapCache(page))
-                                goto rcu_unlock;
-
+                /*
+                 * Only page_lock_anon_vma() understands the subtleties of
+                 * getting a hold on an anon_vma from outside one of its mms.
+                 */
+                anon_vma = page_lock_anon_vma(page);
+                if (anon_vma) {
+                        /*
+                         * Take a reference count on the anon_vma if the
+                         * page is mapped so that it is guaranteed to
+                         * exist when the page is remapped later
+                         */
+                        get_anon_vma(anon_vma);
+                        page_unlock_anon_vma(anon_vma);
+                } else if (PageSwapCache(page)) {
                         /*
                          * We cannot be sure that the anon_vma of an unmapped
                          * swapcache page is safe to use because we don't
@@ -700,13 +727,7 @@ static int unmap_and_move(new_page_t get_new_page, unsigned long private,
                          */
                         remap_swapcache = 0;
                 } else {
-                        /*
-                         * Take a reference count on the anon_vma if the
-                         * page is mapped so that it is guaranteed to
-                         * exist when the page is remapped later
-                         */
-                        anon_vma = page_anon_vma(page);
-                        get_anon_vma(anon_vma);
+                        goto uncharge;
                 }
         }
 
@@ -723,16 +744,10 @@ static int unmap_and_move(new_page_t get_new_page, unsigned long private,
          * free the metadata, so the page can be freed.
          */
         if (!page->mapping) {
-                if (!PageAnon(page) && page_has_private(page)) {
-                        /*
-                         * Go direct to try_to_free_buffers() here because
-                         * a) that's what try_to_release_page() would do anyway
-                         * b) we may be under rcu_read_lock() here, so we can't
-                         *    use GFP_KERNEL which is what try_to_release_page()
-                         *    needs to be effective.
-                         */
+                VM_BUG_ON(PageAnon(page));
+                if (page_has_private(page)) {
                         try_to_free_buffers(page);
-                        goto rcu_unlock;
+                        goto uncharge;
                 }
                 goto skip_unmap;
         }
@@ -746,17 +761,14 @@ skip_unmap:
 
         if (rc && remap_swapcache)
                 remove_migration_ptes(page, page);
-rcu_unlock:
 
         /* Drop an anon_vma reference if we took one */
         if (anon_vma)
                 drop_anon_vma(anon_vma);
 
-        if (rcu_locked)
-                rcu_read_unlock();
 uncharge:
         if (!charge)
-                mem_cgroup_end_migration(mem, page, newpage);
+                mem_cgroup_end_migration(mem, page, newpage, rc == 0);
 unlock:
         unlock_page(page);
 
@@ -810,12 +822,11 @@ move_newpage:
  */
 static int unmap_and_move_huge_page(new_page_t get_new_page,
                                 unsigned long private, struct page *hpage,
-                                int force, int offlining)
+                                int force, bool offlining, bool sync)
 {
         int rc = 0;
         int *result = NULL;
         struct page *new_hpage = get_new_page(hpage, private, &result);
-        int rcu_locked = 0;
         struct anon_vma *anon_vma = NULL;
 
         if (!new_hpage)
@@ -824,18 +835,16 @@ static int unmap_and_move_huge_page(new_page_t get_new_page,
         rc = -EAGAIN;
 
         if (!trylock_page(hpage)) {
-                if (!force)
+                if (!force || !sync)
                         goto out;
                 lock_page(hpage);
         }
 
         if (PageAnon(hpage)) {
-                rcu_read_lock();
-                rcu_locked = 1;
-
-                if (page_mapped(hpage)) {
-                        anon_vma = page_anon_vma(hpage);
-                        atomic_inc(&anon_vma->external_refcount);
+                anon_vma = page_lock_anon_vma(hpage);
+                if (anon_vma) {
+                        get_anon_vma(anon_vma);
+                        page_unlock_anon_vma(anon_vma);
                 }
         }
 
@@ -847,16 +856,8 @@ static int unmap_and_move_huge_page(new_page_t get_new_page,
         if (rc)
                 remove_migration_ptes(hpage, hpage);
 
-        if (anon_vma && atomic_dec_and_lock(&anon_vma->external_refcount,
-                                            &anon_vma->lock)) {
-                int empty = list_empty(&anon_vma->head);
-                spin_unlock(&anon_vma->lock);
-                if (empty)
-                        anon_vma_free(anon_vma);
-        }
-
-        if (rcu_locked)
-                rcu_read_unlock();
+        if (anon_vma)
+                drop_anon_vma(anon_vma);
 out:
         unlock_page(hpage);
 
@@ -892,7 +893,8 @@ out:
  * Return: Number of pages not migrated or error code.
  */
 int migrate_pages(struct list_head *from,
-                new_page_t get_new_page, unsigned long private, int offlining)
+                new_page_t get_new_page, unsigned long private, bool offlining,
+                bool sync)
 {
         int retry = 1;
         int nr_failed = 0;
@@ -912,7 +914,8 @@ int migrate_pages(struct list_head *from,
                         cond_resched();
 
                         rc = unmap_and_move(get_new_page, private,
-                                                page, pass > 2, offlining);
+                                                page, pass > 2, offlining,
+                                                sync);
 
                         switch(rc) {
                         case -ENOMEM:
@@ -941,7 +944,8 @@ out:
 }
 
 int migrate_huge_pages(struct list_head *from,
-                new_page_t get_new_page, unsigned long private, int offlining)
+                new_page_t get_new_page, unsigned long private, bool offlining,
+                bool sync)
 {
         int retry = 1;
         int nr_failed = 0;
@@ -957,7 +961,8 @@ int migrate_huge_pages(struct list_head *from,
                         cond_resched();
 
                         rc = unmap_and_move_huge_page(get_new_page,
-                                        private, page, pass > 2, offlining);
+                                        private, page, pass > 2, offlining,
+                                        sync);
 
                         switch(rc) {
                         case -ENOMEM:
@@ -1042,7 +1047,7 @@ static int do_move_page_to_node_array(struct mm_struct *mm,
                 if (!vma || pp->addr < vma->vm_start || !vma_migratable(vma))
                         goto set_status;
 
-                page = follow_page(vma, pp->addr, FOLL_GET);
+                page = follow_page(vma, pp->addr, FOLL_GET|FOLL_SPLIT);
 
                 err = PTR_ERR(page);
                 if (IS_ERR(page))
@@ -1090,7 +1095,7 @@ set_status:
         err = 0;
         if (!list_empty(&pagelist)) {
                 err = migrate_pages(&pagelist, new_page_node,
-                                (unsigned long)pm, 0);
+                                (unsigned long)pm, 0, true);
                 if (err)
                         putback_lru_pages(&pagelist);
         }