1 files changed, 49 insertions, 16 deletions

diff --git a/mm/migrate.c b/mm/migrate.c
index 449d77d409f5..2a80136b23bb 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -9,7 +9,7 @@
  * IWAMOTO Toshihiro <iwamoto@valinux.co.jp>
  * Hirokazu Takahashi <taka@valinux.co.jp>
  * Dave Hansen <haveblue@us.ibm.com>
- * Christoph Lameter <clameter@sgi.com>
+ * Christoph Lameter
  */
 
 #include <linux/migrate.h>
@@ -30,6 +30,7 @@
 #include <linux/vmalloc.h>
 #include <linux/security.h>
 #include <linux/memcontrol.h>
+#include <linux/syscalls.h>
 
 #include "internal.h"
 
@@ -284,7 +285,15 @@ void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd,
 
         page = migration_entry_to_page(entry);
 
-        get_page(page);
+        /*
+         * Once radix-tree replacement of page migration started, page_count
+         * *must* be zero. And, we don't want to call wait_on_page_locked()
+         * against a page without get_page().
+         * So, we use get_page_unless_zero(), here. Even failed, page fault
+         * will occur again.
+         */
+        if (!get_page_unless_zero(page))
+                goto out;
         pte_unmap_unlock(ptep, ptl);
         wait_on_page_locked(page);
         put_page(page);
@@ -304,6 +313,7 @@ out:
 static int migrate_page_move_mapping(struct address_space *mapping,
                 struct page *newpage, struct page *page)
 {
+        int expected_count;
         void **pslot;
 
         if (!mapping) {
@@ -313,14 +323,20 @@ static int migrate_page_move_mapping(struct address_space *mapping,
                 return 0;
         }
 
-        write_lock_irq(&mapping->tree_lock);
+        spin_lock_irq(&mapping->tree_lock);
 
         pslot = radix_tree_lookup_slot(&mapping->page_tree,
                                         page_index(page));
 
-        if (page_count(page) != 2 + !!PagePrivate(page) ||
+        expected_count = 2 + !!PagePrivate(page);
+        if (page_count(page) != expected_count ||
                         (struct page *)radix_tree_deref_slot(pslot) != page) {
-                write_unlock_irq(&mapping->tree_lock);
+                spin_unlock_irq(&mapping->tree_lock);
+                return -EAGAIN;
+        }
+
+        if (!page_freeze_refs(page, expected_count)) {
+                spin_unlock_irq(&mapping->tree_lock);
                 return -EAGAIN;
         }
 
@@ -337,6 +353,7 @@ static int migrate_page_move_mapping(struct address_space *mapping,
 
         radix_tree_replace_slot(pslot, newpage);
 
+        page_unfreeze_refs(page, expected_count);
         /*
          * Drop cache reference from old page.
          * We know this isn't the last reference.
@@ -356,7 +373,9 @@ static int migrate_page_move_mapping(struct address_space *mapping,
         __dec_zone_page_state(page, NR_FILE_PAGES);
         __inc_zone_page_state(newpage, NR_FILE_PAGES);
 
-        write_unlock_irq(&mapping->tree_lock);
+        spin_unlock_irq(&mapping->tree_lock);
+        if (!PageSwapCache(newpage))
+                mem_cgroup_uncharge_cache_page(page);
 
         return 0;
 }
@@ -586,7 +605,7 @@ static int move_to_new_page(struct page *newpage, struct page *page)
          * establishing additional references. We are the only one
          * holding a reference to the new page at this point.
          */
-        if (TestSetPageLocked(newpage))
+        if (!trylock_page(newpage))
                 BUG();
 
         /* Prepare mapping for the new page.*/
@@ -610,7 +629,6 @@ static int move_to_new_page(struct page *newpage, struct page *page)
                 rc = fallback_migrate_page(mapping, newpage, page);
 
         if (!rc) {
-                mem_cgroup_page_migration(page, newpage);
                 remove_migration_ptes(page, newpage);
         } else
                 newpage->mapping = NULL;
@@ -640,8 +658,16 @@ static int unmap_and_move(new_page_t get_new_page, unsigned long private,
                 /* page was freed from under us. So we are done. */
                 goto move_newpage;
 
+        charge = mem_cgroup_prepare_migration(page, newpage);
+        if (charge == -ENOMEM) {
+                rc = -ENOMEM;
+                goto move_newpage;
+        }
+        /* prepare cgroup just returns 0 or -ENOMEM */
+        BUG_ON(charge);
+
         rc = -EAGAIN;
-        if (TestSetPageLocked(page)) {
+        if (!trylock_page(page)) {
                 if (!force)
                         goto move_newpage;
                 lock_page(page);
@@ -691,19 +717,14 @@ static int unmap_and_move(new_page_t get_new_page, unsigned long private,
                 goto rcu_unlock;
         }
 
-        charge = mem_cgroup_prepare_migration(page);
         /* Establish migration ptes or remove ptes */
         try_to_unmap(page, 1);
 
         if (!page_mapped(page))
                 rc = move_to_new_page(newpage, page);
 
-        if (rc) {
+        if (rc)
                 remove_migration_ptes(page, page);
-                if (charge)
-                        mem_cgroup_end_migration(page);
-        } else if (charge)
-                mem_cgroup_end_migration(newpage);
 rcu_unlock:
         if (rcu_locked)
                 rcu_read_unlock();
@@ -724,6 +745,8 @@ unlock:
         }
 
 move_newpage:
+        if (!charge)
+                mem_cgroup_end_migration(newpage);
         /*
          * Move the new page to the LRU. If migration was not successful
          * then this will free the page.
@@ -865,6 +888,11 @@ static int do_move_pages(struct mm_struct *mm, struct page_to_node *pm,
                         goto set_status;
 
                 page = follow_page(vma, pp->addr, FOLL_GET);
+
+                err = PTR_ERR(page);
+                if (IS_ERR(page))
+                        goto set_status;
+
                 err = -ENOENT;
                 if (!page)
                         goto set_status;
@@ -928,6 +956,11 @@ static int do_pages_stat(struct mm_struct *mm, struct page_to_node *pm)
                         goto set_status;
 
                 page = follow_page(vma, pm->addr, 0);
+
+                err = PTR_ERR(page);
+                if (IS_ERR(page))
+                        goto set_status;
+
                 err = -ENOENT;
                 /* Use PageReserved to check for zero page */
                 if (!page || PageReserved(page))
@@ -1060,7 +1093,6 @@ out2:
         mmput(mm);
         return err;
 }
-#endif
 
 /*
  * Call migration functions in the vma_ops that may prepare
@@ -1082,3 +1114,4 @@ int migrate_vmas(struct mm_struct *mm, const nodemask_t *to,
         }
         return err;
 }
+#endif