1 files changed, 47 insertions, 88 deletions

diff --git a/mm/migrate.c b/mm/migrate.c
index 1a4bf4813780..efddbf0926b2 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -21,6 +21,7 @@
 #include <linux/mm_inline.h>
 #include <linux/nsproxy.h>
 #include <linux/pagevec.h>
+#include <linux/ksm.h>
 #include <linux/rmap.h>
 #include <linux/topology.h>
 #include <linux/cpu.h>
@@ -78,8 +79,8 @@ int putback_lru_pages(struct list_head *l)
 /*
  * Restore a potential migration pte to a working pte entry
  */
-static void remove_migration_pte(struct vm_area_struct *vma,
-                struct page *old, struct page *new)
+static int remove_migration_pte(struct page *new, struct vm_area_struct *vma,
+                                 unsigned long addr, void *old)
 {
         struct mm_struct *mm = vma->vm_mm;
         swp_entry_t entry;
@@ -88,40 +89,37 @@ static void remove_migration_pte(struct vm_area_struct *vma,
         pmd_t *pmd;
         pte_t *ptep, pte;
         spinlock_t *ptl;
-        unsigned long addr = page_address_in_vma(new, vma);
-
-        if (addr == -EFAULT)
-                return;
 
         pgd = pgd_offset(mm, addr);
         if (!pgd_present(*pgd))
-                return;
+                goto out;
 
         pud = pud_offset(pgd, addr);
         if (!pud_present(*pud))
-                return;
+                goto out;
 
         pmd = pmd_offset(pud, addr);
         if (!pmd_present(*pmd))
-                return;
+                goto out;
 
         ptep = pte_offset_map(pmd, addr);
 
         if (!is_swap_pte(*ptep)) {
                 pte_unmap(ptep);
-                return;
+                goto out;
         }
 
         ptl = pte_lockptr(mm, pmd);
         spin_lock(ptl);
         pte = *ptep;
         if (!is_swap_pte(pte))
-                goto out;
+                goto unlock;
 
         entry = pte_to_swp_entry(pte);
 
-        if (!is_migration_entry(entry) || migration_entry_to_page(entry) != old)
-                goto out;
+        if (!is_migration_entry(entry) ||
+            migration_entry_to_page(entry) != old)
+                goto unlock;
 
         get_page(new);
         pte = pte_mkold(mk_pte(new, vma->vm_page_prot));
@@ -137,58 +135,10 @@ static void remove_migration_pte(struct vm_area_struct *vma,
 
         /* No need to invalidate - it was non-present before */
         update_mmu_cache(vma, addr, pte);
-
-out:
+unlock:
         pte_unmap_unlock(ptep, ptl);
-}
-
-/*
- * Note that remove_file_migration_ptes will only work on regular mappings,
- * Nonlinear mappings do not use migration entries.
- */
-static void remove_file_migration_ptes(struct page *old, struct page *new)
-{
-        struct vm_area_struct *vma;
-        struct address_space *mapping = new->mapping;
-        struct prio_tree_iter iter;
-        pgoff_t pgoff = new->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT);
-
-        if (!mapping)
-                return;
-
-        spin_lock(&mapping->i_mmap_lock);
-
-        vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, pgoff, pgoff)
-                remove_migration_pte(vma, old, new);
-
-        spin_unlock(&mapping->i_mmap_lock);
-}
-
-/*
- * Must hold mmap_sem lock on at least one of the vmas containing
- * the page so that the anon_vma cannot vanish.
- */
-static void remove_anon_migration_ptes(struct page *old, struct page *new)
-{
-        struct anon_vma *anon_vma;
-        struct vm_area_struct *vma;
-        unsigned long mapping;
-
-        mapping = (unsigned long)new->mapping;
-
-        if (!mapping || (mapping & PAGE_MAPPING_ANON) == 0)
-                return;
-
-        /*
-         * We hold the mmap_sem lock. So no need to call page_lock_anon_vma.
-         */
-        anon_vma = (struct anon_vma *) (mapping - PAGE_MAPPING_ANON);
-        spin_lock(&anon_vma->lock);
-
-        list_for_each_entry(vma, &anon_vma->head, anon_vma_node)
-                remove_migration_pte(vma, old, new);
-
-        spin_unlock(&anon_vma->lock);
+out:
+        return SWAP_AGAIN;
 }
 
 /*
@@ -197,10 +147,7 @@ static void remove_anon_migration_ptes(struct page *old, struct page *new)
  */
 static void remove_migration_ptes(struct page *old, struct page *new)
 {
-        if (PageAnon(new))
-                remove_anon_migration_ptes(old, new);
-        else
-                remove_file_migration_ptes(old, new);
+        rmap_walk(new, remove_migration_pte, old);
 }
 
 /*
@@ -341,8 +288,8 @@ static void migrate_page_copy(struct page *newpage, struct page *page)
         if (TestClearPageActive(page)) {
                 VM_BUG_ON(PageUnevictable(page));
                 SetPageActive(newpage);
-        } else
-                unevictable_migrate_page(newpage, page);
+        } else if (TestClearPageUnevictable(page))
+                SetPageUnevictable(newpage);
         if (PageChecked(page))
                 SetPageChecked(newpage);
         if (PageMappedToDisk(page))
@@ -361,6 +308,7 @@ static void migrate_page_copy(struct page *newpage, struct page *page)
         }
 
         mlock_migrate_page(newpage, page);
+        ksm_migrate_page(newpage, page);
 
         ClearPageSwapCache(page);
         ClearPagePrivate(page);
@@ -580,9 +528,9 @@ static int move_to_new_page(struct page *newpage, struct page *page)
         else
                 rc = fallback_migrate_page(mapping, newpage, page);
 
-        if (!rc) {
+        if (!rc)
                 remove_migration_ptes(page, newpage);
-        } else
+        else
                 newpage->mapping = NULL;
 
         unlock_page(newpage);
@@ -595,14 +543,14 @@ static int move_to_new_page(struct page *newpage, struct page *page)
  * to the newly allocated page in newpage.
  */
 static int unmap_and_move(new_page_t get_new_page, unsigned long private,
-                        struct page *page, int force)
+                        struct page *page, int force, int offlining)
 {
         int rc = 0;
         int *result = NULL;
         struct page *newpage = get_new_page(page, private, &result);
         int rcu_locked = 0;
         int charge = 0;
-        struct mem_cgroup *mem;
+        struct mem_cgroup *mem = NULL;
 
         if (!newpage)
                 return -ENOMEM;
@@ -621,6 +569,20 @@ static int unmap_and_move(new_page_t get_new_page, unsigned long private,
                 lock_page(page);
         }
 
+        /*
+         * Only memory hotplug's offline_pages() caller has locked out KSM,
+         * and can safely migrate a KSM page.  The other cases have skipped
+         * PageKsm along with PageReserved - but it is only now when we have
+         * the page lock that we can be certain it will not go KSM beneath us
+         * (KSM will not upgrade a page from PageAnon to PageKsm when it sees
+         * its pagecount raised, but only here do we take the page lock which
+         * serializes that).
+         */
+        if (PageKsm(page) && !offlining) {
+                rc = -EBUSY;
+                goto unlock;
+        }
+
         /* charge against new page */
         charge = mem_cgroup_prepare_migration(page, &mem);
         if (charge == -ENOMEM) {
@@ -737,7 +699,7 @@ move_newpage:
  * Return: Number of pages not migrated or error code.
  */
 int migrate_pages(struct list_head *from,
-                new_page_t get_new_page, unsigned long private)
+                new_page_t get_new_page, unsigned long private, int offlining)
 {
         int retry = 1;
         int nr_failed = 0;
@@ -746,13 +708,6 @@ int migrate_pages(struct list_head *from,
         struct page *page2;
         int swapwrite = current->flags & PF_SWAPWRITE;
         int rc;
-        unsigned long flags;
-
-        local_irq_save(flags);
-        list_for_each_entry(page, from, lru)
-                __inc_zone_page_state(page, NR_ISOLATED_ANON +
-                                page_is_file_cache(page));
-        local_irq_restore(flags);
 
         if (!swapwrite)
                 current->flags |= PF_SWAPWRITE;
@@ -764,7 +719,7 @@ int migrate_pages(struct list_head *from,
                         cond_resched();
 
                         rc = unmap_and_move(get_new_page, private,
-                                                page, pass > 2);
+                                                page, pass > 2, offlining);
 
                         switch(rc) {
                         case -ENOMEM:
@@ -860,7 +815,8 @@ static int do_move_page_to_node_array(struct mm_struct *mm,
                 if (!page)
                         goto set_status;
 
-                if (PageReserved(page))         /* Check for zero page */
+                /* Use PageReserved to check for zero page */
+                if (PageReserved(page) || PageKsm(page))
                         goto put_and_set;
 
                 pp->page = page;
@@ -878,8 +834,11 @@ static int do_move_page_to_node_array(struct mm_struct *mm,
                         goto put_and_set;
 
                 err = isolate_lru_page(page);
-                if (!err)
+                if (!err) {
                         list_add_tail(&page->lru, &pagelist);
+                        inc_zone_page_state(page, NR_ISOLATED_ANON +
+                                            page_is_file_cache(page));
+                }
 put_and_set:
                 /*
                  * Either remove the duplicate refcount from
@@ -894,7 +853,7 @@ set_status:
         err = 0;
         if (!list_empty(&pagelist))
                 err = migrate_pages(&pagelist, new_page_node,
-                                (unsigned long)pm);
+                                (unsigned long)pm, 0);
 
         up_read(&mm->mmap_sem);
         return err;
@@ -1015,7 +974,7 @@ static void do_pages_stat_array(struct mm_struct *mm, unsigned long nr_pages,
 
                 err = -ENOENT;
                 /* Use PageReserved to check for zero page */
-                if (!page || PageReserved(page))
+                if (!page || PageReserved(page) || PageKsm(page))
                         goto set_status;
 
                 err = page_to_nid(page);
@@ -1044,7 +1003,7 @@ static int do_pages_stat(struct mm_struct *mm, unsigned long nr_pages,
         int err;
 
         for (i = 0; i < nr_pages; i += chunk_nr) {
-                if (chunk_nr + i > nr_pages)
+                if (chunk_nr > nr_pages - i)
                         chunk_nr = nr_pages - i;
 
                 err = copy_from_user(chunk_pages, &pages[i],