1 files changed, 86 insertions, 30 deletions

diff --git a/mm/mempolicy.c b/mm/mempolicy.c
index 4baf12e534d1..04729647f359 100644
--- a/mm/mempolicy.c
+++ b/mm/mempolicy.c
@@ -123,16 +123,19 @@ static struct mempolicy preferred_node_policy[MAX_NUMNODES];
 static struct mempolicy *get_task_policy(struct task_struct *p)
 {
         struct mempolicy *pol = p->mempolicy;
-        int node;
 
         if (!pol) {
-                node = numa_node_id();
-                if (node != NUMA_NO_NODE)
-                        pol = &preferred_node_policy[node];
+                int node = numa_node_id();
 
-                /* preferred_node_policy is not initialised early in boot */
-                if (!pol->mode)
-                        pol = NULL;
+                if (node != NUMA_NO_NODE) {
+                        pol = &preferred_node_policy[node];
+                        /*
+                         * preferred_node_policy is not initialised early in
+                         * boot
+                         */
+                        if (!pol->mode)
+                                pol = NULL;
+                }
         }
 
         return pol;
@@ -473,8 +476,11 @@ static const struct mempolicy_operations mpol_ops[MPOL_MAX] = {
 static void migrate_page_add(struct page *page, struct list_head *pagelist,
                                 unsigned long flags);
 
-/* Scan through pages checking if pages follow certain conditions. */
-static int check_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
+/*
+ * Scan through pages checking if pages follow certain conditions,
+ * and move them to the pagelist if they do.
+ */
+static int queue_pages_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
                 unsigned long addr, unsigned long end,
                 const nodemask_t *nodes, unsigned long flags,
                 void *private)
@@ -512,7 +518,31 @@ static int check_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
         return addr != end;
 }
 
-static inline int check_pmd_range(struct vm_area_struct *vma, pud_t *pud,
+static void queue_pages_hugetlb_pmd_range(struct vm_area_struct *vma,
+                pmd_t *pmd, const nodemask_t *nodes, unsigned long flags,
+                                    void *private)
+{
+#ifdef CONFIG_HUGETLB_PAGE
+        int nid;
+        struct page *page;
+
+        spin_lock(&vma->vm_mm->page_table_lock);
+        page = pte_page(huge_ptep_get((pte_t *)pmd));
+        nid = page_to_nid(page);
+        if (node_isset(nid, *nodes) == !!(flags & MPOL_MF_INVERT))
+                goto unlock;
+        /* With MPOL_MF_MOVE, we migrate only unshared hugepage. */
+        if (flags & (MPOL_MF_MOVE_ALL) ||
+            (flags & MPOL_MF_MOVE && page_mapcount(page) == 1))
+                isolate_huge_page(page, private);
+unlock:
+        spin_unlock(&vma->vm_mm->page_table_lock);
+#else
+        BUG();
+#endif
+}
+
+static inline int queue_pages_pmd_range(struct vm_area_struct *vma, pud_t *pud,
                 unsigned long addr, unsigned long end,
                 const nodemask_t *nodes, unsigned long flags,
                 void *private)
@@ -523,17 +553,24 @@ static inline int check_pmd_range(struct vm_area_struct *vma, pud_t *pud,
         pmd = pmd_offset(pud, addr);
         do {
                 next = pmd_addr_end(addr, end);
+                if (!pmd_present(*pmd))
+                        continue;
+                if (pmd_huge(*pmd) && is_vm_hugetlb_page(vma)) {
+                        queue_pages_hugetlb_pmd_range(vma, pmd, nodes,
+                                                flags, private);
+                        continue;
+                }
                 split_huge_page_pmd(vma, addr, pmd);
                 if (pmd_none_or_trans_huge_or_clear_bad(pmd))
                         continue;
-                if (check_pte_range(vma, pmd, addr, next, nodes,
+                if (queue_pages_pte_range(vma, pmd, addr, next, nodes,
                                     flags, private))
                         return -EIO;
         } while (pmd++, addr = next, addr != end);
         return 0;
 }
 
-static inline int check_pud_range(struct vm_area_struct *vma, pgd_t *pgd,
+static inline int queue_pages_pud_range(struct vm_area_struct *vma, pgd_t *pgd,
                 unsigned long addr, unsigned long end,
                 const nodemask_t *nodes, unsigned long flags,
                 void *private)
@@ -544,16 +581,18 @@ static inline int check_pud_range(struct vm_area_struct *vma, pgd_t *pgd,
         pud = pud_offset(pgd, addr);
         do {
                 next = pud_addr_end(addr, end);
+                if (pud_huge(*pud) && is_vm_hugetlb_page(vma))
+                        continue;
                 if (pud_none_or_clear_bad(pud))
                         continue;
-                if (check_pmd_range(vma, pud, addr, next, nodes,
+                if (queue_pages_pmd_range(vma, pud, addr, next, nodes,
                                     flags, private))
                         return -EIO;
         } while (pud++, addr = next, addr != end);
         return 0;
 }
 
-static inline int check_pgd_range(struct vm_area_struct *vma,
+static inline int queue_pages_pgd_range(struct vm_area_struct *vma,
                 unsigned long addr, unsigned long end,
                 const nodemask_t *nodes, unsigned long flags,
                 void *private)
@@ -566,7 +605,7 @@ static inline int check_pgd_range(struct vm_area_struct *vma,
                 next = pgd_addr_end(addr, end);
                 if (pgd_none_or_clear_bad(pgd))
                         continue;
-                if (check_pud_range(vma, pgd, addr, next, nodes,
+                if (queue_pages_pud_range(vma, pgd, addr, next, nodes,
                                     flags, private))
                         return -EIO;
         } while (pgd++, addr = next, addr != end);
@@ -604,12 +643,14 @@ static unsigned long change_prot_numa(struct vm_area_struct *vma,
 #endif /* CONFIG_ARCH_USES_NUMA_PROT_NONE */
 
 /*
- * Check if all pages in a range are on a set of nodes.
- * If pagelist != NULL then isolate pages from the LRU and
- * put them on the pagelist.
+ * Walk through page tables and collect pages to be migrated.
+ *
+ * If pages found in a given range are on a set of nodes (determined by
+ * @nodes and @flags,) it's isolated and queued to the pagelist which is
+ * passed via @private.)
  */
 static struct vm_area_struct *
-check_range(struct mm_struct *mm, unsigned long start, unsigned long end,
+queue_pages_range(struct mm_struct *mm, unsigned long start, unsigned long end,
                 const nodemask_t *nodes, unsigned long flags, void *private)
 {
         int err;
@@ -635,9 +676,6 @@ check_range(struct mm_struct *mm, unsigned long start, unsigned long end,
                                 return ERR_PTR(-EFAULT);
                 }
 
-                if (is_vm_hugetlb_page(vma))
-                        goto next;
-
                 if (flags & MPOL_MF_LAZY) {
                         change_prot_numa(vma, start, endvma);
                         goto next;
@@ -647,7 +685,7 @@ check_range(struct mm_struct *mm, unsigned long start, unsigned long end,
                      ((flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) &&
                       vma_migratable(vma))) {
 
-                        err = check_pgd_range(vma, start, endvma, nodes,
+                        err = queue_pages_pgd_range(vma, start, endvma, nodes,
                                                 flags, private);
                         if (err) {
                                 first = ERR_PTR(err);
@@ -990,7 +1028,11 @@ static void migrate_page_add(struct page *page, struct list_head *pagelist,
 
 static struct page *new_node_page(struct page *page, unsigned long node, int **x)
 {
-        return alloc_pages_exact_node(node, GFP_HIGHUSER_MOVABLE, 0);
+        if (PageHuge(page))
+                return alloc_huge_page_node(page_hstate(compound_head(page)),
+                                        node);
+        else
+                return alloc_pages_exact_node(node, GFP_HIGHUSER_MOVABLE, 0);
 }
 
 /*
@@ -1013,14 +1055,14 @@ static int migrate_to_node(struct mm_struct *mm, int source, int dest,
          * space range and MPOL_MF_DISCONTIG_OK, this call can not fail.
          */
         VM_BUG_ON(!(flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)));
-        check_range(mm, mm->mmap->vm_start, mm->task_size, &nmask,
+        queue_pages_range(mm, mm->mmap->vm_start, mm->task_size, &nmask,
                         flags | MPOL_MF_DISCONTIG_OK, &pagelist);
 
         if (!list_empty(&pagelist)) {
                 err = migrate_pages(&pagelist, new_node_page, dest,
                                         MIGRATE_SYNC, MR_SYSCALL);
                 if (err)
-                        putback_lru_pages(&pagelist);
+                        putback_movable_pages(&pagelist);
         }
 
         return err;
@@ -1154,10 +1196,14 @@ static struct page *new_vma_page(struct page *page, unsigned long private, int *
                         break;
                 vma = vma->vm_next;
         }
-
         /*
-         * if !vma, alloc_page_vma() will use task or system default policy
+         * queue_pages_range() confirms that @page belongs to some vma,
+         * so vma shouldn't be NULL.
          */
+        BUG_ON(!vma);
+
+        if (PageHuge(page))
+                return alloc_huge_page_noerr(vma, address, 1);
         return alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, address);
 }
 #else
@@ -1249,7 +1295,7 @@ static long do_mbind(unsigned long start, unsigned long len,
         if (err)
                 goto mpol_out;
 
-        vma = check_range(mm, start, end, nmask,
+        vma = queue_pages_range(mm, start, end, nmask,
                           flags | MPOL_MF_INVERT, &pagelist);
 
         err = PTR_ERR(vma);     /* maybe ... */
@@ -1265,7 +1311,7 @@ static long do_mbind(unsigned long start, unsigned long len,
                                         (unsigned long)vma,
                                         MIGRATE_SYNC, MR_MEMPOLICY_MBIND);
                         if (nr_failed)
-                                putback_lru_pages(&pagelist);
+                                putback_movable_pages(&pagelist);
                 }
 
                 if (nr_failed && (flags & MPOL_MF_STRICT))
@@ -2065,6 +2111,16 @@ retry_cpuset:
 }
 EXPORT_SYMBOL(alloc_pages_current);
 
+int vma_dup_policy(struct vm_area_struct *src, struct vm_area_struct *dst)
+{
+        struct mempolicy *pol = mpol_dup(vma_policy(src));
+
+        if (IS_ERR(pol))
+                return PTR_ERR(pol);
+        dst->vm_policy = pol;
+        return 0;
+}
+
 /*
  * If mpol_dup() sees current->cpuset == cpuset_being_rebound, then it
  * rebinds the mempolicy its copying by calling mpol_rebind_policy()