Merge commit 'v2.6.38' into release

3 files changed, 66 insertions, 36 deletions

diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 3e29781ee762..113e35c47502 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -650,10 +650,10 @@ static inline gfp_t alloc_hugepage_gfpmask(int defrag)
 
 static inline struct page *alloc_hugepage_vma(int defrag,
                                               struct vm_area_struct *vma,
-                                              unsigned long haddr)
+                                              unsigned long haddr, int nd)
 {
         return alloc_pages_vma(alloc_hugepage_gfpmask(defrag),
-                               HPAGE_PMD_ORDER, vma, haddr);
+                               HPAGE_PMD_ORDER, vma, haddr, nd);
 }
 
 #ifndef CONFIG_NUMA
@@ -678,7 +678,7 @@ int do_huge_pmd_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma,
                 if (unlikely(khugepaged_enter(vma)))
                         return VM_FAULT_OOM;
                 page = alloc_hugepage_vma(transparent_hugepage_defrag(vma),
-                                          vma, haddr);
+                                          vma, haddr, numa_node_id());
                 if (unlikely(!page))
                         goto out;
                 if (unlikely(mem_cgroup_newpage_charge(page, mm, GFP_KERNEL))) {
@@ -799,8 +799,8 @@ static int do_huge_pmd_wp_page_fallback(struct mm_struct *mm,
         }
 
         for (i = 0; i < HPAGE_PMD_NR; i++) {
-                pages[i] = alloc_page_vma(GFP_HIGHUSER_MOVABLE,
-                                          vma, address);
+                pages[i] = alloc_page_vma_node(GFP_HIGHUSER_MOVABLE,
+                                               vma, address, page_to_nid(page));
                 if (unlikely(!pages[i] ||
                              mem_cgroup_newpage_charge(pages[i], mm,
                                                        GFP_KERNEL))) {
@@ -902,7 +902,7 @@ int do_huge_pmd_wp_page(struct mm_struct *mm, struct vm_area_struct *vma,
         if (transparent_hugepage_enabled(vma) &&
             !transparent_hugepage_debug_cow())
                 new_page = alloc_hugepage_vma(transparent_hugepage_defrag(vma),
-                                              vma, haddr);
+                                              vma, haddr, numa_node_id());
         else
                 new_page = NULL;
 
@@ -1745,7 +1745,8 @@ static void __collapse_huge_page_copy(pte_t *pte, struct page *page,
 static void collapse_huge_page(struct mm_struct *mm,
                                unsigned long address,
                                struct page **hpage,
-                               struct vm_area_struct *vma)
+                               struct vm_area_struct *vma,
+                               int node)
 {
         pgd_t *pgd;
         pud_t *pud;
@@ -1761,6 +1762,10 @@ static void collapse_huge_page(struct mm_struct *mm,
 #ifndef CONFIG_NUMA
         VM_BUG_ON(!*hpage);
         new_page = *hpage;
+        if (unlikely(mem_cgroup_newpage_charge(new_page, mm, GFP_KERNEL))) {
+                up_read(&mm->mmap_sem);
+                return;
+        }
 #else
         VM_BUG_ON(*hpage);
         /*
@@ -1773,18 +1778,19 @@ static void collapse_huge_page(struct mm_struct *mm,
          * mmap_sem in read mode is good idea also to allow greater
          * scalability.
          */
-        new_page = alloc_hugepage_vma(khugepaged_defrag(), vma, address);
+        new_page = alloc_hugepage_vma(khugepaged_defrag(), vma, address,
+                                      node);
         if (unlikely(!new_page)) {
                 up_read(&mm->mmap_sem);
                 *hpage = ERR_PTR(-ENOMEM);
                 return;
         }
-#endif
         if (unlikely(mem_cgroup_newpage_charge(new_page, mm, GFP_KERNEL))) {
                 up_read(&mm->mmap_sem);
                 put_page(new_page);
                 return;
         }
+#endif
 
         /* after allocating the hugepage upgrade to mmap_sem write mode */
         up_read(&mm->mmap_sem);
@@ -1919,6 +1925,7 @@ static int khugepaged_scan_pmd(struct mm_struct *mm,
         struct page *page;
         unsigned long _address;
         spinlock_t *ptl;
+        int node = -1;
 
         VM_BUG_ON(address & ~HPAGE_PMD_MASK);
 
@@ -1949,6 +1956,13 @@ static int khugepaged_scan_pmd(struct mm_struct *mm,
                 page = vm_normal_page(vma, _address, pteval);
                 if (unlikely(!page))
                         goto out_unmap;
+                /*
+                 * Chose the node of the first page. This could
+                 * be more sophisticated and look at more pages,
+                 * but isn't for now.
+                 */
+                if (node == -1)
+                        node = page_to_nid(page);
                 VM_BUG_ON(PageCompound(page));
                 if (!PageLRU(page) || PageLocked(page) || !PageAnon(page))
                         goto out_unmap;
@@ -1965,7 +1979,7 @@ out_unmap:
         pte_unmap_unlock(pte, ptl);
         if (ret)
                 /* collapse_huge_page will return with the mmap_sem released */
-                collapse_huge_page(mm, address, hpage, vma);
+                collapse_huge_page(mm, address, hpage, vma, node);
 out:
         return ret;
 }
diff --git a/mm/mempolicy.c b/mm/mempolicy.c
index 49355a970be2..b53ec99f1428 100644
--- a/mm/mempolicy.c
+++ b/mm/mempolicy.c
@@ -1524,10 +1524,9 @@ static nodemask_t *policy_nodemask(gfp_t gfp, struct mempolicy *policy)
 }
 
 /* Return a zonelist indicated by gfp for node representing a mempolicy */
-static struct zonelist *policy_zonelist(gfp_t gfp, struct mempolicy *policy)
+static struct zonelist *policy_zonelist(gfp_t gfp, struct mempolicy *policy,
+        int nd)
 {
-        int nd = numa_node_id();
-
         switch (policy->mode) {
         case MPOL_PREFERRED:
                 if (!(policy->flags & MPOL_F_LOCAL))
@@ -1679,7 +1678,7 @@ struct zonelist *huge_zonelist(struct vm_area_struct *vma, unsigned long addr,
                 zl = node_zonelist(interleave_nid(*mpol, vma, addr,
                                 huge_page_shift(hstate_vma(vma))), gfp_flags);
         } else {
-                zl = policy_zonelist(gfp_flags, *mpol);
+                zl = policy_zonelist(gfp_flags, *mpol, numa_node_id());
                 if ((*mpol)->mode == MPOL_BIND)
                         *nodemask = &(*mpol)->v.nodes;
         }
@@ -1820,7 +1819,7 @@ static struct page *alloc_page_interleave(gfp_t gfp, unsigned order,
  */
 struct page *
 alloc_pages_vma(gfp_t gfp, int order, struct vm_area_struct *vma,
-                unsigned long addr)
+                unsigned long addr, int node)
 {
         struct mempolicy *pol = get_vma_policy(current, vma, addr);
         struct zonelist *zl;
@@ -1836,7 +1835,7 @@ alloc_pages_vma(gfp_t gfp, int order, struct vm_area_struct *vma,
                 put_mems_allowed();
                 return page;
         }
-        zl = policy_zonelist(gfp, pol);
+        zl = policy_zonelist(gfp, pol, node);
         if (unlikely(mpol_needs_cond_ref(pol))) {
                 /*
                  * slow path: ref counted shared policy
@@ -1892,7 +1891,8 @@ struct page *alloc_pages_current(gfp_t gfp, unsigned order)
                 page = alloc_page_interleave(gfp, order, interleave_nodes(pol));
         else
                 page = __alloc_pages_nodemask(gfp, order,
-                        policy_zonelist(gfp, pol), policy_nodemask(gfp, pol));
+                                policy_zonelist(gfp, pol, numa_node_id()),
+                                policy_nodemask(gfp, pol));
         put_mems_allowed();
         return page;
 }
diff --git a/mm/rmap.c b/mm/rmap.c
index f21f4a1d6a1c..941bf82e8961 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -497,41 +497,51 @@ int page_referenced_one(struct page *page, struct vm_area_struct *vma,
         struct mm_struct *mm = vma->vm_mm;
         int referenced = 0;
 
-        /*
-         * Don't want to elevate referenced for mlocked page that gets this far,
-         * in order that it progresses to try_to_unmap and is moved to the
-         * unevictable list.
-         */
-        if (vma->vm_flags & VM_LOCKED) {
-                *mapcount = 0;  /* break early from loop */
-                *vm_flags |= VM_LOCKED;
-                goto out;
-        }
-
-        /* Pretend the page is referenced if the task has the
-           swap token and is in the middle of a page fault. */
-        if (mm != current->mm && has_swap_token(mm) &&
-                        rwsem_is_locked(&mm->mmap_sem))
-                referenced++;
-
         if (unlikely(PageTransHuge(page))) {
                 pmd_t *pmd;
 
                 spin_lock(&mm->page_table_lock);
+                /*
+                 * rmap might return false positives; we must filter
+                 * these out using page_check_address_pmd().
+                 */
                 pmd = page_check_address_pmd(page, mm, address,
                                              PAGE_CHECK_ADDRESS_PMD_FLAG);
-                if (pmd && !pmd_trans_splitting(*pmd) &&
-                    pmdp_clear_flush_young_notify(vma, address, pmd))
+                if (!pmd) {
+                        spin_unlock(&mm->page_table_lock);
+                        goto out;
+                }
+
+                if (vma->vm_flags & VM_LOCKED) {
+                        spin_unlock(&mm->page_table_lock);
+                        *mapcount = 0;  /* break early from loop */
+                        *vm_flags |= VM_LOCKED;
+                        goto out;
+                }
+
+                /* go ahead even if the pmd is pmd_trans_splitting() */
+                if (pmdp_clear_flush_young_notify(vma, address, pmd))
                         referenced++;
                 spin_unlock(&mm->page_table_lock);
         } else {
                 pte_t *pte;
                 spinlock_t *ptl;
 
+                /*
+                 * rmap might return false positives; we must filter
+                 * these out using page_check_address().
+                 */
                 pte = page_check_address(page, mm, address, &ptl, 0);
                 if (!pte)
                         goto out;
 
+                if (vma->vm_flags & VM_LOCKED) {
+                        pte_unmap_unlock(pte, ptl);
+                        *mapcount = 0;  /* break early from loop */
+                        *vm_flags |= VM_LOCKED;
+                        goto out;
+                }
+
                 if (ptep_clear_flush_young_notify(vma, address, pte)) {
                         /*
                          * Don't treat a reference through a sequentially read
@@ -546,6 +556,12 @@ int page_referenced_one(struct page *page, struct vm_area_struct *vma,
                 pte_unmap_unlock(pte, ptl);
         }
 
+        /* Pretend the page is referenced if the task has the
+           swap token and is in the middle of a page fault. */
+        if (mm != current->mm && has_swap_token(mm) &&
+                        rwsem_is_locked(&mm->mmap_sem))
+                referenced++;
+
         (*mapcount)--;
 
         if (referenced)