9 files changed, 60 insertions, 37 deletions

diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 3e29781ee762..dbe99a5f2073 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -650,10 +650,10 @@ static inline gfp_t alloc_hugepage_gfpmask(int defrag)
 
 static inline struct page *alloc_hugepage_vma(int defrag,
                                               struct vm_area_struct *vma,
-                                              unsigned long haddr)
+                                              unsigned long haddr, int nd)
 {
         return alloc_pages_vma(alloc_hugepage_gfpmask(defrag),
-                               HPAGE_PMD_ORDER, vma, haddr);
+                               HPAGE_PMD_ORDER, vma, haddr, nd);
 }
 
 #ifndef CONFIG_NUMA
@@ -678,7 +678,7 @@ int do_huge_pmd_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma,
                 if (unlikely(khugepaged_enter(vma)))
                         return VM_FAULT_OOM;
                 page = alloc_hugepage_vma(transparent_hugepage_defrag(vma),
-                                          vma, haddr);
+                                          vma, haddr, numa_node_id());
                 if (unlikely(!page))
                         goto out;
                 if (unlikely(mem_cgroup_newpage_charge(page, mm, GFP_KERNEL))) {
@@ -799,8 +799,8 @@ static int do_huge_pmd_wp_page_fallback(struct mm_struct *mm,
         }
 
         for (i = 0; i < HPAGE_PMD_NR; i++) {
-                pages[i] = alloc_page_vma(GFP_HIGHUSER_MOVABLE,
-                                          vma, address);
+                pages[i] = alloc_page_vma_node(GFP_HIGHUSER_MOVABLE,
+                                               vma, address, page_to_nid(page));
                 if (unlikely(!pages[i] ||
                              mem_cgroup_newpage_charge(pages[i], mm,
                                                        GFP_KERNEL))) {
@@ -902,7 +902,7 @@ int do_huge_pmd_wp_page(struct mm_struct *mm, struct vm_area_struct *vma,
         if (transparent_hugepage_enabled(vma) &&
             !transparent_hugepage_debug_cow())
                 new_page = alloc_hugepage_vma(transparent_hugepage_defrag(vma),
-                                              vma, haddr);
+                                              vma, haddr, numa_node_id());
         else
                 new_page = NULL;
 
@@ -1745,7 +1745,8 @@ static void __collapse_huge_page_copy(pte_t *pte, struct page *page,
 static void collapse_huge_page(struct mm_struct *mm,
                                unsigned long address,
                                struct page **hpage,
-                               struct vm_area_struct *vma)
+                               struct vm_area_struct *vma,
+                               int node)
 {
         pgd_t *pgd;
         pud_t *pud;
@@ -1773,7 +1774,8 @@ static void collapse_huge_page(struct mm_struct *mm,
          * mmap_sem in read mode is good idea also to allow greater
          * scalability.
          */
-        new_page = alloc_hugepage_vma(khugepaged_defrag(), vma, address);
+        new_page = alloc_hugepage_vma(khugepaged_defrag(), vma, address,
+                                      node);
         if (unlikely(!new_page)) {
                 up_read(&mm->mmap_sem);
                 *hpage = ERR_PTR(-ENOMEM);
@@ -1919,6 +1921,7 @@ static int khugepaged_scan_pmd(struct mm_struct *mm,
         struct page *page;
         unsigned long _address;
         spinlock_t *ptl;
+        int node = -1;
 
         VM_BUG_ON(address & ~HPAGE_PMD_MASK);
 
@@ -1949,6 +1952,13 @@ static int khugepaged_scan_pmd(struct mm_struct *mm,
                 page = vm_normal_page(vma, _address, pteval);
                 if (unlikely(!page))
                         goto out_unmap;
+                /*
+                 * Chose the node of the first page. This could
+                 * be more sophisticated and look at more pages,
+                 * but isn't for now.
+                 */
+                if (node == -1)
+                        node = page_to_nid(page);
                 VM_BUG_ON(PageCompound(page));
                 if (!PageLRU(page) || PageLocked(page) || !PageAnon(page))
                         goto out_unmap;
@@ -1965,7 +1975,7 @@ out_unmap:
         pte_unmap_unlock(pte, ptl);
         if (ret)
                 /* collapse_huge_page will return with the mmap_sem released */
-                collapse_huge_page(mm, address, hpage, vma);
+                collapse_huge_page(mm, address, hpage, vma, node);
 out:
         return ret;
 }
diff --git a/mm/memory.c b/mm/memory.c
index 8e8c18324863..5823698c2b71 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -2648,6 +2648,7 @@ void unmap_mapping_range(struct address_space *mapping,
                 details.last_index = ULONG_MAX;
         details.i_mmap_lock = &mapping->i_mmap_lock;
 
+        mutex_lock(&mapping->unmap_mutex);
         spin_lock(&mapping->i_mmap_lock);
 
         /* Protect against endless unmapping loops */
@@ -2664,6 +2665,7 @@ void unmap_mapping_range(struct address_space *mapping,
         if (unlikely(!list_empty(&mapping->i_mmap_nonlinear)))
                 unmap_mapping_range_list(&mapping->i_mmap_nonlinear, &details);
         spin_unlock(&mapping->i_mmap_lock);
+        mutex_unlock(&mapping->unmap_mutex);
 }
 EXPORT_SYMBOL(unmap_mapping_range);
 
diff --git a/mm/mempolicy.c b/mm/mempolicy.c
index 368fc9d23610..b53ec99f1428 100644
--- a/mm/mempolicy.c
+++ b/mm/mempolicy.c
@@ -1524,10 +1524,9 @@ static nodemask_t *policy_nodemask(gfp_t gfp, struct mempolicy *policy)
 }
 
 /* Return a zonelist indicated by gfp for node representing a mempolicy */
-static struct zonelist *policy_zonelist(gfp_t gfp, struct mempolicy *policy)
+static struct zonelist *policy_zonelist(gfp_t gfp, struct mempolicy *policy,
+        int nd)
 {
-        int nd = numa_node_id();
-
         switch (policy->mode) {
         case MPOL_PREFERRED:
                 if (!(policy->flags & MPOL_F_LOCAL))
@@ -1679,7 +1678,7 @@ struct zonelist *huge_zonelist(struct vm_area_struct *vma, unsigned long addr,
                 zl = node_zonelist(interleave_nid(*mpol, vma, addr,
                                 huge_page_shift(hstate_vma(vma))), gfp_flags);
         } else {
-                zl = policy_zonelist(gfp_flags, *mpol);
+                zl = policy_zonelist(gfp_flags, *mpol, numa_node_id());
                 if ((*mpol)->mode == MPOL_BIND)
                         *nodemask = &(*mpol)->v.nodes;
         }
@@ -1820,7 +1819,7 @@ static struct page *alloc_page_interleave(gfp_t gfp, unsigned order,
  */
 struct page *
 alloc_pages_vma(gfp_t gfp, int order, struct vm_area_struct *vma,
-                unsigned long addr)
+                unsigned long addr, int node)
 {
         struct mempolicy *pol = get_vma_policy(current, vma, addr);
         struct zonelist *zl;
@@ -1830,13 +1829,13 @@ alloc_pages_vma(gfp_t gfp, int order, struct vm_area_struct *vma,
         if (unlikely(pol->mode == MPOL_INTERLEAVE)) {
                 unsigned nid;
 
-                nid = interleave_nid(pol, vma, addr, PAGE_SHIFT);
+                nid = interleave_nid(pol, vma, addr, PAGE_SHIFT + order);
                 mpol_cond_put(pol);
                 page = alloc_page_interleave(gfp, order, nid);
                 put_mems_allowed();
                 return page;
         }
-        zl = policy_zonelist(gfp, pol);
+        zl = policy_zonelist(gfp, pol, node);
         if (unlikely(mpol_needs_cond_ref(pol))) {
                 /*
                  * slow path: ref counted shared policy
@@ -1892,7 +1891,8 @@ struct page *alloc_pages_current(gfp_t gfp, unsigned order)
                 page = alloc_page_interleave(gfp, order, interleave_nodes(pol));
         else
                 page = __alloc_pages_nodemask(gfp, order,
-                        policy_zonelist(gfp, pol), policy_nodemask(gfp, pol));
+                                policy_zonelist(gfp, pol, numa_node_id()),
+                                policy_nodemask(gfp, pol));
         put_mems_allowed();
         return page;
 }
diff --git a/mm/migrate.c b/mm/migrate.c
index 766115253807..352de555626c 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -1287,14 +1287,14 @@ SYSCALL_DEFINE6(move_pages, pid_t, pid, unsigned long, nr_pages,
                 return -EPERM;
 
         /* Find the mm_struct */
-        read_lock(&tasklist_lock);
+        rcu_read_lock();
         task = pid ? find_task_by_vpid(pid) : current;
         if (!task) {
-                read_unlock(&tasklist_lock);
+                rcu_read_unlock();
                 return -ESRCH;
         }
         mm = get_task_mm(task);
-        read_unlock(&tasklist_lock);
+        rcu_read_unlock();
 
         if (!mm)
                 return -EINVAL;
diff --git a/mm/mremap.c b/mm/mremap.c
index 9925b6391b80..1de98d492ddc 100644
--- a/mm/mremap.c
+++ b/mm/mremap.c
@@ -94,9 +94,7 @@ static void move_ptes(struct vm_area_struct *vma, pmd_t *old_pmd,
                  */
                 mapping = vma->vm_file->f_mapping;
                 spin_lock(&mapping->i_mmap_lock);
-                if (new_vma->vm_truncate_count &&
-                    new_vma->vm_truncate_count != vma->vm_truncate_count)
-                        new_vma->vm_truncate_count = 0;
+                new_vma->vm_truncate_count = 0;
         }
 
         /*
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index a873e61e312e..cdef1d4b4e47 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -5376,10 +5376,9 @@ __count_immobile_pages(struct zone *zone, struct page *page, int count)
         for (found = 0, iter = 0; iter < pageblock_nr_pages; iter++) {
                 unsigned long check = pfn + iter;
 
-                if (!pfn_valid_within(check)) {
-                        iter++;
+                if (!pfn_valid_within(check))
                         continue;
-                }
+
                 page = pfn_to_page(check);
                 if (!page_count(page)) {
                         if (PageBuddy(page))
diff --git a/mm/swapfile.c b/mm/swapfile.c
index 07a458d72fa8..0341c5700e34 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -1940,7 +1940,7 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags)
 
         error = -EINVAL;
         if (S_ISBLK(inode->i_mode)) {
-                bdev = I_BDEV(inode);
+                bdev = bdgrab(I_BDEV(inode));
                 error = blkdev_get(bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL,
                                    sys_swapon);
                 if (error < 0) {
diff --git a/mm/truncate.c b/mm/truncate.c
index 49feb46e77b8..d64296be00d3 100644
--- a/mm/truncate.c
+++ b/mm/truncate.c
@@ -225,6 +225,7 @@ void truncate_inode_pages_range(struct address_space *mapping,
         next = start;
         while (next <= end &&
                pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) {
+                mem_cgroup_uncharge_start();
                 for (i = 0; i < pagevec_count(&pvec); i++) {
                         struct page *page = pvec.pages[i];
                         pgoff_t page_index = page->index;
@@ -247,6 +248,7 @@ void truncate_inode_pages_range(struct address_space *mapping,
                         unlock_page(page);
                 }
                 pagevec_release(&pvec);
+                mem_cgroup_uncharge_end();
                 cond_resched();
         }
 
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 17497d0cd8b9..6771ea70bfe7 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -1841,16 +1841,28 @@ static inline bool should_continue_reclaim(struct zone *zone,
         if (!(sc->reclaim_mode & RECLAIM_MODE_COMPACTION))
                 return false;
 
-        /*
-         * If we failed to reclaim and have scanned the full list, stop.
-         * NOTE: Checking just nr_reclaimed would exit reclaim/compaction far
-         *       faster but obviously would be less likely to succeed
-         *       allocation. If this is desirable, use GFP_REPEAT to decide
-         *       if both reclaimed and scanned should be checked or just
-         *       reclaimed
-         */
-        if (!nr_reclaimed && !nr_scanned)
-                return false;
+        /* Consider stopping depending on scan and reclaim activity */
+        if (sc->gfp_mask & __GFP_REPEAT) {
+                /*
+                 * For __GFP_REPEAT allocations, stop reclaiming if the
+                 * full LRU list has been scanned and we are still failing
+                 * to reclaim pages. This full LRU scan is potentially
+                 * expensive but a __GFP_REPEAT caller really wants to succeed
+                 */
+                if (!nr_reclaimed && !nr_scanned)
+                        return false;
+        } else {
+                /*
+                 * For non-__GFP_REPEAT allocations which can presumably
+                 * fail without consequence, stop if we failed to reclaim
+                 * any pages from the last SWAP_CLUSTER_MAX number of
+                 * pages that were scanned. This will return to the
+                 * caller faster at the risk reclaim/compaction and
+                 * the resulting allocation attempt fails
+                 */
+                if (!nr_reclaimed)
+                        return false;
+        }
 
         /*
          * If we have not reclaimed enough pages for compaction and the