10 files changed, 97 insertions, 88 deletions

diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 0a619e0e2e0b..470dcda10add 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -244,24 +244,28 @@ static ssize_t single_flag_show(struct kobject *kobj,
                                 struct kobj_attribute *attr, char *buf,
                                 enum transparent_hugepage_flag flag)
 {
-        if (test_bit(flag, &transparent_hugepage_flags))
-                return sprintf(buf, "[yes] no\n");
-        else
-                return sprintf(buf, "yes [no]\n");
+        return sprintf(buf, "%d\n",
+                       !!test_bit(flag, &transparent_hugepage_flags));
 }
+
 static ssize_t single_flag_store(struct kobject *kobj,
                                  struct kobj_attribute *attr,
                                  const char *buf, size_t count,
                                  enum transparent_hugepage_flag flag)
 {
-        if (!memcmp("yes", buf,
-                    min(sizeof("yes")-1, count))) {
+        unsigned long value;
+        int ret;
+
+        ret = kstrtoul(buf, 10, &value);
+        if (ret < 0)
+                return ret;
+        if (value > 1)
+                return -EINVAL;
+
+        if (value)
                 set_bit(flag, &transparent_hugepage_flags);
-        } else if (!memcmp("no", buf,
-                           min(sizeof("no")-1, count))) {
+        else
                 clear_bit(flag, &transparent_hugepage_flags);
-        } else
-                return -EINVAL;
 
         return count;
 }
@@ -680,8 +684,11 @@ int do_huge_pmd_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma,
                         return VM_FAULT_OOM;
                 page = alloc_hugepage_vma(transparent_hugepage_defrag(vma),
                                           vma, haddr, numa_node_id(), 0);
-                if (unlikely(!page))
+                if (unlikely(!page)) {
+                        count_vm_event(THP_FAULT_FALLBACK);
                         goto out;
+                }
+                count_vm_event(THP_FAULT_ALLOC);
                 if (unlikely(mem_cgroup_newpage_charge(page, mm, GFP_KERNEL))) {
                         put_page(page);
                         goto out;
@@ -909,11 +916,13 @@ int do_huge_pmd_wp_page(struct mm_struct *mm, struct vm_area_struct *vma,
                 new_page = NULL;
 
         if (unlikely(!new_page)) {
+                count_vm_event(THP_FAULT_FALLBACK);
                 ret = do_huge_pmd_wp_page_fallback(mm, vma, address,
                                                    pmd, orig_pmd, page, haddr);
                 put_page(page);
                 goto out;
         }
+        count_vm_event(THP_FAULT_ALLOC);
 
         if (unlikely(mem_cgroup_newpage_charge(new_page, mm, GFP_KERNEL))) {
                 put_page(new_page);
@@ -1390,6 +1399,7 @@ int split_huge_page(struct page *page)
 
         BUG_ON(!PageSwapBacked(page));
         __split_huge_page(page, anon_vma);
+        count_vm_event(THP_SPLIT);
 
         BUG_ON(PageCompound(page));
 out_unlock:
@@ -1784,9 +1794,11 @@ static void collapse_huge_page(struct mm_struct *mm,
                                       node, __GFP_OTHER_NODE);
         if (unlikely(!new_page)) {
                 up_read(&mm->mmap_sem);
+                count_vm_event(THP_COLLAPSE_ALLOC_FAILED);
                 *hpage = ERR_PTR(-ENOMEM);
                 return;
         }
+        count_vm_event(THP_COLLAPSE_ALLOC);
         if (unlikely(mem_cgroup_newpage_charge(new_page, mm, GFP_KERNEL))) {
                 up_read(&mm->mmap_sem);
                 put_page(new_page);
@@ -2151,8 +2163,11 @@ static void khugepaged_do_scan(struct page **hpage)
 #ifndef CONFIG_NUMA
                 if (!*hpage) {
                         *hpage = alloc_hugepage(khugepaged_defrag());
-                        if (unlikely(!*hpage))
+                        if (unlikely(!*hpage)) {
+                                count_vm_event(THP_COLLAPSE_ALLOC_FAILED);
                                 break;
+                        }
+                        count_vm_event(THP_COLLAPSE_ALLOC);
                 }
 #else
                 if (IS_ERR(*hpage))
@@ -2192,8 +2207,11 @@ static struct page *khugepaged_alloc_hugepage(void)
 
         do {
                 hpage = alloc_hugepage(khugepaged_defrag());
-                if (!hpage)
+                if (!hpage) {
+                        count_vm_event(THP_COLLAPSE_ALLOC_FAILED);
                         khugepaged_alloc_sleep();
+                } else
+                        count_vm_event(THP_COLLAPSE_ALLOC);
         } while (unlikely(!hpage) &&
                  likely(khugepaged_enabled()));
         return hpage;
@@ -2210,8 +2228,11 @@ static void khugepaged_loop(void)
         while (likely(khugepaged_enabled())) {
 #ifndef CONFIG_NUMA
                 hpage = khugepaged_alloc_hugepage();
-                if (unlikely(!hpage))
+                if (unlikely(!hpage)) {
+                        count_vm_event(THP_COLLAPSE_ALLOC_FAILED);
                         break;
+                }
+                count_vm_event(THP_COLLAPSE_ALLOC);
 #else
                 if (IS_ERR(hpage)) {
                         khugepaged_alloc_sleep();
diff --git a/mm/memory.c b/mm/memory.c
index 9da8cab1b1b0..ce22a250926f 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -1410,6 +1410,13 @@ no_page_table:
         return page;
 }
 
+static inline int stack_guard_page(struct vm_area_struct *vma, unsigned long addr)
+{
+        return (vma->vm_flags & VM_GROWSDOWN) &&
+                (vma->vm_start == addr) &&
+                !vma_stack_continue(vma->vm_prev, addr);
+}
+
 /**
  * __get_user_pages() - pin user pages in memory
  * @tsk:        task_struct of target task
@@ -1488,7 +1495,6 @@ int __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
                 vma = find_extend_vma(mm, start);
                 if (!vma && in_gate_area(mm, start)) {
                         unsigned long pg = start & PAGE_MASK;
-                        struct vm_area_struct *gate_vma = get_gate_vma(mm);
                         pgd_t *pgd;
                         pud_t *pud;
                         pmd_t *pmd;
@@ -1513,10 +1519,11 @@ int __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
                                 pte_unmap(pte);
                                 return i ? : -EFAULT;
                         }
+                        vma = get_gate_vma(mm);
                         if (pages) {
                                 struct page *page;
 
-                                page = vm_normal_page(gate_vma, start, *pte);
+                                page = vm_normal_page(vma, start, *pte);
                                 if (!page) {
                                         if (!(gup_flags & FOLL_DUMP) &&
                                              is_zero_pfn(pte_pfn(*pte)))
@@ -1530,12 +1537,7 @@ int __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
                                 get_page(page);
                         }
                         pte_unmap(pte);
-                        if (vmas)
-                                vmas[i] = gate_vma;
-                        i++;
-                        start += PAGE_SIZE;
-                        nr_pages--;
-                        continue;
+                        goto next_page;
                 }
 
                 if (!vma ||
@@ -1549,6 +1551,13 @@ int __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
                         continue;
                 }
 
+                /*
+                 * If we don't actually want the page itself,
+                 * and it's the stack guard page, just skip it.
+                 */
+                if (!pages && stack_guard_page(vma, start))
+                        goto next_page;
+
                 do {
                         struct page *page;
                         unsigned int foll_flags = gup_flags;
@@ -1631,6 +1640,7 @@ int __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
                                 flush_anon_page(vma, page, start);
                                 flush_dcache_page(page);
                         }
+next_page:
                         if (vmas)
                                 vmas[i] = vma;
                         i++;
@@ -3678,7 +3688,7 @@ static int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm,
                          */
 #ifdef CONFIG_HAVE_IOREMAP_PROT
                         vma = find_vma(mm, addr);
-                        if (!vma)
+                        if (!vma || vma->vm_start > addr)
                                 break;
                         if (vma->vm_ops && vma->vm_ops->access)
                                 ret = vma->vm_ops->access(vma, addr, buf,
diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c
index a2acaf820fe5..9ca1d604f7cd 100644
--- a/mm/memory_hotplug.c
+++ b/mm/memory_hotplug.c
@@ -375,7 +375,7 @@ void online_page(struct page *page)
 #endif
 
 #ifdef CONFIG_FLATMEM
-        max_mapnr = max(page_to_pfn(page), max_mapnr);
+        max_mapnr = max(pfn, max_mapnr);
 #endif
 
         ClearPageReserved(page);
diff --git a/mm/mlock.c b/mm/mlock.c
index 2689a08c79af..6b55e3efe0df 100644
--- a/mm/mlock.c
+++ b/mm/mlock.c
@@ -135,13 +135,6 @@ void munlock_vma_page(struct page *page)
         }
 }
 
-static inline int stack_guard_page(struct vm_area_struct *vma, unsigned long addr)
-{
-        return (vma->vm_flags & VM_GROWSDOWN) &&
-                (vma->vm_start == addr) &&
-                !vma_stack_continue(vma->vm_prev, addr);
-}
-
 /**
  * __mlock_vma_pages_range() -  mlock a range of pages in the vma.
  * @vma:   target vma
@@ -188,12 +181,6 @@ static long __mlock_vma_pages_range(struct vm_area_struct *vma,
         if (vma->vm_flags & VM_LOCKED)
                 gup_flags |= FOLL_MLOCK;
 
-        /* We don't try to access the guard page of a stack vma */
-        if (stack_guard_page(vma, start)) {
-                addr += PAGE_SIZE;
-                nr_pages--;
-        }
-
         return __get_user_pages(current, mm, addr, nr_pages, gup_flags,
                                 NULL, NULL, nonblocking);
 }
diff --git a/mm/mmap.c b/mm/mmap.c
index 2ec8eb5a9cdd..e27e0cf0de03 100644
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -259,7 +259,7 @@ SYSCALL_DEFINE1(brk, unsigned long, brk)
          * randomize_va_space to 2, which will still cause mm->start_brk
          * to be arbitrarily shifted
          */
-        if (mm->start_brk > PAGE_ALIGN(mm->end_data))
+        if (current->brk_randomized)
                 min_brk = mm->start_brk;
         else
                 min_brk = mm->end_data;
@@ -1814,11 +1814,14 @@ static int expand_downwards(struct vm_area_struct *vma,
                 size = vma->vm_end - address;
                 grow = (vma->vm_start - address) >> PAGE_SHIFT;
 
-                error = acct_stack_growth(vma, size, grow);
-                if (!error) {
-                        vma->vm_start = address;
-                        vma->vm_pgoff -= grow;
-                        perf_event_mmap(vma);
+                error = -ENOMEM;
+                if (grow <= vma->vm_pgoff) {
+                        error = acct_stack_growth(vma, size, grow);
+                        if (!error) {
+                                vma->vm_start = address;
+                                vma->vm_pgoff -= grow;
+                                perf_event_mmap(vma);
+                        }
                 }
         }
         vma_unlock_anon_vma(vma);
diff --git a/mm/oom_kill.c b/mm/oom_kill.c
index 6a819d1b2c7d..83fb72c108b7 100644
--- a/mm/oom_kill.c
+++ b/mm/oom_kill.c
@@ -84,24 +84,6 @@ static bool has_intersects_mems_allowed(struct task_struct *tsk,
 #endif /* CONFIG_NUMA */
 
 /*
- * If this is a system OOM (not a memcg OOM) and the task selected to be
- * killed is not already running at high (RT) priorities, speed up the
- * recovery by boosting the dying task to the lowest FIFO priority.
- * That helps with the recovery and avoids interfering with RT tasks.
- */
-static void boost_dying_task_prio(struct task_struct *p,
-                                  struct mem_cgroup *mem)
-{
-        struct sched_param param = { .sched_priority = 1 };
-
-        if (mem)
-                return;
-
-        if (!rt_task(p))
-                sched_setscheduler_nocheck(p, SCHED_FIFO, &param);
-}
-
-/*
  * The process p may have detached its own ->mm while exiting or through
  * use_mm(), but one or more of its subthreads may still have a valid
  * pointer.  Return p, or any of its subthreads with a valid ->mm, with
@@ -452,13 +434,6 @@ static int oom_kill_task(struct task_struct *p, struct mem_cgroup *mem)
         set_tsk_thread_flag(p, TIF_MEMDIE);
         force_sig(SIGKILL, p);
 
-        /*
-         * We give our sacrificial lamb high priority and access to
-         * all the memory it needs. That way it should be able to
-         * exit() and clear out its resources quickly...
-         */
-        boost_dying_task_prio(p, mem);
-
         return 0;
 }
 #undef K
@@ -482,7 +457,6 @@ static int oom_kill_process(struct task_struct *p, gfp_t gfp_mask, int order,
          */
         if (p->flags & PF_EXITING) {
                 set_tsk_thread_flag(p, TIF_MEMDIE);
-                boost_dying_task_prio(p, mem);
                 return 0;
         }
 
@@ -556,7 +530,6 @@ void mem_cgroup_out_of_memory(struct mem_cgroup *mem, gfp_t gfp_mask)
          */
         if (fatal_signal_pending(current)) {
                 set_thread_flag(TIF_MEMDIE);
-                boost_dying_task_prio(current, NULL);
                 return;
         }
 
@@ -712,7 +685,6 @@ void out_of_memory(struct zonelist *zonelist, gfp_t gfp_mask,
          */
         if (fatal_signal_pending(current)) {
                 set_thread_flag(TIF_MEMDIE);
-                boost_dying_task_prio(current, NULL);
                 return;
         }
 
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 2747f5e5abc1..9f8a97b9a350 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -3176,7 +3176,7 @@ static __init_refok int __build_all_zonelists(void *data)
  * Called with zonelists_mutex held always
  * unless system_state == SYSTEM_BOOTING.
  */
-void build_all_zonelists(void *data)
+void __ref build_all_zonelists(void *data)
 {
         set_zonelist_order();
 
diff --git a/mm/shmem.c b/mm/shmem.c
index 58da7c150ba6..8fa27e4e582a 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -421,7 +421,8 @@ static swp_entry_t *shmem_swp_alloc(struct shmem_inode_info *info, unsigned long
                  * a waste to allocate index if we cannot allocate data.
                  */
                 if (sbinfo->max_blocks) {
-                        if (percpu_counter_compare(&sbinfo->used_blocks, (sbinfo->max_blocks - 1)) > 0)
+                        if (percpu_counter_compare(&sbinfo->used_blocks,
+                                                sbinfo->max_blocks - 1) >= 0)
                                 return ERR_PTR(-ENOSPC);
                         percpu_counter_inc(&sbinfo->used_blocks);
                         spin_lock(&inode->i_lock);
@@ -1397,7 +1398,8 @@ repeat:
                 shmem_swp_unmap(entry);
                 sbinfo = SHMEM_SB(inode->i_sb);
                 if (sbinfo->max_blocks) {
-                        if ((percpu_counter_compare(&sbinfo->used_blocks, sbinfo->max_blocks) > 0) ||
+                        if (percpu_counter_compare(&sbinfo->used_blocks,
+                                                sbinfo->max_blocks) >= 0 ||
                             shmem_acct_block(info->flags)) {
                                 spin_unlock(&info->lock);
                                 error = -ENOSPC;
diff --git a/mm/vmscan.c b/mm/vmscan.c
index c7f5a6d4b75b..f6b435c80079 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -41,6 +41,7 @@
 #include <linux/memcontrol.h>
 #include <linux/delayacct.h>
 #include <linux/sysctl.h>
+#include <linux/oom.h>
 
 #include <asm/tlbflush.h>
 #include <asm/div64.h>
@@ -1988,17 +1989,12 @@ static bool zone_reclaimable(struct zone *zone)
         return zone->pages_scanned < zone_reclaimable_pages(zone) * 6;
 }
 
-/*
- * As hibernation is going on, kswapd is freezed so that it can't mark
- * the zone into all_unreclaimable. It can't handle OOM during hibernation.
- * So let's check zone's unreclaimable in direct reclaim as well as kswapd.
- */
+/* All zones in zonelist are unreclaimable? */
 static bool all_unreclaimable(struct zonelist *zonelist,
                 struct scan_control *sc)
 {
         struct zoneref *z;
         struct zone *zone;
-        bool all_unreclaimable = true;
 
         for_each_zone_zonelist_nodemask(zone, z, zonelist,
                         gfp_zone(sc->gfp_mask), sc->nodemask) {
@@ -2006,13 +2002,11 @@ static bool all_unreclaimable(struct zonelist *zonelist,
                         continue;
                 if (!cpuset_zone_allowed_hardwall(zone, GFP_KERNEL))
                         continue;
-                if (zone_reclaimable(zone)) {
-                        all_unreclaimable = false;
-                        break;
-                }
+                if (!zone->all_unreclaimable)
+                        return false;
         }
 
-        return all_unreclaimable;
+        return true;
 }
 
 /*
@@ -2108,6 +2102,14 @@ out:
         if (sc->nr_reclaimed)
                 return sc->nr_reclaimed;
 
+        /*
+         * As hibernation is going on, kswapd is freezed so that it can't mark
+         * the zone into all_unreclaimable. Thus bypassing all_unreclaimable
+         * check.
+         */
+        if (oom_killer_disabled)
+                return 0;
+
         /* top priority shrink_zones still had more to do? don't OOM, then */
         if (scanning_global_lru(sc) && !all_unreclaimable(zonelist, sc))
                 return 1;
diff --git a/mm/vmstat.c b/mm/vmstat.c
index 772b39b87d95..897ea9e88238 100644
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -321,9 +321,12 @@ static inline void mod_state(struct zone *zone,
                 /*
                  * The fetching of the stat_threshold is racy. We may apply
                  * a counter threshold to the wrong the cpu if we get
-                 * rescheduled while executing here. However, the following
-                 * will apply the threshold again and therefore bring the
-                 * counter under the threshold.
+                 * rescheduled while executing here. However, the next
+                 * counter update will apply the threshold again and
+                 * therefore bring the counter under the threshold again.
+                 *
+                 * Most of the time the thresholds are the same anyways
+                 * for all cpus in a zone.
                  */
                 t = this_cpu_read(pcp->stat_threshold);
 
@@ -945,7 +948,16 @@ static const char * const vmstat_text[] = {
         "unevictable_pgs_cleared",
         "unevictable_pgs_stranded",
         "unevictable_pgs_mlockfreed",
+
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+        "thp_fault_alloc",
+        "thp_fault_fallback",
+        "thp_collapse_alloc",
+        "thp_collapse_alloc_failed",
+        "thp_split",
 #endif
+
+#endif /* CONFIG_VM_EVENTS_COUNTERS */
 };
 
 static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat,