8 files changed, 105 insertions, 43 deletions

diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 86f9f8b82f8e..df67b53ae3c5 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -232,7 +232,7 @@ retry:
         return READ_ONCE(huge_zero_page);
 }
 
-static void put_huge_zero_page(void)
+void put_huge_zero_page(void)
 {
         /*
          * Counter should never go to zero here. Only shrinker can put
@@ -1684,12 +1684,12 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
         if (vma_is_dax(vma)) {
                 spin_unlock(ptl);
                 if (is_huge_zero_pmd(orig_pmd))
-                        put_huge_zero_page();
+                        tlb_remove_page(tlb, pmd_page(orig_pmd));
         } else if (is_huge_zero_pmd(orig_pmd)) {
                 pte_free(tlb->mm, pgtable_trans_huge_withdraw(tlb->mm, pmd));
                 atomic_long_dec(&tlb->mm->nr_ptes);
                 spin_unlock(ptl);
-                put_huge_zero_page();
+                tlb_remove_page(tlb, pmd_page(orig_pmd));
         } else {
                 struct page *page = pmd_page(orig_pmd);
                 page_remove_rmap(page, true);
@@ -1960,10 +1960,9 @@ int khugepaged_enter_vma_merge(struct vm_area_struct *vma,
                  * page fault if needed.
                  */
                 return 0;
-        if (vma->vm_ops)
+        if (vma->vm_ops || (vm_flags & VM_NO_THP))
                 /* khugepaged not yet working on file or special mappings */
                 return 0;
-        VM_BUG_ON_VMA(vm_flags & VM_NO_THP, vma);
         hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK;
         hend = vma->vm_end & HPAGE_PMD_MASK;
         if (hstart < hend)
@@ -2352,8 +2351,7 @@ static bool hugepage_vma_check(struct vm_area_struct *vma)
                 return false;
         if (is_vma_temporary_stack(vma))
                 return false;
-        VM_BUG_ON_VMA(vma->vm_flags & VM_NO_THP, vma);
-        return true;
+        return !(vma->vm_flags & VM_NO_THP);
 }
 
 static void collapse_huge_page(struct mm_struct *mm,
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 36db05fa8acb..fe787f5c41bd 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -207,6 +207,7 @@ static void mem_cgroup_oom_notify(struct mem_cgroup *memcg);
 /* "mc" and its members are protected by cgroup_mutex */
 static struct move_charge_struct {
         spinlock_t        lock; /* for from, to */
+        struct mm_struct  *mm;
         struct mem_cgroup *from;
         struct mem_cgroup *to;
         unsigned long flags;
@@ -4667,6 +4668,8 @@ static void __mem_cgroup_clear_mc(void)
 
 static void mem_cgroup_clear_mc(void)
 {
+        struct mm_struct *mm = mc.mm;
+
         /*
          * we must clear moving_task before waking up waiters at the end of
          * task migration.
@@ -4676,7 +4679,10 @@ static void mem_cgroup_clear_mc(void)
         spin_lock(&mc.lock);
         mc.from = NULL;
         mc.to = NULL;
+        mc.mm = NULL;
         spin_unlock(&mc.lock);
+
+        mmput(mm);
 }
 
 static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
@@ -4733,6 +4739,7 @@ static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
                 VM_BUG_ON(mc.moved_swap);
 
                 spin_lock(&mc.lock);
+                mc.mm = mm;
                 mc.from = from;
                 mc.to = memcg;
                 mc.flags = move_flags;
@@ -4742,8 +4749,9 @@ static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
                 ret = mem_cgroup_precharge_mc(mm);
                 if (ret)
                         mem_cgroup_clear_mc();
+        } else {
+                mmput(mm);
         }
-        mmput(mm);
         return ret;
 }
 
@@ -4852,11 +4860,11 @@ put:			/* get_mctgt_type() gets the page */
         return ret;
 }
 
-static void mem_cgroup_move_charge(struct mm_struct *mm)
+static void mem_cgroup_move_charge(void)
 {
         struct mm_walk mem_cgroup_move_charge_walk = {
                 .pmd_entry = mem_cgroup_move_charge_pte_range,
-                .mm = mm,
+                .mm = mc.mm,
         };
 
         lru_add_drain_all();
@@ -4868,7 +4876,7 @@ static void mem_cgroup_move_charge(struct mm_struct *mm)
         atomic_inc(&mc.from->moving_account);
         synchronize_rcu();
 retry:
-        if (unlikely(!down_read_trylock(&mm->mmap_sem))) {
+        if (unlikely(!down_read_trylock(&mc.mm->mmap_sem))) {
                 /*
                  * Someone who are holding the mmap_sem might be waiting in
                  * waitq. So we cancel all extra charges, wake up all waiters,
@@ -4885,23 +4893,16 @@ retry:
          * additional charge, the page walk just aborts.
          */
         walk_page_range(0, ~0UL, &mem_cgroup_move_charge_walk);
-        up_read(&mm->mmap_sem);
+        up_read(&mc.mm->mmap_sem);
         atomic_dec(&mc.from->moving_account);
 }
 
-static void mem_cgroup_move_task(struct cgroup_taskset *tset)
+static void mem_cgroup_move_task(void)
 {
-        struct cgroup_subsys_state *css;
-        struct task_struct *p = cgroup_taskset_first(tset, &css);
-        struct mm_struct *mm = get_task_mm(p);
-
-        if (mm) {
-                if (mc.to)
-                        mem_cgroup_move_charge(mm);
-                mmput(mm);
-        }
-        if (mc.to)
+        if (mc.to) {
+                mem_cgroup_move_charge();
                 mem_cgroup_clear_mc();
+        }
 }
 #else   /* !CONFIG_MMU */
 static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
@@ -4911,7 +4912,7 @@ static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
 static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset)
 {
 }
-static void mem_cgroup_move_task(struct cgroup_taskset *tset)
+static void mem_cgroup_move_task(void)
 {
 }
 #endif
@@ -5195,7 +5196,7 @@ struct cgroup_subsys memory_cgrp_subsys = {
         .css_reset = mem_cgroup_css_reset,
         .can_attach = mem_cgroup_can_attach,
         .cancel_attach = mem_cgroup_cancel_attach,
-        .attach = mem_cgroup_move_task,
+        .post_attach = mem_cgroup_move_task,
         .bind = mem_cgroup_bind,
         .dfl_cftypes = memory_files,
         .legacy_cftypes = mem_cgroup_legacy_files,
diff --git a/mm/memory-failure.c b/mm/memory-failure.c
index 78f5f2641b91..ca5acee53b7a 100644
--- a/mm/memory-failure.c
+++ b/mm/memory-failure.c
@@ -888,7 +888,15 @@ int get_hwpoison_page(struct page *page)
                 }
         }
 
-        return get_page_unless_zero(head);
+        if (get_page_unless_zero(head)) {
+                if (head == compound_head(page))
+                        return 1;
+
+                pr_info("MCE: %#lx cannot catch tail\n", page_to_pfn(page));
+                put_page(head);
+        }
+
+        return 0;
 }
 EXPORT_SYMBOL_GPL(get_hwpoison_page);
 
diff --git a/mm/memory.c b/mm/memory.c
index 93897f23cc11..305537fc8640 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -789,6 +789,46 @@ out:
         return pfn_to_page(pfn);
 }
 
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr,
+                                pmd_t pmd)
+{
+        unsigned long pfn = pmd_pfn(pmd);
+
+        /*
+         * There is no pmd_special() but there may be special pmds, e.g.
+         * in a direct-access (dax) mapping, so let's just replicate the
+         * !HAVE_PTE_SPECIAL case from vm_normal_page() here.
+         */
+        if (unlikely(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))) {
+                if (vma->vm_flags & VM_MIXEDMAP) {
+                        if (!pfn_valid(pfn))
+                                return NULL;
+                        goto out;
+                } else {
+                        unsigned long off;
+                        off = (addr - vma->vm_start) >> PAGE_SHIFT;
+                        if (pfn == vma->vm_pgoff + off)
+                                return NULL;
+                        if (!is_cow_mapping(vma->vm_flags))
+                                return NULL;
+                }
+        }
+
+        if (is_zero_pfn(pfn))
+                return NULL;
+        if (unlikely(pfn > highest_memmap_pfn))
+                return NULL;
+
+        /*
+         * NOTE! We still have PageReserved() pages in the page tables.
+         * eg. VDSO mappings can cause them to exist.
+         */
+out:
+        return pfn_to_page(pfn);
+}
+#endif
+
 /*
  * copy one vm_area from one task to the other. Assumes the page tables
  * already present in the new task to be cleared in the whole range
diff --git a/mm/migrate.c b/mm/migrate.c
index 6c822a7b27e0..f9dfb18a4eba 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -975,7 +975,13 @@ out:
                 dec_zone_page_state(page, NR_ISOLATED_ANON +
                                 page_is_file_cache(page));
                 /* Soft-offlined page shouldn't go through lru cache list */
-                if (reason == MR_MEMORY_FAILURE) {
+                if (reason == MR_MEMORY_FAILURE && rc == MIGRATEPAGE_SUCCESS) {
+                        /*
+                         * With this release, we free successfully migrated
+                         * page and set PG_HWPoison on just freed page
+                         * intentionally. Although it's rather weird, it's how
+                         * HWPoison flag works at the moment.
+                         */
                         put_page(page);
                         if (!test_set_page_hwpoison(page))
                                 num_poisoned_pages_inc();
diff --git a/mm/page_io.c b/mm/page_io.c
index cd92e3d67a32..985f23cfa79b 100644
--- a/mm/page_io.c
+++ b/mm/page_io.c
@@ -353,7 +353,11 @@ int swap_readpage(struct page *page)
 
         ret = bdev_read_page(sis->bdev, swap_page_sector(page), page);
         if (!ret) {
-                swap_slot_free_notify(page);
+                if (trylock_page(page)) {
+                        swap_slot_free_notify(page);
+                        unlock_page(page);
+                }
+
                 count_vm_event(PSWPIN);
                 return 0;
         }
diff --git a/mm/swap.c b/mm/swap.c
index a0bc206b4ac6..03aacbcb013f 100644
--- a/mm/swap.c
+++ b/mm/swap.c
@@ -728,6 +728,11 @@ void release_pages(struct page **pages, int nr, bool cold)
                         zone = NULL;
                 }
 
+                if (is_huge_zero_page(page)) {
+                        put_huge_zero_page();
+                        continue;
+                }
+
                 page = compound_head(page);
                 if (!put_page_testzero(page))
                         continue;
diff --git a/mm/vmscan.c b/mm/vmscan.c
index b934223eaa45..142cb61f4822 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -2553,7 +2553,7 @@ static bool shrink_zones(struct zonelist *zonelist, struct scan_control *sc)
                 sc->gfp_mask |= __GFP_HIGHMEM;
 
         for_each_zone_zonelist_nodemask(zone, z, zonelist,
-                                        requested_highidx, sc->nodemask) {
+                                        gfp_zone(sc->gfp_mask), sc->nodemask) {
                 enum zone_type classzone_idx;
 
                 if (!populated_zone(zone))
@@ -3318,6 +3318,20 @@ static void kswapd_try_to_sleep(pg_data_t *pgdat, int order,
         /* Try to sleep for a short interval */
         if (prepare_kswapd_sleep(pgdat, order, remaining,
                                                 balanced_classzone_idx)) {
+                /*
+                 * Compaction records what page blocks it recently failed to
+                 * isolate pages from and skips them in the future scanning.
+                 * When kswapd is going to sleep, it is reasonable to assume
+                 * that pages and compaction may succeed so reset the cache.
+                 */
+                reset_isolation_suitable(pgdat);
+
+                /*
+                 * We have freed the memory, now we should compact it to make
+                 * allocation of the requested order possible.
+                 */
+                wakeup_kcompactd(pgdat, order, classzone_idx);
+
                 remaining = schedule_timeout(HZ/10);
                 finish_wait(&pgdat->kswapd_wait, &wait);
                 prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE);
@@ -3341,20 +3355,6 @@ static void kswapd_try_to_sleep(pg_data_t *pgdat, int order,
                  */
                 set_pgdat_percpu_threshold(pgdat, calculate_normal_threshold);
 
-                /*
-                 * Compaction records what page blocks it recently failed to
-                 * isolate pages from and skips them in the future scanning.
-                 * When kswapd is going to sleep, it is reasonable to assume
-                 * that pages and compaction may succeed so reset the cache.
-                 */
-                reset_isolation_suitable(pgdat);
-
-                /*
-                 * We have freed the memory, now we should compact it to make
-                 * allocation of the requested order possible.
-                 */
-                wakeup_kcompactd(pgdat, order, classzone_idx);
-
                 if (!kthread_should_stop())
                         schedule();