1 files changed, 27 insertions, 1 deletions

diff --git a/arch/x86/mm/gup.c b/arch/x86/mm/gup.c
index 738e6593799d..dbe34b931374 100644
--- a/arch/x86/mm/gup.c
+++ b/arch/x86/mm/gup.c
@@ -8,6 +8,7 @@
 #include <linux/mm.h>
 #include <linux/vmstat.h>
 #include <linux/highmem.h>
+#include <linux/swap.h>
 
 #include <asm/pgtable.h>
 
@@ -89,6 +90,7 @@ static noinline int gup_pte_range(pmd_t pmd, unsigned long addr,
                 VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
                 page = pte_page(pte);
                 get_page(page);
+                SetPageReferenced(page);
                 pages[*nr] = page;
                 (*nr)++;
 
@@ -103,6 +105,17 @@ static inline void get_head_page_multiple(struct page *page, int nr)
         VM_BUG_ON(page != compound_head(page));
         VM_BUG_ON(page_count(page) == 0);
         atomic_add(nr, &page->_count);
+        SetPageReferenced(page);
+}
+
+static inline void get_huge_page_tail(struct page *page)
+{
+        /*
+         * __split_huge_page_refcount() cannot run
+         * from under us.
+         */
+        VM_BUG_ON(atomic_read(&page->_count) < 0);
+        atomic_inc(&page->_count);
 }
 
 static noinline int gup_huge_pmd(pmd_t pmd, unsigned long addr,
@@ -128,6 +141,8 @@ static noinline int gup_huge_pmd(pmd_t pmd, unsigned long addr,
         do {
                 VM_BUG_ON(compound_head(page) != head);
                 pages[*nr] = page;
+                if (PageTail(page))
+                        get_huge_page_tail(page);
                 (*nr)++;
                 page++;
                 refs++;
@@ -148,7 +163,18 @@ static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end,
                 pmd_t pmd = *pmdp;
 
                 next = pmd_addr_end(addr, end);
-                if (pmd_none(pmd))
+                /*
+                 * The pmd_trans_splitting() check below explains why
+                 * pmdp_splitting_flush has to flush the tlb, to stop
+                 * this gup-fast code from running while we set the
+                 * splitting bit in the pmd. Returning zero will take
+                 * the slow path that will call wait_split_huge_page()
+                 * if the pmd is still in splitting state. gup-fast
+                 * can't because it has irq disabled and
+                 * wait_split_huge_page() would never return as the
+                 * tlb flush IPI wouldn't run.
+                 */
+                if (pmd_none(pmd) || pmd_trans_splitting(pmd))
                         return 0;
                 if (unlikely(pmd_large(pmd))) {
                         if (!gup_huge_pmd(pmd, addr, next, write, pages, nr))