mm, thp: move ptl taking inside page_check_address_pmd()

With split page table lock we can't know which lock we need to take
before we find the relevant pmd.

Let's move lock taking inside the function.

Signed-off-by: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Tested-by: Alex Thorlton <athorlton@sgi.com>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: "Eric W . Biederman" <ebiederm@xmission.com>
Cc: "Paul E . McKenney" <paulmck@linux.vnet.ibm.com>
Cc: Al Viro <viro@zeniv.linux.org.uk>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Dave Jones <davej@redhat.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Kees Cook <keescook@chromium.org>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Michael Kerrisk <mtk.manpages@gmail.com>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Rik van Riel <riel@redhat.com>
Cc: Robin Holt <robinmholt@gmail.com>
Cc: Sedat Dilek <sedat.dilek@gmail.com>
Cc: Srikar Dronamraju <srikar@linux.vnet.ibm.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Hugh Dickins <hughd@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

3 files changed, 34 insertions, 25 deletions

diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h
index 4aca0d8da112..91672e2deec3 100644
--- a/include/linux/huge_mm.h
+++ b/include/linux/huge_mm.h
@@ -54,7 +54,8 @@ enum page_check_address_pmd_flag {
 extern pmd_t *page_check_address_pmd(struct page *page,
                                      struct mm_struct *mm,
                                      unsigned long address,
-                                     enum page_check_address_pmd_flag flag);
+                                     enum page_check_address_pmd_flag flag,
+                                     spinlock_t **ptl);
 
 #define HPAGE_PMD_ORDER (HPAGE_PMD_SHIFT-PAGE_SHIFT)
 #define HPAGE_PMD_NR (1<<HPAGE_PMD_ORDER)
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 471eb04066ff..c2082ab4fc93 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -1552,23 +1552,33 @@ int __pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma,
         return 0;
 }
 
+/*
+ * This function returns whether a given @page is mapped onto the @address
+ * in the virtual space of @mm.
+ *
+ * When it's true, this function returns *pmd with holding the page table lock
+ * and passing it back to the caller via @ptl.
+ * If it's false, returns NULL without holding the page table lock.
+ */
 pmd_t *page_check_address_pmd(struct page *page,
                               struct mm_struct *mm,
                               unsigned long address,
-                              enum page_check_address_pmd_flag flag)
+                              enum page_check_address_pmd_flag flag,
+                              spinlock_t **ptl)
 {
-        pmd_t *pmd, *ret = NULL;
+        pmd_t *pmd;
 
         if (address & ~HPAGE_PMD_MASK)
-                goto out;
+                return NULL;
 
         pmd = mm_find_pmd(mm, address);
         if (!pmd)
-                goto out;
+                return NULL;
+        *ptl = pmd_lock(mm, pmd);
         if (pmd_none(*pmd))
-                goto out;
+                goto unlock;
         if (pmd_page(*pmd) != page)
-                goto out;
+                goto unlock;
         /*
          * split_vma() may create temporary aliased mappings. There is
          * no risk as long as all huge pmd are found and have their
@@ -1578,14 +1588,15 @@ pmd_t *page_check_address_pmd(struct page *page,
          */
         if (flag == PAGE_CHECK_ADDRESS_PMD_NOTSPLITTING_FLAG &&
             pmd_trans_splitting(*pmd))
-                goto out;
+                goto unlock;
         if (pmd_trans_huge(*pmd)) {
                 VM_BUG_ON(flag == PAGE_CHECK_ADDRESS_PMD_SPLITTING_FLAG &&
                           !pmd_trans_splitting(*pmd));
-                ret = pmd;
+                return pmd;
         }
-out:
-        return ret;
+unlock:
+        spin_unlock(*ptl);
+        return NULL;
 }
 
 static int __split_huge_page_splitting(struct page *page,
@@ -1593,6 +1604,7 @@ static int __split_huge_page_splitting(struct page *page,
                                        unsigned long address)
 {
         struct mm_struct *mm = vma->vm_mm;
+        spinlock_t *ptl;
         pmd_t *pmd;
         int ret = 0;
         /* For mmu_notifiers */
@@ -1600,9 +1612,8 @@ static int __split_huge_page_splitting(struct page *page,
         const unsigned long mmun_end   = address + HPAGE_PMD_SIZE;
 
         mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
-        spin_lock(&mm->page_table_lock);
         pmd = page_check_address_pmd(page, mm, address,
-                                     PAGE_CHECK_ADDRESS_PMD_NOTSPLITTING_FLAG);
+                        PAGE_CHECK_ADDRESS_PMD_NOTSPLITTING_FLAG, &ptl);
         if (pmd) {
                 /*
                  * We can't temporarily set the pmd to null in order
@@ -1613,8 +1624,8 @@ static int __split_huge_page_splitting(struct page *page,
                  */
                 pmdp_splitting_flush(vma, address, pmd);
                 ret = 1;
+                spin_unlock(ptl);
         }
-        spin_unlock(&mm->page_table_lock);
         mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
 
         return ret;
@@ -1745,14 +1756,14 @@ static int __split_huge_page_map(struct page *page,
                                  unsigned long address)
 {
         struct mm_struct *mm = vma->vm_mm;
+        spinlock_t *ptl;
         pmd_t *pmd, _pmd;
         int ret = 0, i;
         pgtable_t pgtable;
         unsigned long haddr;
 
-        spin_lock(&mm->page_table_lock);
         pmd = page_check_address_pmd(page, mm, address,
-                                     PAGE_CHECK_ADDRESS_PMD_SPLITTING_FLAG);
+                        PAGE_CHECK_ADDRESS_PMD_SPLITTING_FLAG, &ptl);
         if (pmd) {
                 pgtable = pgtable_trans_huge_withdraw(mm, pmd);
                 pmd_populate(mm, &_pmd, pgtable);
@@ -1807,8 +1818,8 @@ static int __split_huge_page_map(struct page *page,
                 pmdp_invalidate(vma, address, pmd);
                 pmd_populate(mm, pmd, pgtable);
                 ret = 1;
+                spin_unlock(ptl);
         }
-        spin_unlock(&mm->page_table_lock);
 
         return ret;
 }
diff --git a/mm/rmap.c b/mm/rmap.c
index fd3ee7a54a13..b59d741dcf65 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -665,25 +665,23 @@ int page_referenced_one(struct page *page, struct vm_area_struct *vma,
                         unsigned long *vm_flags)
 {
         struct mm_struct *mm = vma->vm_mm;
+        spinlock_t *ptl;
         int referenced = 0;
 
         if (unlikely(PageTransHuge(page))) {
                 pmd_t *pmd;
 
-                spin_lock(&mm->page_table_lock);
                 /*
                  * rmap might return false positives; we must filter
                  * these out using page_check_address_pmd().
                  */
                 pmd = page_check_address_pmd(page, mm, address,
-                                             PAGE_CHECK_ADDRESS_PMD_FLAG);
-                if (!pmd) {
-                        spin_unlock(&mm->page_table_lock);
+                                             PAGE_CHECK_ADDRESS_PMD_FLAG, &ptl);
+                if (!pmd)
                         goto out;
-                }
 
                 if (vma->vm_flags & VM_LOCKED) {
-                        spin_unlock(&mm->page_table_lock);
+                        spin_unlock(ptl);
                         *mapcount = 0;  /* break early from loop */
                         *vm_flags |= VM_LOCKED;
                         goto out;
@@ -692,10 +690,9 @@ int page_referenced_one(struct page *page, struct vm_area_struct *vma,
                 /* go ahead even if the pmd is pmd_trans_splitting() */
                 if (pmdp_clear_flush_young_notify(vma, address, pmd))
                         referenced++;
-                spin_unlock(&mm->page_table_lock);
+                spin_unlock(ptl);
         } else {
                 pte_t *pte;
-                spinlock_t *ptl;
 
                 /*
                  * rmap might return false positives; we must filter