thp: remove assumptions on pgtable_t type

The thp page table pre-allocation code currently assumes that pgtable_t is
of type "struct page *".  This may not be true for all architectures, so
this patch removes that assumption by replacing the functions
prepare_pmd_huge_pte() and get_pmd_huge_pte() with two new functions that
can be defined architecture-specific.

It also removes two VM_BUG_ON checks for page_count() and page_mapcount()
operating on a pgtable_t.  Apart from the VM_BUG_ON removal, there will be
no functional change introduced by this patch.

Signed-off-by: Gerald Schaefer <gerald.schaefer@de.ibm.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Hillf Danton <dhillf@gmail.com>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: Heiko Carstens <heiko.carstens@de.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

2 files changed, 47 insertions, 42 deletions

diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index e19cc426c522..9ea6d1953765 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -598,19 +598,6 @@ out:
 }
 __setup("transparent_hugepage=", setup_transparent_hugepage);
 
-static void prepare_pmd_huge_pte(pgtable_t pgtable,
-                                 struct mm_struct *mm)
-{
-        assert_spin_locked(&mm->page_table_lock);
-
-        /* FIFO */
-        if (!mm->pmd_huge_pte)
-                INIT_LIST_HEAD(&pgtable->lru);
-        else
-                list_add(&pgtable->lru, &mm->pmd_huge_pte->lru);
-        mm->pmd_huge_pte = pgtable;
-}
-
 static inline pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma)
 {
         if (likely(vma->vm_flags & VM_WRITE))
@@ -652,7 +639,7 @@ static int __do_huge_pmd_anonymous_page(struct mm_struct *mm,
                  */
                 page_add_new_anon_rmap(page, vma, haddr);
                 set_pmd_at(mm, haddr, pmd, entry);
-                prepare_pmd_huge_pte(pgtable, mm);
+                pgtable_trans_huge_deposit(mm, pgtable);
                 add_mm_counter(mm, MM_ANONPAGES, HPAGE_PMD_NR);
                 mm->nr_ptes++;
                 spin_unlock(&mm->page_table_lock);
@@ -778,7 +765,7 @@ int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
         pmdp_set_wrprotect(src_mm, addr, src_pmd);
         pmd = pmd_mkold(pmd_wrprotect(pmd));
         set_pmd_at(dst_mm, addr, dst_pmd, pmd);
-        prepare_pmd_huge_pte(pgtable, dst_mm);
+        pgtable_trans_huge_deposit(dst_mm, pgtable);
         dst_mm->nr_ptes++;
 
         ret = 0;
@@ -789,25 +776,6 @@ out:
         return ret;
 }
 
-/* no "address" argument so destroys page coloring of some arch */
-pgtable_t get_pmd_huge_pte(struct mm_struct *mm)
-{
-        pgtable_t pgtable;
-
-        assert_spin_locked(&mm->page_table_lock);
-
-        /* FIFO */
-        pgtable = mm->pmd_huge_pte;
-        if (list_empty(&pgtable->lru))
-                mm->pmd_huge_pte = NULL;
-        else {
-                mm->pmd_huge_pte = list_entry(pgtable->lru.next,
-                                              struct page, lru);
-                list_del(&pgtable->lru);
-        }
-        return pgtable;
-}
-
 static int do_huge_pmd_wp_page_fallback(struct mm_struct *mm,
                                         struct vm_area_struct *vma,
                                         unsigned long address,
@@ -863,7 +831,7 @@ static int do_huge_pmd_wp_page_fallback(struct mm_struct *mm,
         pmdp_clear_flush_notify(vma, haddr, pmd);
         /* leave pmd empty until pte is filled */
 
-        pgtable = get_pmd_huge_pte(mm);
+        pgtable = pgtable_trans_huge_withdraw(mm);
         pmd_populate(mm, &_pmd, pgtable);
 
         for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) {
@@ -1028,7 +996,7 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
         if (__pmd_trans_huge_lock(pmd, vma) == 1) {
                 struct page *page;
                 pgtable_t pgtable;
-                pgtable = get_pmd_huge_pte(tlb->mm);
+                pgtable = pgtable_trans_huge_withdraw(tlb->mm);
                 page = pmd_page(*pmd);
                 pmd_clear(pmd);
                 tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
@@ -1345,11 +1313,11 @@ static int __split_huge_page_map(struct page *page,
         pmd = page_check_address_pmd(page, mm, address,
                                      PAGE_CHECK_ADDRESS_PMD_SPLITTING_FLAG);
         if (pmd) {
-                pgtable = get_pmd_huge_pte(mm);
+                pgtable = pgtable_trans_huge_withdraw(mm);
                 pmd_populate(mm, &_pmd, pgtable);
 
-                for (i = 0, haddr = address; i < HPAGE_PMD_NR;
-                     i++, haddr += PAGE_SIZE) {
+                haddr = address;
+                for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) {
                         pte_t *pte, entry;
                         BUG_ON(PageCompound(page+i));
                         entry = mk_pte(page + i, vma->vm_page_prot);
@@ -2017,8 +1985,6 @@ static void collapse_huge_page(struct mm_struct *mm,
         pte_unmap(pte);
         __SetPageUptodate(new_page);
         pgtable = pmd_pgtable(_pmd);
-        VM_BUG_ON(page_count(pgtable) != 1);
-        VM_BUG_ON(page_mapcount(pgtable) != 0);
 
         _pmd = mk_pmd(new_page, vma->vm_page_prot);
         _pmd = maybe_pmd_mkwrite(pmd_mkdirty(_pmd), vma);
@@ -2036,7 +2002,7 @@ static void collapse_huge_page(struct mm_struct *mm,
         page_add_new_anon_rmap(new_page, vma, address);
         set_pmd_at(mm, address, pmd, _pmd);
         update_mmu_cache(vma, address, _pmd);
-        prepare_pmd_huge_pte(pgtable, mm);
+        pgtable_trans_huge_deposit(mm, pgtable);
         spin_unlock(&mm->page_table_lock);
 
         *hpage = NULL;
diff --git a/mm/pgtable-generic.c b/mm/pgtable-generic.c
index 74c0ddaa6fa0..29867e083d37 100644
--- a/mm/pgtable-generic.c
+++ b/mm/pgtable-generic.c
@@ -120,3 +120,42 @@ void pmdp_splitting_flush(struct vm_area_struct *vma, unsigned long address,
 }
 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
 #endif
+
+#ifndef __HAVE_ARCH_PGTABLE_DEPOSIT
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+void pgtable_trans_huge_deposit(struct mm_struct *mm, pgtable_t pgtable)
+{
+        assert_spin_locked(&mm->page_table_lock);
+
+        /* FIFO */
+        if (!mm->pmd_huge_pte)
+                INIT_LIST_HEAD(&pgtable->lru);
+        else
+                list_add(&pgtable->lru, &mm->pmd_huge_pte->lru);
+        mm->pmd_huge_pte = pgtable;
+}
+#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
+#endif
+
+#ifndef __HAVE_ARCH_PGTABLE_WITHDRAW
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+/* no "address" argument so destroys page coloring of some arch */
+pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm)
+{
+        pgtable_t pgtable;
+
+        assert_spin_locked(&mm->page_table_lock);
+
+        /* FIFO */
+        pgtable = mm->pmd_huge_pte;
+        if (list_empty(&pgtable->lru))
+                mm->pmd_huge_pte = NULL;
+        else {
+                mm->pmd_huge_pte = list_entry(pgtable->lru.next,
+                                              struct page, lru);
+                list_del(&pgtable->lru);
+        }
+        return pgtable;
+}
+#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
+#endif