mm: THP page cache support for ppc64

Add arch specific callback in the generic THP page cache code that will
deposit and withdarw preallocated page table.  Archs like ppc64 use this
preallocated table to store the hash pte slot information.

Testing:
kernel build of the patch series on tmpfs mounted with option huge=always

The related thp stat:
thp_fault_alloc 72939
thp_fault_fallback 60547
thp_collapse_alloc 603
thp_collapse_alloc_failed 0
thp_file_alloc 253763
thp_file_mapped 4251
thp_split_page 51518
thp_split_page_failed 1
thp_deferred_split_page 73566
thp_split_pmd 665
thp_zero_page_alloc 3
thp_zero_page_alloc_failed 0

[akpm@linux-foundation.org: remove unneeded parentheses, per Kirill]
Link: http://lkml.kernel.org/r/20161113150025.17942-2-aneesh.kumar@linux.vnet.ibm.com
Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Acked-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Michael Neuling <mikey@neuling.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Balbir Singh <bsingharora@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

1 files changed, 50 insertions, 10 deletions

diff --git a/mm/memory.c b/mm/memory.c
index 0a72f821ccdc..32e9b7aec366 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -2935,6 +2935,19 @@ static inline bool transhuge_vma_suitable(struct vm_area_struct *vma,
         return true;
 }
 
+static void deposit_prealloc_pte(struct fault_env *fe)
+{
+        struct vm_area_struct *vma = fe->vma;
+
+        pgtable_trans_huge_deposit(vma->vm_mm, fe->pmd, fe->prealloc_pte);
+        /*
+         * We are going to consume the prealloc table,
+         * count that as nr_ptes.
+         */
+        atomic_long_inc(&vma->vm_mm->nr_ptes);
+        fe->prealloc_pte = 0;
+}
+
 static int do_set_pmd(struct fault_env *fe, struct page *page)
 {
         struct vm_area_struct *vma = fe->vma;
@@ -2949,6 +2962,17 @@ static int do_set_pmd(struct fault_env *fe, struct page *page)
         ret = VM_FAULT_FALLBACK;
         page = compound_head(page);
 
+        /*
+         * Archs like ppc64 need additonal space to store information
+         * related to pte entry. Use the preallocated table for that.
+         */
+        if (arch_needs_pgtable_deposit() && !fe->prealloc_pte) {
+                fe->prealloc_pte = pte_alloc_one(vma->vm_mm, fe->address);
+                if (!fe->prealloc_pte)
+                        return VM_FAULT_OOM;
+                smp_wmb(); /* See comment in __pte_alloc() */
+        }
+
         fe->ptl = pmd_lock(vma->vm_mm, fe->pmd);
         if (unlikely(!pmd_none(*fe->pmd)))
                 goto out;
@@ -2962,6 +2986,11 @@ static int do_set_pmd(struct fault_env *fe, struct page *page)
 
         add_mm_counter(vma->vm_mm, MM_FILEPAGES, HPAGE_PMD_NR);
         page_add_file_rmap(page, true);
+        /*
+         * deposit and withdraw with pmd lock held
+         */
+        if (arch_needs_pgtable_deposit())
+                deposit_prealloc_pte(fe);
 
         set_pmd_at(vma->vm_mm, haddr, fe->pmd, entry);
 
@@ -2971,6 +3000,13 @@ static int do_set_pmd(struct fault_env *fe, struct page *page)
         ret = 0;
         count_vm_event(THP_FILE_MAPPED);
 out:
+        /*
+         * If we are going to fallback to pte mapping, do a
+         * withdraw with pmd lock held.
+         */
+        if (arch_needs_pgtable_deposit() && ret == VM_FAULT_FALLBACK)
+                fe->prealloc_pte = pgtable_trans_huge_withdraw(vma->vm_mm,
+                                                               fe->pmd);
         spin_unlock(fe->ptl);
         return ret;
 }
@@ -3010,18 +3046,20 @@ int alloc_set_pte(struct fault_env *fe, struct mem_cgroup *memcg,
 
                 ret = do_set_pmd(fe, page);
                 if (ret != VM_FAULT_FALLBACK)
-                        return ret;
+                        goto fault_handled;
         }
 
         if (!fe->pte) {
                 ret = pte_alloc_one_map(fe);
                 if (ret)
-                        return ret;
+                        goto fault_handled;
         }
 
         /* Re-check under ptl */
-        if (unlikely(!pte_none(*fe->pte)))
-                return VM_FAULT_NOPAGE;
+        if (unlikely(!pte_none(*fe->pte))) {
+                ret = VM_FAULT_NOPAGE;
+                goto fault_handled;
+        }
 
         flush_icache_page(vma, page);
         entry = mk_pte(page, vma->vm_page_prot);
@@ -3041,8 +3079,15 @@ int alloc_set_pte(struct fault_env *fe, struct mem_cgroup *memcg,
 
         /* no need to invalidate: a not-present page won't be cached */
         update_mmu_cache(vma, fe->address, fe->pte);
+        ret = 0;
 
-        return 0;
+fault_handled:
+        /* preallocated pagetable is unused: free it */
+        if (fe->prealloc_pte) {
+                pte_free(fe->vma->vm_mm, fe->prealloc_pte);
+                fe->prealloc_pte = 0;
+        }
+        return ret;
 }
 
 static unsigned long fault_around_bytes __read_mostly =
@@ -3141,11 +3186,6 @@ static int do_fault_around(struct fault_env *fe, pgoff_t start_pgoff)
 
         fe->vma->vm_ops->map_pages(fe, start_pgoff, end_pgoff);
 
-        /* preallocated pagetable is unused: free it */
-        if (fe->prealloc_pte) {
-                pte_free(fe->vma->vm_mm, fe->prealloc_pte);
-                fe->prealloc_pte = 0;
-        }
         /* Huge page is mapped? Page fault is solved */
         if (pmd_trans_huge(*fe->pmd)) {
                 ret = VM_FAULT_NOPAGE;