9 files changed, 102 insertions, 33 deletions

diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h
index beabb099c040..62e6ea0a7650 100644
--- a/arch/powerpc/include/asm/book3s/64/pgtable.h
+++ b/arch/powerpc/include/asm/book3s/64/pgtable.h
@@ -1350,5 +1350,26 @@ static inline bool is_pte_rw_upgrade(unsigned long old_val, unsigned long new_va
         return false;
 }
 
+/*
+ * Like pmd_huge() and pmd_large(), but works regardless of config options
+ */
+#define pmd_is_leaf pmd_is_leaf
+static inline bool pmd_is_leaf(pmd_t pmd)
+{
+        return !!(pmd_raw(pmd) & cpu_to_be64(_PAGE_PTE));
+}
+
+#define pud_is_leaf pud_is_leaf
+static inline bool pud_is_leaf(pud_t pud)
+{
+        return !!(pud_raw(pud) & cpu_to_be64(_PAGE_PTE));
+}
+
+#define pgd_is_leaf pgd_is_leaf
+static inline bool pgd_is_leaf(pgd_t pgd)
+{
+        return !!(pgd_raw(pgd) & cpu_to_be64(_PAGE_PTE));
+}
+
 #endif /* __ASSEMBLY__ */
 #endif /* _ASM_POWERPC_BOOK3S_64_PGTABLE_H_ */
diff --git a/arch/powerpc/include/asm/pgtable.h b/arch/powerpc/include/asm/pgtable.h
index 3f53be60fb01..bf7d771f342e 100644
--- a/arch/powerpc/include/asm/pgtable.h
+++ b/arch/powerpc/include/asm/pgtable.h
@@ -140,6 +140,30 @@ static inline void pte_frag_set(mm_context_t *ctx, void *p)
 }
 #endif
 
+#ifndef pmd_is_leaf
+#define pmd_is_leaf pmd_is_leaf
+static inline bool pmd_is_leaf(pmd_t pmd)
+{
+        return false;
+}
+#endif
+
+#ifndef pud_is_leaf
+#define pud_is_leaf pud_is_leaf
+static inline bool pud_is_leaf(pud_t pud)
+{
+        return false;
+}
+#endif
+
+#ifndef pgd_is_leaf
+#define pgd_is_leaf pgd_is_leaf
+static inline bool pgd_is_leaf(pgd_t pgd)
+{
+        return false;
+}
+#endif
+
 #endif /* __ASSEMBLY__ */
 
 #endif /* _ASM_POWERPC_PGTABLE_H */
diff --git a/arch/powerpc/include/asm/pte-walk.h b/arch/powerpc/include/asm/pte-walk.h
index 2d633e9d686c..33fa5dd8ee6a 100644
--- a/arch/powerpc/include/asm/pte-walk.h
+++ b/arch/powerpc/include/asm/pte-walk.h
@@ -10,8 +10,20 @@ extern pte_t *__find_linux_pte(pgd_t *pgdir, unsigned long ea,
 static inline pte_t *find_linux_pte(pgd_t *pgdir, unsigned long ea,
                                     bool *is_thp, unsigned *hshift)
 {
+        pte_t *pte;
+
         VM_WARN(!arch_irqs_disabled(), "%s called with irq enabled\n", __func__);
-        return __find_linux_pte(pgdir, ea, is_thp, hshift);
+        pte = __find_linux_pte(pgdir, ea, is_thp, hshift);
+
+#if defined(CONFIG_DEBUG_VM) &&                                         \
+        !(defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE))
+        /*
+         * We should not find huge page if these configs are not enabled.
+         */
+        if (hshift)
+                WARN_ON(*hshift);
+#endif
+        return pte;
 }
 
 static inline pte_t *find_init_mm_pte(unsigned long ea, unsigned *hshift)
@@ -26,10 +38,22 @@ static inline pte_t *find_init_mm_pte(unsigned long ea, unsigned *hshift)
 static inline pte_t *find_current_mm_pte(pgd_t *pgdir, unsigned long ea,
                                          bool *is_thp, unsigned *hshift)
 {
+        pte_t *pte;
+
         VM_WARN(!arch_irqs_disabled(), "%s called with irq enabled\n", __func__);
         VM_WARN(pgdir != current->mm->pgd,
                 "%s lock less page table lookup called on wrong mm\n", __func__);
-        return __find_linux_pte(pgdir, ea, is_thp, hshift);
+        pte = __find_linux_pte(pgdir, ea, is_thp, hshift);
+
+#if defined(CONFIG_DEBUG_VM) &&                                         \
+        !(defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE))
+        /*
+         * We should not find huge page if these configs are not enabled.
+         */
+        if (hshift)
+                WARN_ON(*hshift);
+#endif
+        return pte;
 }
 
 #endif /* _ASM_POWERPC_PTE_WALK_H */
diff --git a/arch/powerpc/kvm/book3s_64_mmu_radix.c b/arch/powerpc/kvm/book3s_64_mmu_radix.c
index f55ef071883f..91efee7f0329 100644
--- a/arch/powerpc/kvm/book3s_64_mmu_radix.c
+++ b/arch/powerpc/kvm/book3s_64_mmu_radix.c
@@ -363,12 +363,6 @@ static void kvmppc_pte_free(pte_t *ptep)
         kmem_cache_free(kvm_pte_cache, ptep);
 }
 
-/* Like pmd_huge() and pmd_large(), but works regardless of config options */
-static inline int pmd_is_leaf(pmd_t pmd)
-{
-        return !!(pmd_val(pmd) & _PAGE_PTE);
-}
-
 static pmd_t *kvmppc_pmd_alloc(void)
 {
         return kmem_cache_alloc(kvm_pmd_cache, GFP_KERNEL);
@@ -489,7 +483,7 @@ static void kvmppc_unmap_free_pud(struct kvm *kvm, pud_t *pud,
         for (iu = 0; iu < PTRS_PER_PUD; ++iu, ++p) {
                 if (!pud_present(*p))
                         continue;
-                if (pud_huge(*p)) {
+                if (pud_is_leaf(*p)) {
                         pud_clear(p);
                 } else {
                         pmd_t *pmd;
@@ -588,7 +582,7 @@ int kvmppc_create_pte(struct kvm *kvm, pgd_t *pgtable, pte_t pte,
                 new_pud = pud_alloc_one(kvm->mm, gpa);
 
         pmd = NULL;
-        if (pud && pud_present(*pud) && !pud_huge(*pud))
+        if (pud && pud_present(*pud) && !pud_is_leaf(*pud))
                 pmd = pmd_offset(pud, gpa);
         else if (level <= 1)
                 new_pmd = kvmppc_pmd_alloc();
@@ -611,7 +605,7 @@ int kvmppc_create_pte(struct kvm *kvm, pgd_t *pgtable, pte_t pte,
                 new_pud = NULL;
         }
         pud = pud_offset(pgd, gpa);
-        if (pud_huge(*pud)) {
+        if (pud_is_leaf(*pud)) {
                 unsigned long hgpa = gpa & PUD_MASK;
 
                 /* Check if we raced and someone else has set the same thing */
diff --git a/arch/powerpc/mm/book3s64/radix_pgtable.c b/arch/powerpc/mm/book3s64/radix_pgtable.c
index b573fa024b32..e92c6472a20c 100644
--- a/arch/powerpc/mm/book3s64/radix_pgtable.c
+++ b/arch/powerpc/mm/book3s64/radix_pgtable.c
@@ -203,14 +203,14 @@ void radix__change_memory_range(unsigned long start, unsigned long end,
                 pudp = pud_alloc(&init_mm, pgdp, idx);
                 if (!pudp)
                         continue;
-                if (pud_huge(*pudp)) {
+                if (pud_is_leaf(*pudp)) {
                         ptep = (pte_t *)pudp;
                         goto update_the_pte;
                 }
                 pmdp = pmd_alloc(&init_mm, pudp, idx);
                 if (!pmdp)
                         continue;
-                if (pmd_huge(*pmdp)) {
+                if (pmd_is_leaf(*pmdp)) {
                         ptep = pmdp_ptep(pmdp);
                         goto update_the_pte;
                 }
@@ -835,7 +835,7 @@ static void remove_pmd_table(pmd_t *pmd_start, unsigned long addr,
                 if (!pmd_present(*pmd))
                         continue;
 
-                if (pmd_huge(*pmd)) {
+                if (pmd_is_leaf(*pmd)) {
                         split_kernel_mapping(addr, end, PMD_SIZE, (pte_t *)pmd);
                         continue;
                 }
@@ -860,7 +860,7 @@ static void remove_pud_table(pud_t *pud_start, unsigned long addr,
                 if (!pud_present(*pud))
                         continue;
 
-                if (pud_huge(*pud)) {
+                if (pud_is_leaf(*pud)) {
                         split_kernel_mapping(addr, end, PUD_SIZE, (pte_t *)pud);
                         continue;
                 }
@@ -886,7 +886,7 @@ static void __meminit remove_pagetable(unsigned long start, unsigned long end)
                 if (!pgd_present(*pgd))
                         continue;
 
-                if (pgd_huge(*pgd)) {
+                if (pgd_is_leaf(*pgd)) {
                         split_kernel_mapping(addr, end, PGDIR_SIZE, (pte_t *)pgd);
                         continue;
                 }
diff --git a/arch/powerpc/mm/pgtable.c b/arch/powerpc/mm/pgtable.c
index 533fc6fa6726..2029e585e5c3 100644
--- a/arch/powerpc/mm/pgtable.c
+++ b/arch/powerpc/mm/pgtable.c
@@ -340,10 +340,11 @@ pte_t *__find_linux_pte(pgd_t *pgdir, unsigned long ea,
         if (pgd_none(pgd))
                 return NULL;
 
-        if (pgd_huge(pgd)) {
+        if (pgd_is_leaf(pgd)) {
                 ret_pte = (pte_t *)pgdp;
                 goto out;
         }
+
         if (is_hugepd(__hugepd(pgd_val(pgd)))) {
                 hpdp = (hugepd_t *)&pgd;
                 goto out_huge;
@@ -361,14 +362,16 @@ pte_t *__find_linux_pte(pgd_t *pgdir, unsigned long ea,
         if (pud_none(pud))
                 return NULL;
 
-        if (pud_huge(pud)) {
+        if (pud_is_leaf(pud)) {
                 ret_pte = (pte_t *)pudp;
                 goto out;
         }
+
         if (is_hugepd(__hugepd(pud_val(pud)))) {
                 hpdp = (hugepd_t *)&pud;
                 goto out_huge;
         }
+
         pdshift = PMD_SHIFT;
         pmdp = pmd_offset(&pud, ea);
         pmd  = READ_ONCE(*pmdp);
@@ -397,15 +400,12 @@ pte_t *__find_linux_pte(pgd_t *pgdir, unsigned long ea,
                 ret_pte = (pte_t *)pmdp;
                 goto out;
         }
-        /*
-         * pmd_large check below will handle the swap pmd pte
-         * we need to do both the check because they are config
-         * dependent.
-         */
-        if (pmd_huge(pmd) || pmd_large(pmd)) {
+
+        if (pmd_is_leaf(pmd)) {
                 ret_pte = (pte_t *)pmdp;
                 goto out;
         }
+
         if (is_hugepd(__hugepd(pmd_val(pmd)))) {
                 hpdp = (hugepd_t *)&pmd;
                 goto out_huge;
diff --git a/arch/powerpc/mm/pgtable_64.c b/arch/powerpc/mm/pgtable_64.c
index 63cd81130643..2892246a6fef 100644
--- a/arch/powerpc/mm/pgtable_64.c
+++ b/arch/powerpc/mm/pgtable_64.c
@@ -309,16 +309,20 @@ EXPORT_SYMBOL(__iounmap_at);
 /* 4 level page table */
 struct page *pgd_page(pgd_t pgd)
 {
-        if (pgd_huge(pgd))
+        if (pgd_is_leaf(pgd)) {
+                VM_WARN_ON(!pgd_huge(pgd));
                 return pte_page(pgd_pte(pgd));
+        }
         return virt_to_page(pgd_page_vaddr(pgd));
 }
 #endif
 
 struct page *pud_page(pud_t pud)
 {
-        if (pud_huge(pud))
+        if (pud_is_leaf(pud)) {
+                VM_WARN_ON(!pud_huge(pud));
                 return pte_page(pud_pte(pud));
+        }
         return virt_to_page(pud_page_vaddr(pud));
 }
 
@@ -328,8 +332,10 @@ struct page *pud_page(pud_t pud)
  */
 struct page *pmd_page(pmd_t pmd)
 {
-        if (pmd_large(pmd) || pmd_huge(pmd) || pmd_devmap(pmd))
+        if (pmd_is_leaf(pmd)) {
+                VM_WARN_ON(!(pmd_large(pmd) || pmd_huge(pmd) || pmd_devmap(pmd)));
                 return pte_page(pmd_pte(pmd));
+        }
         return virt_to_page(pmd_page_vaddr(pmd));
 }
 
diff --git a/arch/powerpc/mm/ptdump/ptdump.c b/arch/powerpc/mm/ptdump/ptdump.c
index 646876d9da64..abe60d25b4e6 100644
--- a/arch/powerpc/mm/ptdump/ptdump.c
+++ b/arch/powerpc/mm/ptdump/ptdump.c
@@ -277,7 +277,7 @@ static void walk_pmd(struct pg_state *st, pud_t *pud, unsigned long start)
 
         for (i = 0; i < PTRS_PER_PMD; i++, pmd++) {
                 addr = start + i * PMD_SIZE;
-                if (!pmd_none(*pmd) && !pmd_huge(*pmd))
+                if (!pmd_none(*pmd) && !pmd_is_leaf(*pmd))
                         /* pmd exists */
                         walk_pte(st, pmd, addr);
                 else
@@ -293,7 +293,7 @@ static void walk_pud(struct pg_state *st, pgd_t *pgd, unsigned long start)
 
         for (i = 0; i < PTRS_PER_PUD; i++, pud++) {
                 addr = start + i * PUD_SIZE;
-                if (!pud_none(*pud) && !pud_huge(*pud))
+                if (!pud_none(*pud) && !pud_is_leaf(*pud))
                         /* pud exists */
                         walk_pmd(st, pud, addr);
                 else
@@ -314,7 +314,7 @@ static void walk_pagetables(struct pg_state *st)
          * the hash pagetable.
          */
         for (i = 0; i < PTRS_PER_PGD; i++, pgd++, addr += PGDIR_SIZE) {
-                if (!pgd_none(*pgd) && !pgd_huge(*pgd))
+                if (!pgd_none(*pgd) && !pgd_is_leaf(*pgd))
                         /* pgd exists */
                         walk_pud(st, pgd, addr);
                 else
diff --git a/arch/powerpc/xmon/xmon.c b/arch/powerpc/xmon/xmon.c
index f879e9fe9733..2ec20a5bb556 100644
--- a/arch/powerpc/xmon/xmon.c
+++ b/arch/powerpc/xmon/xmon.c
@@ -3098,7 +3098,7 @@ static void show_pte(unsigned long addr)
 
         printf("pgd  @ 0x%px\n", pgdir);
 
-        if (pgd_huge(*pgdp)) {
+        if (pgd_is_leaf(*pgdp)) {
                 format_pte(pgdp, pgd_val(*pgdp));
                 return;
         }
@@ -3111,7 +3111,7 @@ static void show_pte(unsigned long addr)
                 return;
         }
 
-        if (pud_huge(*pudp)) {
+        if (pud_is_leaf(*pudp)) {
                 format_pte(pudp, pud_val(*pudp));
                 return;
         }
@@ -3125,7 +3125,7 @@ static void show_pte(unsigned long addr)
                 return;
         }
 
-        if (pmd_huge(*pmdp)) {
+        if (pmd_is_leaf(*pmdp)) {
                 format_pte(pmdp, pmd_val(*pmdp));
                 return;
         }