powerpc/mm: Move hash table ops to a separate structure

Moving probe_machine() to after mmu init will cause the ppc_md
fields relative to the hash table management to be overwritten.

Since we have essentially disconnected the machine type from
the hash backend ops, finish the job by moving them to a different
structure.

The only callback that didn't quite fix is update_partition_table
since this is not specific to hash, so I moved it to a standalone
variable for now. We can revisit later if needed.

Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
[mpe: Fix ppc64e build failure in kexec]
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>

6 files changed, 79 insertions, 66 deletions

diff --git a/arch/powerpc/mm/hash64_4k.c b/arch/powerpc/mm/hash64_4k.c
index 6333b273d2d5..42c702b3be1f 100644
--- a/arch/powerpc/mm/hash64_4k.c
+++ b/arch/powerpc/mm/hash64_4k.c
@@ -70,8 +70,8 @@ int __hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
                 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
                 slot += (old_pte & H_PAGE_F_GIX) >> H_PAGE_F_GIX_SHIFT;
 
-                if (ppc_md.hpte_updatepp(slot, rflags, vpn, MMU_PAGE_4K,
-                                         MMU_PAGE_4K, ssize, flags) == -1)
+                if (mmu_hash_ops.hpte_updatepp(slot, rflags, vpn, MMU_PAGE_4K,
+                                               MMU_PAGE_4K, ssize, flags) == -1)
                         old_pte &= ~_PAGE_HPTEFLAGS;
         }
 
@@ -84,21 +84,23 @@ repeat:
                 hpte_group = ((hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL;
 
                 /* Insert into the hash table, primary slot */
-                slot = ppc_md.hpte_insert(hpte_group, vpn, pa, rflags, 0,
-                                  MMU_PAGE_4K, MMU_PAGE_4K, ssize);
+                slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa, rflags, 0,
+                                                MMU_PAGE_4K, MMU_PAGE_4K, ssize);
                 /*
                  * Primary is full, try the secondary
                  */
                 if (unlikely(slot == -1)) {
                         hpte_group = ((~hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL;
-                        slot = ppc_md.hpte_insert(hpte_group, vpn, pa,
-                                                  rflags, HPTE_V_SECONDARY,
-                                                  MMU_PAGE_4K, MMU_PAGE_4K, ssize);
+                        slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa,
+                                                        rflags,
+                                                        HPTE_V_SECONDARY,
+                                                        MMU_PAGE_4K,
+                                                        MMU_PAGE_4K, ssize);
                         if (slot == -1) {
                                 if (mftb() & 0x1)
                                         hpte_group = ((hash & htab_hash_mask) *
                                                       HPTES_PER_GROUP) & ~0x7UL;
-                                ppc_md.hpte_remove(hpte_group);
+                                mmu_hash_ops.hpte_remove(hpte_group);
                                 /*
                                  * FIXME!! Should be try the group from which we removed ?
                                  */
diff --git a/arch/powerpc/mm/hash64_64k.c b/arch/powerpc/mm/hash64_64k.c
index 16644e1f4e6b..3bbbea07378c 100644
--- a/arch/powerpc/mm/hash64_64k.c
+++ b/arch/powerpc/mm/hash64_64k.c
@@ -133,9 +133,9 @@ int __hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
                 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
                 slot += hidx & _PTEIDX_GROUP_IX;
 
-                ret = ppc_md.hpte_updatepp(slot, rflags, vpn,
-                                           MMU_PAGE_4K, MMU_PAGE_4K,
-                                           ssize, flags);
+                ret = mmu_hash_ops.hpte_updatepp(slot, rflags, vpn,
+                                                 MMU_PAGE_4K, MMU_PAGE_4K,
+                                                 ssize, flags);
                 /*
                  *if we failed because typically the HPTE wasn't really here
                  * we try an insertion.
@@ -166,21 +166,22 @@ repeat:
         hpte_group = ((hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL;
 
         /* Insert into the hash table, primary slot */
-        slot = ppc_md.hpte_insert(hpte_group, vpn, pa, rflags, 0,
-                                  MMU_PAGE_4K, MMU_PAGE_4K, ssize);
+        slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa, rflags, 0,
+                                        MMU_PAGE_4K, MMU_PAGE_4K, ssize);
         /*
          * Primary is full, try the secondary
          */
         if (unlikely(slot == -1)) {
                 hpte_group = ((~hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL;
-                slot = ppc_md.hpte_insert(hpte_group, vpn, pa,
-                                          rflags, HPTE_V_SECONDARY,
-                                          MMU_PAGE_4K, MMU_PAGE_4K, ssize);
+                slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa,
+                                                rflags, HPTE_V_SECONDARY,
+                                                MMU_PAGE_4K, MMU_PAGE_4K,
+                                                ssize);
                 if (slot == -1) {
                         if (mftb() & 0x1)
                                 hpte_group = ((hash & htab_hash_mask) *
                                               HPTES_PER_GROUP) & ~0x7UL;
-                        ppc_md.hpte_remove(hpte_group);
+                        mmu_hash_ops.hpte_remove(hpte_group);
                         /*
                          * FIXME!! Should be try the group from which we removed ?
                          */
@@ -272,8 +273,9 @@ int __hash_page_64K(unsigned long ea, unsigned long access,
                 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
                 slot += (old_pte & H_PAGE_F_GIX) >> H_PAGE_F_GIX_SHIFT;
 
-                if (ppc_md.hpte_updatepp(slot, rflags, vpn, MMU_PAGE_64K,
-                                         MMU_PAGE_64K, ssize, flags) == -1)
+                if (mmu_hash_ops.hpte_updatepp(slot, rflags, vpn, MMU_PAGE_64K,
+                                               MMU_PAGE_64K, ssize,
+                                               flags) == -1)
                         old_pte &= ~_PAGE_HPTEFLAGS;
         }
 
@@ -286,21 +288,24 @@ repeat:
                 hpte_group = ((hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL;
 
                 /* Insert into the hash table, primary slot */
-                slot = ppc_md.hpte_insert(hpte_group, vpn, pa, rflags, 0,
-                                  MMU_PAGE_64K, MMU_PAGE_64K, ssize);
+                slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa, rflags, 0,
+                                                MMU_PAGE_64K, MMU_PAGE_64K,
+                                                ssize);
                 /*
                  * Primary is full, try the secondary
                  */
                 if (unlikely(slot == -1)) {
                         hpte_group = ((~hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL;
-                        slot = ppc_md.hpte_insert(hpte_group, vpn, pa,
-                                                  rflags, HPTE_V_SECONDARY,
-                                                  MMU_PAGE_64K, MMU_PAGE_64K, ssize);
+                        slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa,
+                                                        rflags,
+                                                        HPTE_V_SECONDARY,
+                                                        MMU_PAGE_64K,
+                                                        MMU_PAGE_64K, ssize);
                         if (slot == -1) {
                                 if (mftb() & 0x1)
                                         hpte_group = ((hash & htab_hash_mask) *
                                                       HPTES_PER_GROUP) & ~0x7UL;
-                                ppc_md.hpte_remove(hpte_group);
+                                mmu_hash_ops.hpte_remove(hpte_group);
                                 /*
                                  * FIXME!! Should be try the group from which we removed ?
                                  */
diff --git a/arch/powerpc/mm/hash_native_64.c b/arch/powerpc/mm/hash_native_64.c
index cb3b4c98b637..88ce7d212320 100644
--- a/arch/powerpc/mm/hash_native_64.c
+++ b/arch/powerpc/mm/hash_native_64.c
@@ -739,14 +739,14 @@ static int native_register_proc_table(unsigned long base, unsigned long page_siz
 
 void __init hpte_init_native(void)
 {
-        ppc_md.hpte_invalidate  = native_hpte_invalidate;
-        ppc_md.hpte_updatepp    = native_hpte_updatepp;
-        ppc_md.hpte_updateboltedpp = native_hpte_updateboltedpp;
-        ppc_md.hpte_insert      = native_hpte_insert;
-        ppc_md.hpte_remove      = native_hpte_remove;
-        ppc_md.hpte_clear_all   = native_hpte_clear;
-        ppc_md.flush_hash_range = native_flush_hash_range;
-        ppc_md.hugepage_invalidate   = native_hugepage_invalidate;
+        mmu_hash_ops.hpte_invalidate    = native_hpte_invalidate;
+        mmu_hash_ops.hpte_updatepp      = native_hpte_updatepp;
+        mmu_hash_ops.hpte_updateboltedpp = native_hpte_updateboltedpp;
+        mmu_hash_ops.hpte_insert        = native_hpte_insert;
+        mmu_hash_ops.hpte_remove        = native_hpte_remove;
+        mmu_hash_ops.hpte_clear_all     = native_hpte_clear;
+        mmu_hash_ops.flush_hash_range = native_flush_hash_range;
+        mmu_hash_ops.hugepage_invalidate   = native_hugepage_invalidate;
 
         if (cpu_has_feature(CPU_FTR_ARCH_300))
                 ppc_md.register_process_table = native_register_proc_table;
diff --git a/arch/powerpc/mm/hash_utils_64.c b/arch/powerpc/mm/hash_utils_64.c
index eab3074a6a37..341632471b9d 100644
--- a/arch/powerpc/mm/hash_utils_64.c
+++ b/arch/powerpc/mm/hash_utils_64.c
@@ -118,6 +118,8 @@ static u8 *linear_map_hash_slots;
 static unsigned long linear_map_hash_count;
 static DEFINE_SPINLOCK(linear_map_hash_lock);
 #endif /* CONFIG_DEBUG_PAGEALLOC */
+struct mmu_hash_ops mmu_hash_ops;
+EXPORT_SYMBOL(mmu_hash_ops);
 
 /* There are definitions of page sizes arrays to be used when none
  * is provided by the firmware.
@@ -276,9 +278,10 @@ int htab_bolt_mapping(unsigned long vstart, unsigned long vend,
                 hash = hpt_hash(vpn, shift, ssize);
                 hpteg = ((hash & htab_hash_mask) * HPTES_PER_GROUP);
 
-                BUG_ON(!ppc_md.hpte_insert);
-                ret = ppc_md.hpte_insert(hpteg, vpn, paddr, tprot,
-                                         HPTE_V_BOLTED, psize, psize, ssize);
+                BUG_ON(!mmu_hash_ops.hpte_insert);
+                ret = mmu_hash_ops.hpte_insert(hpteg, vpn, paddr, tprot,
+                                               HPTE_V_BOLTED, psize, psize,
+                                               ssize);
 
                 if (ret < 0)
                         break;
@@ -303,11 +306,11 @@ int htab_remove_mapping(unsigned long vstart, unsigned long vend,
         shift = mmu_psize_defs[psize].shift;
         step = 1 << shift;
 
-        if (!ppc_md.hpte_removebolted)
+        if (!mmu_hash_ops.hpte_removebolted)
                 return -ENODEV;
 
         for (vaddr = vstart; vaddr < vend; vaddr += step) {
-                rc = ppc_md.hpte_removebolted(vaddr, psize, ssize);
+                rc = mmu_hash_ops.hpte_removebolted(vaddr, psize, ssize);
                 if (rc == -ENOENT) {
                         ret = -ENOENT;
                         continue;
@@ -789,8 +792,8 @@ static void __init htab_initialize(void)
                  * Clear the htab if firmware assisted dump is active so
                  * that we dont end up using old mappings.
                  */
-                if (is_fadump_active() && ppc_md.hpte_clear_all)
-                        ppc_md.hpte_clear_all();
+                if (is_fadump_active() && mmu_hash_ops.hpte_clear_all)
+                        mmu_hash_ops.hpte_clear_all();
 #endif
         } else {
                 unsigned long limit = MEMBLOCK_ALLOC_ANYWHERE;
@@ -1480,7 +1483,8 @@ void flush_hash_page(unsigned long vpn, real_pte_t pte, int psize, int ssize,
                  * We use same base page size and actual psize, because we don't
                  * use these functions for hugepage
                  */
-                ppc_md.hpte_invalidate(slot, vpn, psize, psize, ssize, local);
+                mmu_hash_ops.hpte_invalidate(slot, vpn, psize, psize,
+                                             ssize, local);
         } pte_iterate_hashed_end();
 
 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
@@ -1521,9 +1525,9 @@ void flush_hash_hugepage(unsigned long vsid, unsigned long addr,
         if (!hpte_slot_array)
                 return;
 
-        if (ppc_md.hugepage_invalidate) {
-                ppc_md.hugepage_invalidate(vsid, s_addr, hpte_slot_array,
-                                           psize, ssize, local);
+        if (mmu_hash_ops.hugepage_invalidate) {
+                mmu_hash_ops.hugepage_invalidate(vsid, s_addr, hpte_slot_array,
+                                                 psize, ssize, local);
                 goto tm_abort;
         }
         /*
@@ -1550,8 +1554,8 @@ void flush_hash_hugepage(unsigned long vsid, unsigned long addr,
 
                 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
                 slot += hidx & _PTEIDX_GROUP_IX;
-                ppc_md.hpte_invalidate(slot, vpn, psize,
-                                       MMU_PAGE_16M, ssize, local);
+                mmu_hash_ops.hpte_invalidate(slot, vpn, psize,
+                                             MMU_PAGE_16M, ssize, local);
         }
 tm_abort:
 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
@@ -1575,8 +1579,8 @@ tm_abort:
 
 void flush_hash_range(unsigned long number, int local)
 {
-        if (ppc_md.flush_hash_range)
-                ppc_md.flush_hash_range(number, local);
+        if (mmu_hash_ops.flush_hash_range)
+                mmu_hash_ops.flush_hash_range(number, local);
         else {
                 int i;
                 struct ppc64_tlb_batch *batch =
@@ -1621,22 +1625,22 @@ repeat:
                        HPTES_PER_GROUP) & ~0x7UL;
 
         /* Insert into the hash table, primary slot */
-        slot = ppc_md.hpte_insert(hpte_group, vpn, pa, rflags, vflags,
-                                  psize, psize, ssize);
+        slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa, rflags, vflags,
+                                        psize, psize, ssize);
 
         /* Primary is full, try the secondary */
         if (unlikely(slot == -1)) {
                 hpte_group = ((~hash & htab_hash_mask) *
                               HPTES_PER_GROUP) & ~0x7UL;
-                slot = ppc_md.hpte_insert(hpte_group, vpn, pa, rflags,
-                                          vflags | HPTE_V_SECONDARY,
-                                          psize, psize, ssize);
+                slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa, rflags,
+                                                vflags | HPTE_V_SECONDARY,
+                                                psize, psize, ssize);
                 if (slot == -1) {
                         if (mftb() & 0x1)
                                 hpte_group = ((hash & htab_hash_mask) *
                                               HPTES_PER_GROUP)&~0x7UL;
 
-                        ppc_md.hpte_remove(hpte_group);
+                        mmu_hash_ops.hpte_remove(hpte_group);
                         goto repeat;
                 }
         }
@@ -1686,8 +1690,9 @@ static void kernel_unmap_linear_page(unsigned long vaddr, unsigned long lmi)
                 hash = ~hash;
         slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
         slot += hidx & _PTEIDX_GROUP_IX;
-        ppc_md.hpte_invalidate(slot, vpn, mmu_linear_psize, mmu_linear_psize,
-                               mmu_kernel_ssize, 0);
+        mmu_hash_ops.hpte_invalidate(slot, vpn, mmu_linear_psize,
+                                     mmu_linear_psize,
+                                     mmu_kernel_ssize, 0);
 }
 
 void __kernel_map_pages(struct page *page, int numpages, int enable)
diff --git a/arch/powerpc/mm/hugepage-hash64.c b/arch/powerpc/mm/hugepage-hash64.c
index ba3fc229468a..f20d16f849c5 100644
--- a/arch/powerpc/mm/hugepage-hash64.c
+++ b/arch/powerpc/mm/hugepage-hash64.c
@@ -103,8 +103,8 @@ int __hash_page_thp(unsigned long ea, unsigned long access, unsigned long vsid,
                 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
                 slot += hidx & _PTEIDX_GROUP_IX;
 
-                ret = ppc_md.hpte_updatepp(slot, rflags, vpn,
-                                           psize, lpsize, ssize, flags);
+                ret = mmu_hash_ops.hpte_updatepp(slot, rflags, vpn,
+                                                 psize, lpsize, ssize, flags);
                 /*
                  * We failed to update, try to insert a new entry.
                  */
@@ -131,23 +131,24 @@ repeat:
                 hpte_group = ((hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL;
 
                 /* Insert into the hash table, primary slot */
-                slot = ppc_md.hpte_insert(hpte_group, vpn, pa, rflags, 0,
-                                          psize, lpsize, ssize);
+                slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa, rflags, 0,
+                                                psize, lpsize, ssize);
                 /*
                  * Primary is full, try the secondary
                  */
                 if (unlikely(slot == -1)) {
                         hpte_group = ((~hash & htab_hash_mask) *
                                       HPTES_PER_GROUP) & ~0x7UL;
-                        slot = ppc_md.hpte_insert(hpte_group, vpn, pa,
-                                                  rflags, HPTE_V_SECONDARY,
-                                                  psize, lpsize, ssize);
+                        slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa,
+                                                        rflags,
+                                                        HPTE_V_SECONDARY,
+                                                        psize, lpsize, ssize);
                         if (slot == -1) {
                                 if (mftb() & 0x1)
                                         hpte_group = ((hash & htab_hash_mask) *
                                                       HPTES_PER_GROUP) & ~0x7UL;
 
-                                ppc_md.hpte_remove(hpte_group);
+                                mmu_hash_ops.hpte_remove(hpte_group);
                                 goto repeat;
                         }
                 }
diff --git a/arch/powerpc/mm/hugetlbpage-hash64.c b/arch/powerpc/mm/hugetlbpage-hash64.c
index 3058560b6121..d5026f3800b6 100644
--- a/arch/powerpc/mm/hugetlbpage-hash64.c
+++ b/arch/powerpc/mm/hugetlbpage-hash64.c
@@ -79,8 +79,8 @@ int __hash_page_huge(unsigned long ea, unsigned long access, unsigned long vsid,
                 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
                 slot += (old_pte & H_PAGE_F_GIX) >> H_PAGE_F_GIX_SHIFT;
 
-                if (ppc_md.hpte_updatepp(slot, rflags, vpn, mmu_psize,
-                                         mmu_psize, ssize, flags) == -1)
+                if (mmu_hash_ops.hpte_updatepp(slot, rflags, vpn, mmu_psize,
+                                               mmu_psize, ssize, flags) == -1)
                         old_pte &= ~_PAGE_HPTEFLAGS;
         }