aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBenjamin Herrenschmidt <benh@kernel.crashing.org>2016-07-05 01:03:58 -0400
committerMichael Ellerman <mpe@ellerman.id.au>2016-07-21 04:59:09 -0400
commit7025776ed1ebdfa1959932e7a4662c2f88607df0 (patch)
tree4b1b0e0be66d8a0b9d2a266a036416c01877ea41
parentb521f576df6c49fcc06fbc06a349f7590f223a38 (diff)
powerpc/mm: Move hash table ops to a separate structure
Moving probe_machine() to after mmu init will cause the ppc_md fields relative to the hash table management to be overwritten. Since we have essentially disconnected the machine type from the hash backend ops, finish the job by moving them to a different structure. The only callback that didn't quite fix is update_partition_table since this is not specific to hash, so I moved it to a standalone variable for now. We can revisit later if needed. Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org> [mpe: Fix ppc64e build failure in kexec] Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Diffstat
-rw-r--r--arch/powerpc/include/asm/book3s/64/mmu-hash.h39
-rw-r--r--arch/powerpc/include/asm/machdep.h36
-rw-r--r--arch/powerpc/kernel/machine_kexec_64.c9
-rw-r--r--arch/powerpc/kernel/misc_64.S2
-rw-r--r--arch/powerpc/kvm/book3s_64_mmu_host.c18
-rw-r--r--arch/powerpc/mm/hash64_4k.c18
-rw-r--r--arch/powerpc/mm/hash64_64k.c39
-rw-r--r--arch/powerpc/mm/hash_native_64.c16
-rw-r--r--arch/powerpc/mm/hash_utils_64.c51
-rw-r--r--arch/powerpc/mm/hugepage-hash64.c17
-rw-r--r--arch/powerpc/mm/hugetlbpage-hash64.c4
-rw-r--r--arch/powerpc/platforms/ps3/htab.c12
-rw-r--r--arch/powerpc/platforms/pseries/lpar.c18
13 files changed, 151 insertions, 128 deletions
diff --git a/arch/powerpc/include/asm/book3s/64/mmu-hash.h b/arch/powerpc/include/asm/book3s/64/mmu-hash.h
index 2c2d55580ae2..b0f4dffe12ae 100644
--- a/arch/powerpc/include/asm/book3s/64/mmu-hash.h
+++ b/arch/powerpc/include/asm/book3s/64/mmu-hash.h
@@ -124,6 +124,45 @@
124 124
125#ifndef __ASSEMBLY__ 125#ifndef __ASSEMBLY__
126 126
127struct mmu_hash_ops {
128 void (*hpte_invalidate)(unsigned long slot,
129 unsigned long vpn,
130 int bpsize, int apsize,
131 int ssize, int local);
132 long (*hpte_updatepp)(unsigned long slot,
133 unsigned long newpp,
134 unsigned long vpn,
135 int bpsize, int apsize,
136 int ssize, unsigned long flags);
137 void (*hpte_updateboltedpp)(unsigned long newpp,
138 unsigned long ea,
139 int psize, int ssize);
140 long (*hpte_insert)(unsigned long hpte_group,
141 unsigned long vpn,
142 unsigned long prpn,
143 unsigned long rflags,
144 unsigned long vflags,
145 int psize, int apsize,
146 int ssize);
147 long (*hpte_remove)(unsigned long hpte_group);
148 int (*hpte_removebolted)(unsigned long ea,
149 int psize, int ssize);
150 void (*flush_hash_range)(unsigned long number, int local);
151 void (*hugepage_invalidate)(unsigned long vsid,
152 unsigned long addr,
153 unsigned char *hpte_slot_array,
154 int psize, int ssize, int local);
155 /*
156 * Special for kexec.
157 * To be called in real mode with interrupts disabled. No locks are
158 * taken as such, concurrent access on pre POWER5 hardware could result
159 * in a deadlock.
160 * The linear mapping is destroyed as well.
161 */
162 void (*hpte_clear_all)(void);
163};
164extern struct mmu_hash_ops mmu_hash_ops;
165
127struct hash_pte { 166struct hash_pte {
128 __be64 v; 167 __be64 v;
129 __be64 r; 168 __be64 r;
diff --git a/arch/powerpc/include/asm/machdep.h b/arch/powerpc/include/asm/machdep.h
index e62e7d33bda0..5b2edf5ba114 100644
--- a/arch/powerpc/include/asm/machdep.h
+++ b/arch/powerpc/include/asm/machdep.h
@@ -34,42 +34,6 @@ struct pci_host_bridge;
34struct machdep_calls { 34struct machdep_calls {
35 char *name; 35 char *name;
36#ifdef CONFIG_PPC64 36#ifdef CONFIG_PPC64
37 void (*hpte_invalidate)(unsigned long slot,
38 unsigned long vpn,
39 int bpsize, int apsize,
40 int ssize, int local);
41 long (*hpte_updatepp)(unsigned long slot,
42 unsigned long newpp,
43 unsigned long vpn,
44 int bpsize, int apsize,
45 int ssize, unsigned long flags);
46 void (*hpte_updateboltedpp)(unsigned long newpp,
47 unsigned long ea,
48 int psize, int ssize);
49 long (*hpte_insert)(unsigned long hpte_group,
50 unsigned long vpn,
51 unsigned long prpn,
52 unsigned long rflags,
53 unsigned long vflags,
54 int psize, int apsize,
55 int ssize);
56 long (*hpte_remove)(unsigned long hpte_group);
57 int (*hpte_removebolted)(unsigned long ea,
58 int psize, int ssize);
59 void (*flush_hash_range)(unsigned long number, int local);
60 void (*hugepage_invalidate)(unsigned long vsid,
61 unsigned long addr,
62 unsigned char *hpte_slot_array,
63 int psize, int ssize, int local);
64 /*
65 * Special for kexec.
66 * To be called in real mode with interrupts disabled. No locks are
67 * taken as such, concurrent access on pre POWER5 hardware could result
68 * in a deadlock.
69 * The linear mapping is destroyed as well.
70 */
71 void (*hpte_clear_all)(void);
72
73 void __iomem * (*ioremap)(phys_addr_t addr, unsigned long size, 37 void __iomem * (*ioremap)(phys_addr_t addr, unsigned long size,
74 unsigned long flags, void *caller); 38 unsigned long flags, void *caller);
75 void (*iounmap)(volatile void __iomem *token); 39 void (*iounmap)(volatile void __iomem *token);
diff --git a/arch/powerpc/kernel/machine_kexec_64.c b/arch/powerpc/kernel/machine_kexec_64.c
index 50bf55135ef8..4c780a342282 100644
--- a/arch/powerpc/kernel/machine_kexec_64.c
+++ b/arch/powerpc/kernel/machine_kexec_64.c
@@ -55,7 +55,7 @@ int default_machine_kexec_prepare(struct kimage *image)
55 const unsigned long *basep; 55 const unsigned long *basep;
56 const unsigned int *sizep; 56 const unsigned int *sizep;
57 57
58 if (!ppc_md.hpte_clear_all) 58 if (!mmu_hash_ops.hpte_clear_all)
59 return -ENOENT; 59 return -ENOENT;
60 60
61 /* 61 /*
@@ -380,7 +380,12 @@ void default_machine_kexec(struct kimage *image)
380 */ 380 */
381 kexec_sequence(&kexec_stack, image->start, image, 381 kexec_sequence(&kexec_stack, image->start, image,
382 page_address(image->control_code_page), 382 page_address(image->control_code_page),
383 ppc_md.hpte_clear_all); 383#ifdef CONFIG_PPC_STD_MMU
384 mmu_hash_ops.hpte_clear_all
385#else
386 NULL
387#endif
388 );
384 /* NOTREACHED */ 389 /* NOTREACHED */
385} 390}
386 391
diff --git a/arch/powerpc/kernel/misc_64.S b/arch/powerpc/kernel/misc_64.S
index 7a8519052b14..cb195157b318 100644
--- a/arch/powerpc/kernel/misc_64.S
+++ b/arch/powerpc/kernel/misc_64.S
@@ -667,7 +667,7 @@ _GLOBAL(kexec_sequence)
667 mr r12,r27 667 mr r12,r27
668#endif 668#endif
669 mtctr r12 669 mtctr r12
670 bctrl /* ppc_md.hpte_clear_all(void); */ 670 bctrl /* mmu_hash_ops.hpte_clear_all(void); */
671#endif /* !CONFIG_PPC_BOOK3E */ 671#endif /* !CONFIG_PPC_BOOK3E */
672 672
673/* 673/*
diff --git a/arch/powerpc/kvm/book3s_64_mmu_host.c b/arch/powerpc/kvm/book3s_64_mmu_host.c
index 114edace6cdd..a587e8f4fd26 100644
--- a/arch/powerpc/kvm/book3s_64_mmu_host.c
+++ b/arch/powerpc/kvm/book3s_64_mmu_host.c
@@ -34,9 +34,9 @@
34 34
35void kvmppc_mmu_invalidate_pte(struct kvm_vcpu *vcpu, struct hpte_cache *pte) 35void kvmppc_mmu_invalidate_pte(struct kvm_vcpu *vcpu, struct hpte_cache *pte)
36{ 36{
37 ppc_md.hpte_invalidate(pte->slot, pte->host_vpn, 37 mmu_hash_ops.hpte_invalidate(pte->slot, pte->host_vpn,
38 pte->pagesize, pte->pagesize, MMU_SEGSIZE_256M, 38 pte->pagesize, pte->pagesize,
39 false); 39 MMU_SEGSIZE_256M, false);
40} 40}
41 41
42/* We keep 512 gvsid->hvsid entries, mapping the guest ones to the array using 42/* We keep 512 gvsid->hvsid entries, mapping the guest ones to the array using
@@ -169,13 +169,13 @@ map_again:
169 169
170 /* In case we tried normal mapping already, let's nuke old entries */ 170 /* In case we tried normal mapping already, let's nuke old entries */
171 if (attempt > 1) 171 if (attempt > 1)
172 if (ppc_md.hpte_remove(hpteg) < 0) { 172 if (mmu_hash_ops.hpte_remove(hpteg) < 0) {
173 r = -1; 173 r = -1;
174 goto out_unlock; 174 goto out_unlock;
175 } 175 }
176 176
177 ret = ppc_md.hpte_insert(hpteg, vpn, hpaddr, rflags, vflags, 177 ret = mmu_hash_ops.hpte_insert(hpteg, vpn, hpaddr, rflags, vflags,
178 hpsize, hpsize, MMU_SEGSIZE_256M); 178 hpsize, hpsize, MMU_SEGSIZE_256M);
179 179
180 if (ret < 0) { 180 if (ret < 0) {
181 /* If we couldn't map a primary PTE, try a secondary */ 181 /* If we couldn't map a primary PTE, try a secondary */
@@ -187,8 +187,10 @@ map_again:
187 trace_kvm_book3s_64_mmu_map(rflags, hpteg, 187 trace_kvm_book3s_64_mmu_map(rflags, hpteg,
188 vpn, hpaddr, orig_pte); 188 vpn, hpaddr, orig_pte);
189 189
190 /* The ppc_md code may give us a secondary entry even though we 190 /*
191 asked for a primary. Fix up. */ 191 * The mmu_hash_ops code may give us a secondary entry even
192 * though we asked for a primary. Fix up.
193 */
192 if ((ret & _PTEIDX_SECONDARY) && !(vflags & HPTE_V_SECONDARY)) { 194 if ((ret & _PTEIDX_SECONDARY) && !(vflags & HPTE_V_SECONDARY)) {
193 hash = ~hash; 195 hash = ~hash;
194 hpteg = ((hash & htab_hash_mask) * HPTES_PER_GROUP); 196 hpteg = ((hash & htab_hash_mask) * HPTES_PER_GROUP);
diff --git a/arch/powerpc/mm/hash64_4k.c b/arch/powerpc/mm/hash64_4k.c
index 6333b273d2d5..42c702b3be1f 100644
--- a/arch/powerpc/mm/hash64_4k.c
+++ b/arch/powerpc/mm/hash64_4k.c
@@ -70,8 +70,8 @@ int __hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
70 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; 70 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
71 slot += (old_pte & H_PAGE_F_GIX) >> H_PAGE_F_GIX_SHIFT; 71 slot += (old_pte & H_PAGE_F_GIX) >> H_PAGE_F_GIX_SHIFT;
72 72
73 if (ppc_md.hpte_updatepp(slot, rflags, vpn, MMU_PAGE_4K, 73 if (mmu_hash_ops.hpte_updatepp(slot, rflags, vpn, MMU_PAGE_4K,
74 MMU_PAGE_4K, ssize, flags) == -1) 74 MMU_PAGE_4K, ssize, flags) == -1)
75 old_pte &= ~_PAGE_HPTEFLAGS; 75 old_pte &= ~_PAGE_HPTEFLAGS;
76 } 76 }
77 77
@@ -84,21 +84,23 @@ repeat:
84 hpte_group = ((hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL; 84 hpte_group = ((hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL;
85 85
86 /* Insert into the hash table, primary slot */ 86 /* Insert into the hash table, primary slot */
87 slot = ppc_md.hpte_insert(hpte_group, vpn, pa, rflags, 0, 87 slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa, rflags, 0,
88 MMU_PAGE_4K, MMU_PAGE_4K, ssize); 88 MMU_PAGE_4K, MMU_PAGE_4K, ssize);
89 /* 89 /*
90 * Primary is full, try the secondary 90 * Primary is full, try the secondary
91 */ 91 */
92 if (unlikely(slot == -1)) { 92 if (unlikely(slot == -1)) {
93 hpte_group = ((~hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL; 93 hpte_group = ((~hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL;
94 slot = ppc_md.hpte_insert(hpte_group, vpn, pa, 94 slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa,
95 rflags, HPTE_V_SECONDARY, 95 rflags,
96 MMU_PAGE_4K, MMU_PAGE_4K, ssize); 96 HPTE_V_SECONDARY,
97 MMU_PAGE_4K,
98 MMU_PAGE_4K, ssize);
97 if (slot == -1) { 99 if (slot == -1) {
98 if (mftb() & 0x1) 100 if (mftb() & 0x1)
99 hpte_group = ((hash & htab_hash_mask) * 101 hpte_group = ((hash & htab_hash_mask) *
100 HPTES_PER_GROUP) & ~0x7UL; 102 HPTES_PER_GROUP) & ~0x7UL;
101 ppc_md.hpte_remove(hpte_group); 103 mmu_hash_ops.hpte_remove(hpte_group);
102 /* 104 /*
103 * FIXME!! Should be try the group from which we removed ? 105 * FIXME!! Should be try the group from which we removed ?
104 */ 106 */
diff --git a/arch/powerpc/mm/hash64_64k.c b/arch/powerpc/mm/hash64_64k.c
index 16644e1f4e6b..3bbbea07378c 100644
--- a/arch/powerpc/mm/hash64_64k.c
+++ b/arch/powerpc/mm/hash64_64k.c
@@ -133,9 +133,9 @@ int __hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
133 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; 133 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
134 slot += hidx & _PTEIDX_GROUP_IX; 134 slot += hidx & _PTEIDX_GROUP_IX;
135 135
136 ret = ppc_md.hpte_updatepp(slot, rflags, vpn, 136 ret = mmu_hash_ops.hpte_updatepp(slot, rflags, vpn,
137 MMU_PAGE_4K, MMU_PAGE_4K, 137 MMU_PAGE_4K, MMU_PAGE_4K,
138 ssize, flags); 138 ssize, flags);
139 /* 139 /*
140 *if we failed because typically the HPTE wasn't really here 140 *if we failed because typically the HPTE wasn't really here
141 * we try an insertion. 141 * we try an insertion.
@@ -166,21 +166,22 @@ repeat:
166 hpte_group = ((hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL; 166 hpte_group = ((hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL;
167 167
168 /* Insert into the hash table, primary slot */ 168 /* Insert into the hash table, primary slot */
169 slot = ppc_md.hpte_insert(hpte_group, vpn, pa, rflags, 0, 169 slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa, rflags, 0,
170 MMU_PAGE_4K, MMU_PAGE_4K, ssize); 170 MMU_PAGE_4K, MMU_PAGE_4K, ssize);
171 /* 171 /*
172 * Primary is full, try the secondary 172 * Primary is full, try the secondary
173 */ 173 */
174 if (unlikely(slot == -1)) { 174 if (unlikely(slot == -1)) {
175 hpte_group = ((~hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL; 175 hpte_group = ((~hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL;
176 slot = ppc_md.hpte_insert(hpte_group, vpn, pa, 176 slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa,
177 rflags, HPTE_V_SECONDARY, 177 rflags, HPTE_V_SECONDARY,
178 MMU_PAGE_4K, MMU_PAGE_4K, ssize); 178 MMU_PAGE_4K, MMU_PAGE_4K,
179 ssize);
179 if (slot == -1) { 180 if (slot == -1) {
180 if (mftb() & 0x1) 181 if (mftb() & 0x1)
181 hpte_group = ((hash & htab_hash_mask) * 182 hpte_group = ((hash & htab_hash_mask) *
182 HPTES_PER_GROUP) & ~0x7UL; 183 HPTES_PER_GROUP) & ~0x7UL;
183 ppc_md.hpte_remove(hpte_group); 184 mmu_hash_ops.hpte_remove(hpte_group);
184 /* 185 /*
185 * FIXME!! Should be try the group from which we removed ? 186 * FIXME!! Should be try the group from which we removed ?
186 */ 187 */
@@ -272,8 +273,9 @@ int __hash_page_64K(unsigned long ea, unsigned long access,
272 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; 273 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
273 slot += (old_pte & H_PAGE_F_GIX) >> H_PAGE_F_GIX_SHIFT; 274 slot += (old_pte & H_PAGE_F_GIX) >> H_PAGE_F_GIX_SHIFT;
274 275
275 if (ppc_md.hpte_updatepp(slot, rflags, vpn, MMU_PAGE_64K, 276 if (mmu_hash_ops.hpte_updatepp(slot, rflags, vpn, MMU_PAGE_64K,
276 MMU_PAGE_64K, ssize, flags) == -1) 277 MMU_PAGE_64K, ssize,
278 flags) == -1)
277 old_pte &= ~_PAGE_HPTEFLAGS; 279 old_pte &= ~_PAGE_HPTEFLAGS;
278 } 280 }
279 281
@@ -286,21 +288,24 @@ repeat:
286 hpte_group = ((hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL; 288 hpte_group = ((hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL;
287 289
288 /* Insert into the hash table, primary slot */ 290 /* Insert into the hash table, primary slot */
289 slot = ppc_md.hpte_insert(hpte_group, vpn, pa, rflags, 0, 291 slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa, rflags, 0,
290 MMU_PAGE_64K, MMU_PAGE_64K, ssize); 292 MMU_PAGE_64K, MMU_PAGE_64K,
293 ssize);
291 /* 294 /*
292 * Primary is full, try the secondary 295 * Primary is full, try the secondary
293 */ 296 */
294 if (unlikely(slot == -1)) { 297 if (unlikely(slot == -1)) {
295 hpte_group = ((~hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL; 298 hpte_group = ((~hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL;
296 slot = ppc_md.hpte_insert(hpte_group, vpn, pa, 299 slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa,
297 rflags, HPTE_V_SECONDARY, 300 rflags,
298 MMU_PAGE_64K, MMU_PAGE_64K, ssize); 301 HPTE_V_SECONDARY,
302 MMU_PAGE_64K,
303 MMU_PAGE_64K, ssize);
299 if (slot == -1) { 304 if (slot == -1) {
300 if (mftb() & 0x1) 305 if (mftb() & 0x1)
301 hpte_group = ((hash & htab_hash_mask) * 306 hpte_group = ((hash & htab_hash_mask) *
302 HPTES_PER_GROUP) & ~0x7UL; 307 HPTES_PER_GROUP) & ~0x7UL;
303 ppc_md.hpte_remove(hpte_group); 308 mmu_hash_ops.hpte_remove(hpte_group);
304 /* 309 /*
305 * FIXME!! Should be try the group from which we removed ? 310 * FIXME!! Should be try the group from which we removed ?
306 */ 311 */
diff --git a/arch/powerpc/mm/hash_native_64.c b/arch/powerpc/mm/hash_native_64.c
index cb3b4c98b637..88ce7d212320 100644
--- a/arch/powerpc/mm/hash_native_64.c
+++ b/arch/powerpc/mm/hash_native_64.c
@@ -739,14 +739,14 @@ static int native_register_proc_table(unsigned long base, unsigned long page_siz
739 739
740void __init hpte_init_native(void) 740void __init hpte_init_native(void)
741{ 741{
742 ppc_md.hpte_invalidate = native_hpte_invalidate; 742 mmu_hash_ops.hpte_invalidate = native_hpte_invalidate;
743 ppc_md.hpte_updatepp = native_hpte_updatepp; 743 mmu_hash_ops.hpte_updatepp = native_hpte_updatepp;
744 ppc_md.hpte_updateboltedpp = native_hpte_updateboltedpp; 744 mmu_hash_ops.hpte_updateboltedpp = native_hpte_updateboltedpp;
745 ppc_md.hpte_insert = native_hpte_insert; 745 mmu_hash_ops.hpte_insert = native_hpte_insert;
746 ppc_md.hpte_remove = native_hpte_remove; 746 mmu_hash_ops.hpte_remove = native_hpte_remove;
747 ppc_md.hpte_clear_all = native_hpte_clear; 747 mmu_hash_ops.hpte_clear_all = native_hpte_clear;
748 ppc_md.flush_hash_range = native_flush_hash_range; 748 mmu_hash_ops.flush_hash_range = native_flush_hash_range;
749 ppc_md.hugepage_invalidate = native_hugepage_invalidate; 749 mmu_hash_ops.hugepage_invalidate = native_hugepage_invalidate;
750 750
751 if (cpu_has_feature(CPU_FTR_ARCH_300)) 751 if (cpu_has_feature(CPU_FTR_ARCH_300))
752 ppc_md.register_process_table = native_register_proc_table; 752 ppc_md.register_process_table = native_register_proc_table;
diff --git a/arch/powerpc/mm/hash_utils_64.c b/arch/powerpc/mm/hash_utils_64.c
index eab3074a6a37..341632471b9d 100644
--- a/arch/powerpc/mm/hash_utils_64.c
+++ b/arch/powerpc/mm/hash_utils_64.c
@@ -118,6 +118,8 @@ static u8 *linear_map_hash_slots;
118static unsigned long linear_map_hash_count; 118static unsigned long linear_map_hash_count;
119static DEFINE_SPINLOCK(linear_map_hash_lock); 119static DEFINE_SPINLOCK(linear_map_hash_lock);
120#endif /* CONFIG_DEBUG_PAGEALLOC */ 120#endif /* CONFIG_DEBUG_PAGEALLOC */
121struct mmu_hash_ops mmu_hash_ops;
122EXPORT_SYMBOL(mmu_hash_ops);
121 123
122/* There are definitions of page sizes arrays to be used when none 124/* There are definitions of page sizes arrays to be used when none
123 * is provided by the firmware. 125 * is provided by the firmware.
@@ -276,9 +278,10 @@ int htab_bolt_mapping(unsigned long vstart, unsigned long vend,
276 hash = hpt_hash(vpn, shift, ssize); 278 hash = hpt_hash(vpn, shift, ssize);
277 hpteg = ((hash & htab_hash_mask) * HPTES_PER_GROUP); 279 hpteg = ((hash & htab_hash_mask) * HPTES_PER_GROUP);
278 280
279 BUG_ON(!ppc_md.hpte_insert); 281 BUG_ON(!mmu_hash_ops.hpte_insert);
280 ret = ppc_md.hpte_insert(hpteg, vpn, paddr, tprot, 282 ret = mmu_hash_ops.hpte_insert(hpteg, vpn, paddr, tprot,
281 HPTE_V_BOLTED, psize, psize, ssize); 283 HPTE_V_BOLTED, psize, psize,
284 ssize);
282 285
283 if (ret < 0) 286 if (ret < 0)
284 break; 287 break;
@@ -303,11 +306,11 @@ int htab_remove_mapping(unsigned long vstart, unsigned long vend,
303 shift = mmu_psize_defs[psize].shift; 306 shift = mmu_psize_defs[psize].shift;
304 step = 1 << shift; 307 step = 1 << shift;
305 308
306 if (!ppc_md.hpte_removebolted) 309 if (!mmu_hash_ops.hpte_removebolted)
307 return -ENODEV; 310 return -ENODEV;
308 311
309 for (vaddr = vstart; vaddr < vend; vaddr += step) { 312 for (vaddr = vstart; vaddr < vend; vaddr += step) {
310 rc = ppc_md.hpte_removebolted(vaddr, psize, ssize); 313 rc = mmu_hash_ops.hpte_removebolted(vaddr, psize, ssize);
311 if (rc == -ENOENT) { 314 if (rc == -ENOENT) {
312 ret = -ENOENT; 315 ret = -ENOENT;
313 continue; 316 continue;
@@ -789,8 +792,8 @@ static void __init htab_initialize(void)
789 * Clear the htab if firmware assisted dump is active so 792 * Clear the htab if firmware assisted dump is active so
790 * that we dont end up using old mappings. 793 * that we dont end up using old mappings.
791 */ 794 */
792 if (is_fadump_active() && ppc_md.hpte_clear_all) 795 if (is_fadump_active() && mmu_hash_ops.hpte_clear_all)
793 ppc_md.hpte_clear_all(); 796 mmu_hash_ops.hpte_clear_all();
794#endif 797#endif
795 } else { 798 } else {
796 unsigned long limit = MEMBLOCK_ALLOC_ANYWHERE; 799 unsigned long limit = MEMBLOCK_ALLOC_ANYWHERE;
@@ -1480,7 +1483,8 @@ void flush_hash_page(unsigned long vpn, real_pte_t pte, int psize, int ssize,
1480 * We use same base page size and actual psize, because we don't 1483 * We use same base page size and actual psize, because we don't
1481 * use these functions for hugepage 1484 * use these functions for hugepage
1482 */ 1485 */
1483 ppc_md.hpte_invalidate(slot, vpn, psize, psize, ssize, local); 1486 mmu_hash_ops.hpte_invalidate(slot, vpn, psize, psize,
1487 ssize, local);
1484 } pte_iterate_hashed_end(); 1488 } pte_iterate_hashed_end();
1485 1489
1486#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1490#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
@@ -1521,9 +1525,9 @@ void flush_hash_hugepage(unsigned long vsid, unsigned long addr,
1521 if (!hpte_slot_array) 1525 if (!hpte_slot_array)
1522 return; 1526 return;
1523 1527
1524 if (ppc_md.hugepage_invalidate) { 1528 if (mmu_hash_ops.hugepage_invalidate) {
1525 ppc_md.hugepage_invalidate(vsid, s_addr, hpte_slot_array, 1529 mmu_hash_ops.hugepage_invalidate(vsid, s_addr, hpte_slot_array,
1526 psize, ssize, local); 1530 psize, ssize, local);
1527 goto tm_abort; 1531 goto tm_abort;
1528 } 1532 }
1529 /* 1533 /*
@@ -1550,8 +1554,8 @@ void flush_hash_hugepage(unsigned long vsid, unsigned long addr,
1550 1554
1551 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; 1555 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
1552 slot += hidx & _PTEIDX_GROUP_IX; 1556 slot += hidx & _PTEIDX_GROUP_IX;
1553 ppc_md.hpte_invalidate(slot, vpn, psize, 1557 mmu_hash_ops.hpte_invalidate(slot, vpn, psize,
1554 MMU_PAGE_16M, ssize, local); 1558 MMU_PAGE_16M, ssize, local);
1555 } 1559 }
1556tm_abort: 1560tm_abort:
1557#ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1561#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
@@ -1575,8 +1579,8 @@ tm_abort:
1575 1579
1576void flush_hash_range(unsigned long number, int local) 1580void flush_hash_range(unsigned long number, int local)
1577{ 1581{
1578 if (ppc_md.flush_hash_range) 1582 if (mmu_hash_ops.flush_hash_range)
1579 ppc_md.flush_hash_range(number, local); 1583 mmu_hash_ops.flush_hash_range(number, local);
1580 else { 1584 else {
1581 int i; 1585 int i;
1582 struct ppc64_tlb_batch *batch = 1586 struct ppc64_tlb_batch *batch =
@@ -1621,22 +1625,22 @@ repeat:
1621 HPTES_PER_GROUP) & ~0x7UL; 1625 HPTES_PER_GROUP) & ~0x7UL;
1622 1626
1623 /* Insert into the hash table, primary slot */ 1627 /* Insert into the hash table, primary slot */
1624 slot = ppc_md.hpte_insert(hpte_group, vpn, pa, rflags, vflags, 1628 slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa, rflags, vflags,
1625 psize, psize, ssize); 1629 psize, psize, ssize);
1626 1630
1627 /* Primary is full, try the secondary */ 1631 /* Primary is full, try the secondary */
1628 if (unlikely(slot == -1)) { 1632 if (unlikely(slot == -1)) {
1629 hpte_group = ((~hash & htab_hash_mask) * 1633 hpte_group = ((~hash & htab_hash_mask) *
1630 HPTES_PER_GROUP) & ~0x7UL; 1634 HPTES_PER_GROUP) & ~0x7UL;
1631 slot = ppc_md.hpte_insert(hpte_group, vpn, pa, rflags, 1635 slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa, rflags,
1632 vflags | HPTE_V_SECONDARY, 1636 vflags | HPTE_V_SECONDARY,
1633 psize, psize, ssize); 1637 psize, psize, ssize);
1634 if (slot == -1) { 1638 if (slot == -1) {
1635 if (mftb() & 0x1) 1639 if (mftb() & 0x1)
1636 hpte_group = ((hash & htab_hash_mask) * 1640 hpte_group = ((hash & htab_hash_mask) *
1637 HPTES_PER_GROUP)&~0x7UL; 1641 HPTES_PER_GROUP)&~0x7UL;
1638 1642
1639 ppc_md.hpte_remove(hpte_group); 1643 mmu_hash_ops.hpte_remove(hpte_group);
1640 goto repeat; 1644 goto repeat;
1641 } 1645 }
1642 } 1646 }
@@ -1686,8 +1690,9 @@ static void kernel_unmap_linear_page(unsigned long vaddr, unsigned long lmi)
1686 hash = ~hash; 1690 hash = ~hash;
1687 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; 1691 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
1688 slot += hidx & _PTEIDX_GROUP_IX; 1692 slot += hidx & _PTEIDX_GROUP_IX;
1689 ppc_md.hpte_invalidate(slot, vpn, mmu_linear_psize, mmu_linear_psize, 1693 mmu_hash_ops.hpte_invalidate(slot, vpn, mmu_linear_psize,
1690 mmu_kernel_ssize, 0); 1694 mmu_linear_psize,
1695 mmu_kernel_ssize, 0);
1691} 1696}
1692 1697
1693void __kernel_map_pages(struct page *page, int numpages, int enable) 1698void __kernel_map_pages(struct page *page, int numpages, int enable)
diff --git a/arch/powerpc/mm/hugepage-hash64.c b/arch/powerpc/mm/hugepage-hash64.c
index ba3fc229468a..f20d16f849c5 100644
--- a/arch/powerpc/mm/hugepage-hash64.c
+++ b/arch/powerpc/mm/hugepage-hash64.c
@@ -103,8 +103,8 @@ int __hash_page_thp(unsigned long ea, unsigned long access, unsigned long vsid,
103 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; 103 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
104 slot += hidx & _PTEIDX_GROUP_IX; 104 slot += hidx & _PTEIDX_GROUP_IX;
105 105
106 ret = ppc_md.hpte_updatepp(slot, rflags, vpn, 106 ret = mmu_hash_ops.hpte_updatepp(slot, rflags, vpn,
107 psize, lpsize, ssize, flags); 107 psize, lpsize, ssize, flags);
108 /* 108 /*
109 * We failed to update, try to insert a new entry. 109 * We failed to update, try to insert a new entry.
110 */ 110 */
@@ -131,23 +131,24 @@ repeat:
131 hpte_group = ((hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL; 131 hpte_group = ((hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL;
132 132
133 /* Insert into the hash table, primary slot */ 133 /* Insert into the hash table, primary slot */
134 slot = ppc_md.hpte_insert(hpte_group, vpn, pa, rflags, 0, 134 slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa, rflags, 0,
135 psize, lpsize, ssize); 135 psize, lpsize, ssize);
136 /* 136 /*
137 * Primary is full, try the secondary 137 * Primary is full, try the secondary
138 */ 138 */
139 if (unlikely(slot == -1)) { 139 if (unlikely(slot == -1)) {
140 hpte_group = ((~hash & htab_hash_mask) * 140 hpte_group = ((~hash & htab_hash_mask) *
141 HPTES_PER_GROUP) & ~0x7UL; 141 HPTES_PER_GROUP) & ~0x7UL;
142 slot = ppc_md.hpte_insert(hpte_group, vpn, pa, 142 slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa,
143 rflags, HPTE_V_SECONDARY, 143 rflags,
144 psize, lpsize, ssize); 144 HPTE_V_SECONDARY,
145 psize, lpsize, ssize);
145 if (slot == -1) { 146 if (slot == -1) {
146 if (mftb() & 0x1) 147 if (mftb() & 0x1)
147 hpte_group = ((hash & htab_hash_mask) * 148 hpte_group = ((hash & htab_hash_mask) *
148 HPTES_PER_GROUP) & ~0x7UL; 149 HPTES_PER_GROUP) & ~0x7UL;
149 150
150 ppc_md.hpte_remove(hpte_group); 151 mmu_hash_ops.hpte_remove(hpte_group);
151 goto repeat; 152 goto repeat;
152 } 153 }
153 } 154 }
diff --git a/arch/powerpc/mm/hugetlbpage-hash64.c b/arch/powerpc/mm/hugetlbpage-hash64.c
index 3058560b6121..d5026f3800b6 100644
--- a/arch/powerpc/mm/hugetlbpage-hash64.c
+++ b/arch/powerpc/mm/hugetlbpage-hash64.c
@@ -79,8 +79,8 @@ int __hash_page_huge(unsigned long ea, unsigned long access, unsigned long vsid,
79 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; 79 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
80 slot += (old_pte & H_PAGE_F_GIX) >> H_PAGE_F_GIX_SHIFT; 80 slot += (old_pte & H_PAGE_F_GIX) >> H_PAGE_F_GIX_SHIFT;
81 81
82 if (ppc_md.hpte_updatepp(slot, rflags, vpn, mmu_psize, 82 if (mmu_hash_ops.hpte_updatepp(slot, rflags, vpn, mmu_psize,
83 mmu_psize, ssize, flags) == -1) 83 mmu_psize, ssize, flags) == -1)
84 old_pte &= ~_PAGE_HPTEFLAGS; 84 old_pte &= ~_PAGE_HPTEFLAGS;
85 } 85 }
86 86
diff --git a/arch/powerpc/platforms/ps3/htab.c b/arch/powerpc/platforms/ps3/htab.c
index c9a3e677192a..cb3c50328de8 100644
--- a/arch/powerpc/platforms/ps3/htab.c
+++ b/arch/powerpc/platforms/ps3/htab.c
@@ -195,12 +195,12 @@ static void ps3_hpte_clear(void)
195 195
196void __init ps3_hpte_init(unsigned long htab_size) 196void __init ps3_hpte_init(unsigned long htab_size)
197{ 197{
198 ppc_md.hpte_invalidate = ps3_hpte_invalidate; 198 mmu_hash_ops.hpte_invalidate = ps3_hpte_invalidate;
199 ppc_md.hpte_updatepp = ps3_hpte_updatepp; 199 mmu_hash_ops.hpte_updatepp = ps3_hpte_updatepp;
200 ppc_md.hpte_updateboltedpp = ps3_hpte_updateboltedpp; 200 mmu_hash_ops.hpte_updateboltedpp = ps3_hpte_updateboltedpp;
201 ppc_md.hpte_insert = ps3_hpte_insert; 201 mmu_hash_ops.hpte_insert = ps3_hpte_insert;
202 ppc_md.hpte_remove = ps3_hpte_remove; 202 mmu_hash_ops.hpte_remove = ps3_hpte_remove;
203 ppc_md.hpte_clear_all = ps3_hpte_clear; 203 mmu_hash_ops.hpte_clear_all = ps3_hpte_clear;
204 204
205 ppc64_pft_size = __ilog2(htab_size); 205 ppc64_pft_size = __ilog2(htab_size);
206} 206}
diff --git a/arch/powerpc/platforms/pseries/lpar.c b/arch/powerpc/platforms/pseries/lpar.c
index 03c732afef34..0e91388d0af9 100644
--- a/arch/powerpc/platforms/pseries/lpar.c
+++ b/arch/powerpc/platforms/pseries/lpar.c
@@ -591,15 +591,15 @@ __setup("bulk_remove=", disable_bulk_remove);
591 591
592void __init hpte_init_lpar(void) 592void __init hpte_init_lpar(void)
593{ 593{
594 ppc_md.hpte_invalidate = pSeries_lpar_hpte_invalidate; 594 mmu_hash_ops.hpte_invalidate = pSeries_lpar_hpte_invalidate;
595 ppc_md.hpte_updatepp = pSeries_lpar_hpte_updatepp; 595 mmu_hash_ops.hpte_updatepp = pSeries_lpar_hpte_updatepp;
596 ppc_md.hpte_updateboltedpp = pSeries_lpar_hpte_updateboltedpp; 596 mmu_hash_ops.hpte_updateboltedpp = pSeries_lpar_hpte_updateboltedpp;
597 ppc_md.hpte_insert = pSeries_lpar_hpte_insert; 597 mmu_hash_ops.hpte_insert = pSeries_lpar_hpte_insert;
598 ppc_md.hpte_remove = pSeries_lpar_hpte_remove; 598 mmu_hash_ops.hpte_remove = pSeries_lpar_hpte_remove;
599 ppc_md.hpte_removebolted = pSeries_lpar_hpte_removebolted; 599 mmu_hash_ops.hpte_removebolted = pSeries_lpar_hpte_removebolted;
600 ppc_md.flush_hash_range = pSeries_lpar_flush_hash_range; 600 mmu_hash_ops.flush_hash_range = pSeries_lpar_flush_hash_range;
601 ppc_md.hpte_clear_all = pSeries_lpar_hptab_clear; 601 mmu_hash_ops.hpte_clear_all = pSeries_lpar_hptab_clear;
602 ppc_md.hugepage_invalidate = pSeries_lpar_hugepage_invalidate; 602 mmu_hash_ops.hugepage_invalidate = pSeries_lpar_hugepage_invalidate;
603} 603}
604 604
605#ifdef CONFIG_PPC_SMLPAR 605#ifdef CONFIG_PPC_SMLPAR
generated by cgit v1.2.2 (git 2.25.0) at 2025-04-13 03:07:26 -0400