diff options
author | Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com> | 2015-11-30 22:36:49 -0500 |
---|---|---|
committer | Michael Ellerman <mpe@ellerman.id.au> | 2015-12-13 23:19:12 -0500 |
commit | a43c0eb8364c022725df586e91dd753633374d66 (patch) | |
tree | 3615f416ee6ab584340c89691689f58f35fc1694 | |
parent | 89ff725051d177556b23d80f2a30f880a657a6c1 (diff) |
powerpc/mm: Convert 4k insert from asm to C
This is similar to 64K insert. May be we want to consolidate
Acked-by: Scott Wood <scottwood@freescale.com>
Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
-rw-r--r-- | arch/powerpc/mm/Makefile | 6 | ||||
-rw-r--r-- | arch/powerpc/mm/hash64_4k.c | 139 | ||||
-rw-r--r-- | arch/powerpc/mm/hash_low_64.S | 331 | ||||
-rw-r--r-- | arch/powerpc/mm/hash_utils_64.c | 26 |
4 files changed, 142 insertions, 360 deletions
diff --git a/arch/powerpc/mm/Makefile b/arch/powerpc/mm/Makefile index f80ad1a76cc8..1ffeda85c086 100644 --- a/arch/powerpc/mm/Makefile +++ b/arch/powerpc/mm/Makefile | |||
@@ -14,11 +14,11 @@ obj-$(CONFIG_PPC_MMU_NOHASH) += mmu_context_nohash.o tlb_nohash.o \ | |||
14 | obj-$(CONFIG_PPC_BOOK3E) += tlb_low_$(CONFIG_WORD_SIZE)e.o | 14 | obj-$(CONFIG_PPC_BOOK3E) += tlb_low_$(CONFIG_WORD_SIZE)e.o |
15 | hash64-$(CONFIG_PPC_NATIVE) := hash_native_64.o | 15 | hash64-$(CONFIG_PPC_NATIVE) := hash_native_64.o |
16 | obj-$(CONFIG_PPC_STD_MMU_64) += hash_utils_64.o slb_low.o slb.o $(hash64-y) | 16 | obj-$(CONFIG_PPC_STD_MMU_64) += hash_utils_64.o slb_low.o slb.o $(hash64-y) |
17 | obj-$(CONFIG_PPC_STD_MMU_32) += ppc_mmu_32.o | 17 | obj-$(CONFIG_PPC_STD_MMU_32) += ppc_mmu_32.o hash_low_32.o |
18 | obj-$(CONFIG_PPC_STD_MMU) += hash_low_$(CONFIG_WORD_SIZE).o \ | 18 | obj-$(CONFIG_PPC_STD_MMU) += tlb_hash$(CONFIG_WORD_SIZE).o \ |
19 | tlb_hash$(CONFIG_WORD_SIZE).o \ | ||
20 | mmu_context_hash$(CONFIG_WORD_SIZE).o | 19 | mmu_context_hash$(CONFIG_WORD_SIZE).o |
21 | ifeq ($(CONFIG_PPC_STD_MMU_64),y) | 20 | ifeq ($(CONFIG_PPC_STD_MMU_64),y) |
21 | obj-$(CONFIG_PPC_4K_PAGES) += hash64_4k.o | ||
22 | obj-$(CONFIG_PPC_64K_PAGES) += hash64_64k.o | 22 | obj-$(CONFIG_PPC_64K_PAGES) += hash64_64k.o |
23 | endif | 23 | endif |
24 | obj-$(CONFIG_PPC_ICSWX) += icswx.o | 24 | obj-$(CONFIG_PPC_ICSWX) += icswx.o |
diff --git a/arch/powerpc/mm/hash64_4k.c b/arch/powerpc/mm/hash64_4k.c new file mode 100644 index 000000000000..3b49c6f18741 --- /dev/null +++ b/arch/powerpc/mm/hash64_4k.c | |||
@@ -0,0 +1,139 @@ | |||
1 | /* | ||
2 | * Copyright IBM Corporation, 2015 | ||
3 | * Author Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com> | ||
4 | * | ||
5 | * This program is free software; you can redistribute it and/or modify it | ||
6 | * under the terms of version 2 of the GNU Lesser General Public License | ||
7 | * as published by the Free Software Foundation. | ||
8 | * | ||
9 | * This program is distributed in the hope that it would be useful, but | ||
10 | * WITHOUT ANY WARRANTY; without even the implied warranty of | ||
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. | ||
12 | * | ||
13 | */ | ||
14 | |||
15 | #include <linux/mm.h> | ||
16 | #include <asm/machdep.h> | ||
17 | #include <asm/mmu.h> | ||
18 | |||
19 | int __hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid, | ||
20 | pte_t *ptep, unsigned long trap, unsigned long flags, | ||
21 | int ssize, int subpg_prot) | ||
22 | { | ||
23 | unsigned long hpte_group; | ||
24 | unsigned long rflags, pa; | ||
25 | unsigned long old_pte, new_pte; | ||
26 | unsigned long vpn, hash, slot; | ||
27 | unsigned long shift = mmu_psize_defs[MMU_PAGE_4K].shift; | ||
28 | |||
29 | /* | ||
30 | * atomically mark the linux large page PTE busy and dirty | ||
31 | */ | ||
32 | do { | ||
33 | pte_t pte = READ_ONCE(*ptep); | ||
34 | |||
35 | old_pte = pte_val(pte); | ||
36 | /* If PTE busy, retry the access */ | ||
37 | if (unlikely(old_pte & _PAGE_BUSY)) | ||
38 | return 0; | ||
39 | /* If PTE permissions don't match, take page fault */ | ||
40 | if (unlikely(access & ~old_pte)) | ||
41 | return 1; | ||
42 | /* | ||
43 | * Try to lock the PTE, add ACCESSED and DIRTY if it was | ||
44 | * a write access. Since this is 4K insert of 64K page size | ||
45 | * also add _PAGE_COMBO | ||
46 | */ | ||
47 | new_pte = old_pte | _PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE; | ||
48 | if (access & _PAGE_RW) | ||
49 | new_pte |= _PAGE_DIRTY; | ||
50 | } while (old_pte != __cmpxchg_u64((unsigned long *)ptep, | ||
51 | old_pte, new_pte)); | ||
52 | /* | ||
53 | * PP bits. _PAGE_USER is already PP bit 0x2, so we only | ||
54 | * need to add in 0x1 if it's a read-only user page | ||
55 | */ | ||
56 | rflags = new_pte & _PAGE_USER; | ||
57 | if ((new_pte & _PAGE_USER) && !((new_pte & _PAGE_RW) && | ||
58 | (new_pte & _PAGE_DIRTY))) | ||
59 | rflags |= 0x1; | ||
60 | /* | ||
61 | * _PAGE_EXEC -> HW_NO_EXEC since it's inverted | ||
62 | */ | ||
63 | rflags |= ((new_pte & _PAGE_EXEC) ? 0 : HPTE_R_N); | ||
64 | /* | ||
65 | * Always add C and Memory coherence bit | ||
66 | */ | ||
67 | rflags |= HPTE_R_C | HPTE_R_M; | ||
68 | /* | ||
69 | * Add in WIMG bits | ||
70 | */ | ||
71 | rflags |= (new_pte & (_PAGE_WRITETHRU | _PAGE_NO_CACHE | | ||
72 | _PAGE_COHERENT | _PAGE_GUARDED)); | ||
73 | |||
74 | if (!cpu_has_feature(CPU_FTR_NOEXECUTE) && | ||
75 | !cpu_has_feature(CPU_FTR_COHERENT_ICACHE)) | ||
76 | rflags = hash_page_do_lazy_icache(rflags, __pte(old_pte), trap); | ||
77 | |||
78 | vpn = hpt_vpn(ea, vsid, ssize); | ||
79 | if (unlikely(old_pte & _PAGE_HASHPTE)) { | ||
80 | /* | ||
81 | * There MIGHT be an HPTE for this pte | ||
82 | */ | ||
83 | hash = hpt_hash(vpn, shift, ssize); | ||
84 | if (old_pte & _PAGE_F_SECOND) | ||
85 | hash = ~hash; | ||
86 | slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; | ||
87 | slot += (old_pte & _PAGE_F_GIX) >> _PAGE_F_GIX_SHIFT; | ||
88 | |||
89 | if (ppc_md.hpte_updatepp(slot, rflags, vpn, MMU_PAGE_4K, | ||
90 | MMU_PAGE_4K, ssize, flags) == -1) | ||
91 | old_pte &= ~_PAGE_HPTEFLAGS; | ||
92 | } | ||
93 | |||
94 | if (likely(!(old_pte & _PAGE_HASHPTE))) { | ||
95 | |||
96 | pa = pte_pfn(__pte(old_pte)) << PAGE_SHIFT; | ||
97 | hash = hpt_hash(vpn, shift, ssize); | ||
98 | |||
99 | repeat: | ||
100 | hpte_group = ((hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL; | ||
101 | |||
102 | /* Insert into the hash table, primary slot */ | ||
103 | slot = ppc_md.hpte_insert(hpte_group, vpn, pa, rflags, 0, | ||
104 | MMU_PAGE_4K, MMU_PAGE_4K, ssize); | ||
105 | /* | ||
106 | * Primary is full, try the secondary | ||
107 | */ | ||
108 | if (unlikely(slot == -1)) { | ||
109 | hpte_group = ((~hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL; | ||
110 | slot = ppc_md.hpte_insert(hpte_group, vpn, pa, | ||
111 | rflags, HPTE_V_SECONDARY, | ||
112 | MMU_PAGE_4K, MMU_PAGE_4K, ssize); | ||
113 | if (slot == -1) { | ||
114 | if (mftb() & 0x1) | ||
115 | hpte_group = ((hash & htab_hash_mask) * | ||
116 | HPTES_PER_GROUP) & ~0x7UL; | ||
117 | ppc_md.hpte_remove(hpte_group); | ||
118 | /* | ||
119 | * FIXME!! Should be try the group from which we removed ? | ||
120 | */ | ||
121 | goto repeat; | ||
122 | } | ||
123 | } | ||
124 | /* | ||
125 | * Hypervisor failure. Restore old pmd and return -1 | ||
126 | * similar to __hash_page_* | ||
127 | */ | ||
128 | if (unlikely(slot == -2)) { | ||
129 | *ptep = __pte(old_pte); | ||
130 | hash_failure_debug(ea, access, vsid, trap, ssize, | ||
131 | MMU_PAGE_4K, MMU_PAGE_4K, old_pte); | ||
132 | return -1; | ||
133 | } | ||
134 | new_pte = (new_pte & ~_PAGE_HPTEFLAGS) | _PAGE_HASHPTE; | ||
135 | new_pte |= (slot << _PAGE_F_GIX_SHIFT) & (_PAGE_F_SECOND | _PAGE_F_GIX); | ||
136 | } | ||
137 | *ptep = __pte(new_pte & ~_PAGE_BUSY); | ||
138 | return 0; | ||
139 | } | ||
diff --git a/arch/powerpc/mm/hash_low_64.S b/arch/powerpc/mm/hash_low_64.S deleted file mode 100644 index f7d49cf0ccb7..000000000000 --- a/arch/powerpc/mm/hash_low_64.S +++ /dev/null | |||
@@ -1,331 +0,0 @@ | |||
1 | /* | ||
2 | * ppc64 MMU hashtable management routines | ||
3 | * | ||
4 | * (c) Copyright IBM Corp. 2003, 2005 | ||
5 | * | ||
6 | * Maintained by: Benjamin Herrenschmidt | ||
7 | * <benh@kernel.crashing.org> | ||
8 | * | ||
9 | * This file is covered by the GNU Public Licence v2 as | ||
10 | * described in the kernel's COPYING file. | ||
11 | */ | ||
12 | |||
13 | #include <asm/reg.h> | ||
14 | #include <asm/pgtable.h> | ||
15 | #include <asm/mmu.h> | ||
16 | #include <asm/page.h> | ||
17 | #include <asm/types.h> | ||
18 | #include <asm/ppc_asm.h> | ||
19 | #include <asm/asm-offsets.h> | ||
20 | #include <asm/cputable.h> | ||
21 | |||
22 | .text | ||
23 | |||
24 | /* | ||
25 | * Stackframe: | ||
26 | * | ||
27 | * +-> Back chain (SP + 256) | ||
28 | * | General register save area (SP + 112) | ||
29 | * | Parameter save area (SP + 48) | ||
30 | * | TOC save area (SP + 40) | ||
31 | * | link editor doubleword (SP + 32) | ||
32 | * | compiler doubleword (SP + 24) | ||
33 | * | LR save area (SP + 16) | ||
34 | * | CR save area (SP + 8) | ||
35 | * SP ---> +-- Back chain (SP + 0) | ||
36 | */ | ||
37 | |||
38 | #ifndef CONFIG_PPC_64K_PAGES | ||
39 | |||
40 | /***************************************************************************** | ||
41 | * * | ||
42 | * 4K SW & 4K HW pages implementation * | ||
43 | * * | ||
44 | *****************************************************************************/ | ||
45 | |||
46 | |||
47 | /* | ||
48 | * _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid, | ||
49 | * pte_t *ptep, unsigned long trap, unsigned long flags, | ||
50 | * int ssize) | ||
51 | * | ||
52 | * Adds a 4K page to the hash table in a segment of 4K pages only | ||
53 | */ | ||
54 | |||
55 | _GLOBAL(__hash_page_4K) | ||
56 | mflr r0 | ||
57 | std r0,16(r1) | ||
58 | stdu r1,-STACKFRAMESIZE(r1) | ||
59 | /* Save all params that we need after a function call */ | ||
60 | std r6,STK_PARAM(R6)(r1) | ||
61 | std r8,STK_PARAM(R8)(r1) | ||
62 | std r9,STK_PARAM(R9)(r1) | ||
63 | |||
64 | /* Save non-volatile registers. | ||
65 | * r31 will hold "old PTE" | ||
66 | * r30 is "new PTE" | ||
67 | * r29 is vpn | ||
68 | * r28 is a hash value | ||
69 | * r27 is hashtab mask (maybe dynamic patched instead ?) | ||
70 | */ | ||
71 | std r27,STK_REG(R27)(r1) | ||
72 | std r28,STK_REG(R28)(r1) | ||
73 | std r29,STK_REG(R29)(r1) | ||
74 | std r30,STK_REG(R30)(r1) | ||
75 | std r31,STK_REG(R31)(r1) | ||
76 | |||
77 | /* Step 1: | ||
78 | * | ||
79 | * Check permissions, atomically mark the linux PTE busy | ||
80 | * and hashed. | ||
81 | */ | ||
82 | 1: | ||
83 | ldarx r31,0,r6 | ||
84 | /* Check access rights (access & ~(pte_val(*ptep))) */ | ||
85 | andc. r0,r4,r31 | ||
86 | bne- htab_wrong_access | ||
87 | /* Check if PTE is busy */ | ||
88 | andi. r0,r31,_PAGE_BUSY | ||
89 | /* If so, just bail out and refault if needed. Someone else | ||
90 | * is changing this PTE anyway and might hash it. | ||
91 | */ | ||
92 | bne- htab_bail_ok | ||
93 | |||
94 | /* Prepare new PTE value (turn access RW into DIRTY, then | ||
95 | * add BUSY,HASHPTE and ACCESSED) | ||
96 | */ | ||
97 | rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */ | ||
98 | or r30,r30,r31 | ||
99 | ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE | ||
100 | /* Write the linux PTE atomically (setting busy) */ | ||
101 | stdcx. r30,0,r6 | ||
102 | bne- 1b | ||
103 | isync | ||
104 | |||
105 | /* Step 2: | ||
106 | * | ||
107 | * Insert/Update the HPTE in the hash table. At this point, | ||
108 | * r4 (access) is re-useable, we use it for the new HPTE flags | ||
109 | */ | ||
110 | |||
111 | BEGIN_FTR_SECTION | ||
112 | cmpdi r9,0 /* check segment size */ | ||
113 | bne 3f | ||
114 | END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT) | ||
115 | /* Calc vpn and put it in r29 */ | ||
116 | sldi r29,r5,SID_SHIFT - VPN_SHIFT | ||
117 | rldicl r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT - VPN_SHIFT) | ||
118 | or r29,r28,r29 | ||
119 | /* | ||
120 | * Calculate hash value for primary slot and store it in r28 | ||
121 | * r3 = va, r5 = vsid | ||
122 | * r0 = (va >> 12) & ((1ul << (28 - 12)) -1) | ||
123 | */ | ||
124 | rldicl r0,r3,64-12,48 | ||
125 | xor r28,r5,r0 /* hash */ | ||
126 | b 4f | ||
127 | |||
128 | 3: /* Calc vpn and put it in r29 */ | ||
129 | sldi r29,r5,SID_SHIFT_1T - VPN_SHIFT | ||
130 | rldicl r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT_1T - VPN_SHIFT) | ||
131 | or r29,r28,r29 | ||
132 | |||
133 | /* | ||
134 | * calculate hash value for primary slot and | ||
135 | * store it in r28 for 1T segment | ||
136 | * r3 = va, r5 = vsid | ||
137 | */ | ||
138 | sldi r28,r5,25 /* vsid << 25 */ | ||
139 | /* r0 = (va >> 12) & ((1ul << (40 - 12)) -1) */ | ||
140 | rldicl r0,r3,64-12,36 | ||
141 | xor r28,r28,r5 /* vsid ^ ( vsid << 25) */ | ||
142 | xor r28,r28,r0 /* hash */ | ||
143 | |||
144 | /* Convert linux PTE bits into HW equivalents */ | ||
145 | 4: andi. r3,r30,0x1fe /* Get basic set of flags */ | ||
146 | xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */ | ||
147 | rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */ | ||
148 | rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */ | ||
149 | and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/ | ||
150 | andc r0,r30,r0 /* r0 = pte & ~r0 */ | ||
151 | rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */ | ||
152 | /* | ||
153 | * Always add "C" bit for perf. Memory coherence is always enabled | ||
154 | */ | ||
155 | ori r3,r3,HPTE_R_C | HPTE_R_M | ||
156 | |||
157 | /* We eventually do the icache sync here (maybe inline that | ||
158 | * code rather than call a C function...) | ||
159 | */ | ||
160 | BEGIN_FTR_SECTION | ||
161 | mr r4,r30 | ||
162 | mr r5,r7 | ||
163 | bl hash_page_do_lazy_icache | ||
164 | END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE) | ||
165 | |||
166 | /* At this point, r3 contains new PP bits, save them in | ||
167 | * place of "access" in the param area (sic) | ||
168 | */ | ||
169 | std r3,STK_PARAM(R4)(r1) | ||
170 | |||
171 | /* Get htab_hash_mask */ | ||
172 | ld r4,htab_hash_mask@got(2) | ||
173 | ld r27,0(r4) /* htab_hash_mask -> r27 */ | ||
174 | |||
175 | /* Check if we may already be in the hashtable, in this case, we | ||
176 | * go to out-of-line code to try to modify the HPTE | ||
177 | */ | ||
178 | andi. r0,r31,_PAGE_HASHPTE | ||
179 | bne htab_modify_pte | ||
180 | |||
181 | htab_insert_pte: | ||
182 | /* Clear hpte bits in new pte (we also clear BUSY btw) and | ||
183 | * add _PAGE_HASHPTE | ||
184 | */ | ||
185 | lis r0,_PAGE_HPTEFLAGS@h | ||
186 | ori r0,r0,_PAGE_HPTEFLAGS@l | ||
187 | andc r30,r30,r0 | ||
188 | ori r30,r30,_PAGE_HASHPTE | ||
189 | |||
190 | /* physical address r5 */ | ||
191 | rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT | ||
192 | sldi r5,r5,PAGE_SHIFT | ||
193 | |||
194 | /* Calculate primary group hash */ | ||
195 | and r0,r28,r27 | ||
196 | rldicr r3,r0,3,63-3 /* r3 = (hash & mask) << 3 */ | ||
197 | |||
198 | /* Call ppc_md.hpte_insert */ | ||
199 | ld r6,STK_PARAM(R4)(r1) /* Retrieve new pp bits */ | ||
200 | mr r4,r29 /* Retrieve vpn */ | ||
201 | li r7,0 /* !bolted, !secondary */ | ||
202 | li r8,MMU_PAGE_4K /* page size */ | ||
203 | li r9,MMU_PAGE_4K /* actual page size */ | ||
204 | ld r10,STK_PARAM(R9)(r1) /* segment size */ | ||
205 | .globl htab_call_hpte_insert1 | ||
206 | htab_call_hpte_insert1: | ||
207 | bl . /* Patched by htab_finish_init() */ | ||
208 | cmpdi 0,r3,0 | ||
209 | bge htab_pte_insert_ok /* Insertion successful */ | ||
210 | cmpdi 0,r3,-2 /* Critical failure */ | ||
211 | beq- htab_pte_insert_failure | ||
212 | |||
213 | /* Now try secondary slot */ | ||
214 | |||
215 | /* physical address r5 */ | ||
216 | rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT | ||
217 | sldi r5,r5,PAGE_SHIFT | ||
218 | |||
219 | /* Calculate secondary group hash */ | ||
220 | andc r0,r27,r28 | ||
221 | rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */ | ||
222 | |||
223 | /* Call ppc_md.hpte_insert */ | ||
224 | ld r6,STK_PARAM(R4)(r1) /* Retrieve new pp bits */ | ||
225 | mr r4,r29 /* Retrieve vpn */ | ||
226 | li r7,HPTE_V_SECONDARY /* !bolted, secondary */ | ||
227 | li r8,MMU_PAGE_4K /* page size */ | ||
228 | li r9,MMU_PAGE_4K /* actual page size */ | ||
229 | ld r10,STK_PARAM(R9)(r1) /* segment size */ | ||
230 | .globl htab_call_hpte_insert2 | ||
231 | htab_call_hpte_insert2: | ||
232 | bl . /* Patched by htab_finish_init() */ | ||
233 | cmpdi 0,r3,0 | ||
234 | bge+ htab_pte_insert_ok /* Insertion successful */ | ||
235 | cmpdi 0,r3,-2 /* Critical failure */ | ||
236 | beq- htab_pte_insert_failure | ||
237 | |||
238 | /* Both are full, we need to evict something */ | ||
239 | mftb r0 | ||
240 | /* Pick a random group based on TB */ | ||
241 | andi. r0,r0,1 | ||
242 | mr r5,r28 | ||
243 | bne 2f | ||
244 | not r5,r5 | ||
245 | 2: and r0,r5,r27 | ||
246 | rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */ | ||
247 | /* Call ppc_md.hpte_remove */ | ||
248 | .globl htab_call_hpte_remove | ||
249 | htab_call_hpte_remove: | ||
250 | bl . /* Patched by htab_finish_init() */ | ||
251 | |||
252 | /* Try all again */ | ||
253 | b htab_insert_pte | ||
254 | |||
255 | htab_bail_ok: | ||
256 | li r3,0 | ||
257 | b htab_bail | ||
258 | |||
259 | htab_pte_insert_ok: | ||
260 | /* Insert slot number & secondary bit in PTE */ | ||
261 | rldimi r30,r3,12,63-15 | ||
262 | |||
263 | /* Write out the PTE with a normal write | ||
264 | * (maybe add eieio may be good still ?) | ||
265 | */ | ||
266 | htab_write_out_pte: | ||
267 | ld r6,STK_PARAM(R6)(r1) | ||
268 | std r30,0(r6) | ||
269 | li r3, 0 | ||
270 | htab_bail: | ||
271 | ld r27,STK_REG(R27)(r1) | ||
272 | ld r28,STK_REG(R28)(r1) | ||
273 | ld r29,STK_REG(R29)(r1) | ||
274 | ld r30,STK_REG(R30)(r1) | ||
275 | ld r31,STK_REG(R31)(r1) | ||
276 | addi r1,r1,STACKFRAMESIZE | ||
277 | ld r0,16(r1) | ||
278 | mtlr r0 | ||
279 | blr | ||
280 | |||
281 | htab_modify_pte: | ||
282 | /* Keep PP bits in r4 and slot idx from the PTE around in r3 */ | ||
283 | mr r4,r3 | ||
284 | rlwinm r3,r31,32-12,29,31 | ||
285 | |||
286 | /* Secondary group ? if yes, get a inverted hash value */ | ||
287 | mr r5,r28 | ||
288 | andi. r0,r31,_PAGE_F_SECOND | ||
289 | beq 1f | ||
290 | not r5,r5 | ||
291 | 1: | ||
292 | /* Calculate proper slot value for ppc_md.hpte_updatepp */ | ||
293 | and r0,r5,r27 | ||
294 | rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */ | ||
295 | add r3,r0,r3 /* add slot idx */ | ||
296 | |||
297 | /* Call ppc_md.hpte_updatepp */ | ||
298 | mr r5,r29 /* vpn */ | ||
299 | li r6,MMU_PAGE_4K /* base page size */ | ||
300 | li r7,MMU_PAGE_4K /* actual page size */ | ||
301 | ld r8,STK_PARAM(R9)(r1) /* segment size */ | ||
302 | ld r9,STK_PARAM(R8)(r1) /* get "flags" param */ | ||
303 | .globl htab_call_hpte_updatepp | ||
304 | htab_call_hpte_updatepp: | ||
305 | bl . /* Patched by htab_finish_init() */ | ||
306 | |||
307 | /* if we failed because typically the HPTE wasn't really here | ||
308 | * we try an insertion. | ||
309 | */ | ||
310 | cmpdi 0,r3,-1 | ||
311 | beq- htab_insert_pte | ||
312 | |||
313 | /* Clear the BUSY bit and Write out the PTE */ | ||
314 | li r0,_PAGE_BUSY | ||
315 | andc r30,r30,r0 | ||
316 | b htab_write_out_pte | ||
317 | |||
318 | htab_wrong_access: | ||
319 | /* Bail out clearing reservation */ | ||
320 | stdcx. r31,0,r6 | ||
321 | li r3,1 | ||
322 | b htab_bail | ||
323 | |||
324 | htab_pte_insert_failure: | ||
325 | /* Bail out restoring old PTE */ | ||
326 | ld r6,STK_PARAM(R6)(r1) | ||
327 | std r31,0(r6) | ||
328 | li r3,-1 | ||
329 | b htab_bail | ||
330 | |||
331 | #endif | ||
diff --git a/arch/powerpc/mm/hash_utils_64.c b/arch/powerpc/mm/hash_utils_64.c index fb6e15c607a6..cab2deb8d20b 100644 --- a/arch/powerpc/mm/hash_utils_64.c +++ b/arch/powerpc/mm/hash_utils_64.c | |||
@@ -629,31 +629,6 @@ int remove_section_mapping(unsigned long start, unsigned long end) | |||
629 | } | 629 | } |
630 | #endif /* CONFIG_MEMORY_HOTPLUG */ | 630 | #endif /* CONFIG_MEMORY_HOTPLUG */ |
631 | 631 | ||
632 | extern u32 htab_call_hpte_insert1[]; | ||
633 | extern u32 htab_call_hpte_insert2[]; | ||
634 | extern u32 htab_call_hpte_remove[]; | ||
635 | extern u32 htab_call_hpte_updatepp[]; | ||
636 | |||
637 | static void __init htab_finish_init(void) | ||
638 | { | ||
639 | |||
640 | #ifdef CONFIG_PPC_4K_PAGES | ||
641 | patch_branch(htab_call_hpte_insert1, | ||
642 | ppc_function_entry(ppc_md.hpte_insert), | ||
643 | BRANCH_SET_LINK); | ||
644 | patch_branch(htab_call_hpte_insert2, | ||
645 | ppc_function_entry(ppc_md.hpte_insert), | ||
646 | BRANCH_SET_LINK); | ||
647 | patch_branch(htab_call_hpte_remove, | ||
648 | ppc_function_entry(ppc_md.hpte_remove), | ||
649 | BRANCH_SET_LINK); | ||
650 | patch_branch(htab_call_hpte_updatepp, | ||
651 | ppc_function_entry(ppc_md.hpte_updatepp), | ||
652 | BRANCH_SET_LINK); | ||
653 | #endif | ||
654 | |||
655 | } | ||
656 | |||
657 | static void __init htab_initialize(void) | 632 | static void __init htab_initialize(void) |
658 | { | 633 | { |
659 | unsigned long table; | 634 | unsigned long table; |
@@ -800,7 +775,6 @@ static void __init htab_initialize(void) | |||
800 | mmu_linear_psize, mmu_kernel_ssize)); | 775 | mmu_linear_psize, mmu_kernel_ssize)); |
801 | } | 776 | } |
802 | 777 | ||
803 | htab_finish_init(); | ||
804 | 778 | ||
805 | DBG(" <- htab_initialize()\n"); | 779 | DBG(" <- htab_initialize()\n"); |
806 | } | 780 | } |