diff options
-rw-r--r-- | arch/sparc/include/asm/tsb.h | 51 | ||||
-rw-r--r-- | arch/sparc/kernel/ktlb.S | 24 | ||||
-rw-r--r-- | arch/sparc/kernel/vmlinux.lds.S | 10 | ||||
-rw-r--r-- | arch/sparc/mm/init_64.c | 40 |
4 files changed, 85 insertions, 40 deletions
diff --git a/arch/sparc/include/asm/tsb.h b/arch/sparc/include/asm/tsb.h index 83c571d8c8a..1a8afd1ad04 100644 --- a/arch/sparc/include/asm/tsb.h +++ b/arch/sparc/include/asm/tsb.h | |||
@@ -133,29 +133,6 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end; | |||
133 | sub TSB, 0x8, TSB; \ | 133 | sub TSB, 0x8, TSB; \ |
134 | TSB_STORE(TSB, TAG); | 134 | TSB_STORE(TSB, TAG); |
135 | 135 | ||
136 | #define KTSB_LOAD_QUAD(TSB, REG) \ | ||
137 | ldda [TSB] ASI_NUCLEUS_QUAD_LDD, REG; | ||
138 | |||
139 | #define KTSB_STORE(ADDR, VAL) \ | ||
140 | stxa VAL, [ADDR] ASI_N; | ||
141 | |||
142 | #define KTSB_LOCK_TAG(TSB, REG1, REG2) \ | ||
143 | 99: lduwa [TSB] ASI_N, REG1; \ | ||
144 | sethi %hi(TSB_TAG_LOCK_HIGH), REG2;\ | ||
145 | andcc REG1, REG2, %g0; \ | ||
146 | bne,pn %icc, 99b; \ | ||
147 | nop; \ | ||
148 | casa [TSB] ASI_N, REG1, REG2;\ | ||
149 | cmp REG1, REG2; \ | ||
150 | bne,pn %icc, 99b; \ | ||
151 | nop; \ | ||
152 | |||
153 | #define KTSB_WRITE(TSB, TTE, TAG) \ | ||
154 | add TSB, 0x8, TSB; \ | ||
155 | stxa TTE, [TSB] ASI_N; \ | ||
156 | sub TSB, 0x8, TSB; \ | ||
157 | stxa TAG, [TSB] ASI_N; | ||
158 | |||
159 | /* Do a kernel page table walk. Leaves physical PTE pointer in | 136 | /* Do a kernel page table walk. Leaves physical PTE pointer in |
160 | * REG1. Jumps to FAIL_LABEL on early page table walk termination. | 137 | * REG1. Jumps to FAIL_LABEL on early page table walk termination. |
161 | * VADDR will not be clobbered, but REG2 will. | 138 | * VADDR will not be clobbered, but REG2 will. |
@@ -239,6 +216,8 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end; | |||
239 | (KERNEL_TSB_SIZE_BYTES / 16) | 216 | (KERNEL_TSB_SIZE_BYTES / 16) |
240 | #define KERNEL_TSB4M_NENTRIES 4096 | 217 | #define KERNEL_TSB4M_NENTRIES 4096 |
241 | 218 | ||
219 | #define KTSB_PHYS_SHIFT 15 | ||
220 | |||
242 | /* Do a kernel TSB lookup at tl>0 on VADDR+TAG, branch to OK_LABEL | 221 | /* Do a kernel TSB lookup at tl>0 on VADDR+TAG, branch to OK_LABEL |
243 | * on TSB hit. REG1, REG2, REG3, and REG4 are used as temporaries | 222 | * on TSB hit. REG1, REG2, REG3, and REG4 are used as temporaries |
244 | * and the found TTE will be left in REG1. REG3 and REG4 must | 223 | * and the found TTE will be left in REG1. REG3 and REG4 must |
@@ -247,13 +226,22 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end; | |||
247 | * VADDR and TAG will be preserved and not clobbered by this macro. | 226 | * VADDR and TAG will be preserved and not clobbered by this macro. |
248 | */ | 227 | */ |
249 | #define KERN_TSB_LOOKUP_TL1(VADDR, TAG, REG1, REG2, REG3, REG4, OK_LABEL) \ | 228 | #define KERN_TSB_LOOKUP_TL1(VADDR, TAG, REG1, REG2, REG3, REG4, OK_LABEL) \ |
250 | sethi %hi(swapper_tsb), REG1; \ | 229 | 661: sethi %hi(swapper_tsb), REG1; \ |
251 | or REG1, %lo(swapper_tsb), REG1; \ | 230 | or REG1, %lo(swapper_tsb), REG1; \ |
231 | .section .swapper_tsb_phys_patch, "ax"; \ | ||
232 | .word 661b; \ | ||
233 | .previous; \ | ||
234 | 661: nop; \ | ||
235 | .section .tsb_ldquad_phys_patch, "ax"; \ | ||
236 | .word 661b; \ | ||
237 | sllx REG1, KTSB_PHYS_SHIFT, REG1; \ | ||
238 | sllx REG1, KTSB_PHYS_SHIFT, REG1; \ | ||
239 | .previous; \ | ||
252 | srlx VADDR, PAGE_SHIFT, REG2; \ | 240 | srlx VADDR, PAGE_SHIFT, REG2; \ |
253 | and REG2, (KERNEL_TSB_NENTRIES - 1), REG2; \ | 241 | and REG2, (KERNEL_TSB_NENTRIES - 1), REG2; \ |
254 | sllx REG2, 4, REG2; \ | 242 | sllx REG2, 4, REG2; \ |
255 | add REG1, REG2, REG2; \ | 243 | add REG1, REG2, REG2; \ |
256 | KTSB_LOAD_QUAD(REG2, REG3); \ | 244 | TSB_LOAD_QUAD(REG2, REG3); \ |
257 | cmp REG3, TAG; \ | 245 | cmp REG3, TAG; \ |
258 | be,a,pt %xcc, OK_LABEL; \ | 246 | be,a,pt %xcc, OK_LABEL; \ |
259 | mov REG4, REG1; | 247 | mov REG4, REG1; |
@@ -263,12 +251,21 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end; | |||
263 | * we can make use of that for the index computation. | 251 | * we can make use of that for the index computation. |
264 | */ | 252 | */ |
265 | #define KERN_TSB4M_LOOKUP_TL1(TAG, REG1, REG2, REG3, REG4, OK_LABEL) \ | 253 | #define KERN_TSB4M_LOOKUP_TL1(TAG, REG1, REG2, REG3, REG4, OK_LABEL) \ |
266 | sethi %hi(swapper_4m_tsb), REG1; \ | 254 | 661: sethi %hi(swapper_4m_tsb), REG1; \ |
267 | or REG1, %lo(swapper_4m_tsb), REG1; \ | 255 | or REG1, %lo(swapper_4m_tsb), REG1; \ |
256 | .section .swapper_4m_tsb_phys_patch, "ax"; \ | ||
257 | .word 661b; \ | ||
258 | .previous; \ | ||
259 | 661: nop; \ | ||
260 | .section .tsb_ldquad_phys_patch, "ax"; \ | ||
261 | .word 661b; \ | ||
262 | sllx REG1, KTSB_PHYS_SHIFT, REG1; \ | ||
263 | sllx REG1, KTSB_PHYS_SHIFT, REG1; \ | ||
264 | .previous; \ | ||
268 | and TAG, (KERNEL_TSB4M_NENTRIES - 1), REG2; \ | 265 | and TAG, (KERNEL_TSB4M_NENTRIES - 1), REG2; \ |
269 | sllx REG2, 4, REG2; \ | 266 | sllx REG2, 4, REG2; \ |
270 | add REG1, REG2, REG2; \ | 267 | add REG1, REG2, REG2; \ |
271 | KTSB_LOAD_QUAD(REG2, REG3); \ | 268 | TSB_LOAD_QUAD(REG2, REG3); \ |
272 | cmp REG3, TAG; \ | 269 | cmp REG3, TAG; \ |
273 | be,a,pt %xcc, OK_LABEL; \ | 270 | be,a,pt %xcc, OK_LABEL; \ |
274 | mov REG4, REG1; | 271 | mov REG4, REG1; |
diff --git a/arch/sparc/kernel/ktlb.S b/arch/sparc/kernel/ktlb.S index 1d361477d7d..79f31036484 100644 --- a/arch/sparc/kernel/ktlb.S +++ b/arch/sparc/kernel/ktlb.S | |||
@@ -47,16 +47,16 @@ kvmap_itlb_tsb_miss: | |||
47 | kvmap_itlb_vmalloc_addr: | 47 | kvmap_itlb_vmalloc_addr: |
48 | KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_itlb_longpath) | 48 | KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_itlb_longpath) |
49 | 49 | ||
50 | KTSB_LOCK_TAG(%g1, %g2, %g7) | 50 | TSB_LOCK_TAG(%g1, %g2, %g7) |
51 | 51 | ||
52 | /* Load and check PTE. */ | 52 | /* Load and check PTE. */ |
53 | ldxa [%g5] ASI_PHYS_USE_EC, %g5 | 53 | ldxa [%g5] ASI_PHYS_USE_EC, %g5 |
54 | mov 1, %g7 | 54 | mov 1, %g7 |
55 | sllx %g7, TSB_TAG_INVALID_BIT, %g7 | 55 | sllx %g7, TSB_TAG_INVALID_BIT, %g7 |
56 | brgez,a,pn %g5, kvmap_itlb_longpath | 56 | brgez,a,pn %g5, kvmap_itlb_longpath |
57 | KTSB_STORE(%g1, %g7) | 57 | TSB_STORE(%g1, %g7) |
58 | 58 | ||
59 | KTSB_WRITE(%g1, %g5, %g6) | 59 | TSB_WRITE(%g1, %g5, %g6) |
60 | 60 | ||
61 | /* fallthrough to TLB load */ | 61 | /* fallthrough to TLB load */ |
62 | 62 | ||
@@ -102,9 +102,9 @@ kvmap_itlb_longpath: | |||
102 | kvmap_itlb_obp: | 102 | kvmap_itlb_obp: |
103 | OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_itlb_longpath) | 103 | OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_itlb_longpath) |
104 | 104 | ||
105 | KTSB_LOCK_TAG(%g1, %g2, %g7) | 105 | TSB_LOCK_TAG(%g1, %g2, %g7) |
106 | 106 | ||
107 | KTSB_WRITE(%g1, %g5, %g6) | 107 | TSB_WRITE(%g1, %g5, %g6) |
108 | 108 | ||
109 | ba,pt %xcc, kvmap_itlb_load | 109 | ba,pt %xcc, kvmap_itlb_load |
110 | nop | 110 | nop |
@@ -112,17 +112,17 @@ kvmap_itlb_obp: | |||
112 | kvmap_dtlb_obp: | 112 | kvmap_dtlb_obp: |
113 | OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_dtlb_longpath) | 113 | OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_dtlb_longpath) |
114 | 114 | ||
115 | KTSB_LOCK_TAG(%g1, %g2, %g7) | 115 | TSB_LOCK_TAG(%g1, %g2, %g7) |
116 | 116 | ||
117 | KTSB_WRITE(%g1, %g5, %g6) | 117 | TSB_WRITE(%g1, %g5, %g6) |
118 | 118 | ||
119 | ba,pt %xcc, kvmap_dtlb_load | 119 | ba,pt %xcc, kvmap_dtlb_load |
120 | nop | 120 | nop |
121 | 121 | ||
122 | .align 32 | 122 | .align 32 |
123 | kvmap_dtlb_tsb4m_load: | 123 | kvmap_dtlb_tsb4m_load: |
124 | KTSB_LOCK_TAG(%g1, %g2, %g7) | 124 | TSB_LOCK_TAG(%g1, %g2, %g7) |
125 | KTSB_WRITE(%g1, %g5, %g6) | 125 | TSB_WRITE(%g1, %g5, %g6) |
126 | ba,pt %xcc, kvmap_dtlb_load | 126 | ba,pt %xcc, kvmap_dtlb_load |
127 | nop | 127 | nop |
128 | 128 | ||
@@ -222,16 +222,16 @@ kvmap_linear_patch: | |||
222 | kvmap_dtlb_vmalloc_addr: | 222 | kvmap_dtlb_vmalloc_addr: |
223 | KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath) | 223 | KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath) |
224 | 224 | ||
225 | KTSB_LOCK_TAG(%g1, %g2, %g7) | 225 | TSB_LOCK_TAG(%g1, %g2, %g7) |
226 | 226 | ||
227 | /* Load and check PTE. */ | 227 | /* Load and check PTE. */ |
228 | ldxa [%g5] ASI_PHYS_USE_EC, %g5 | 228 | ldxa [%g5] ASI_PHYS_USE_EC, %g5 |
229 | mov 1, %g7 | 229 | mov 1, %g7 |
230 | sllx %g7, TSB_TAG_INVALID_BIT, %g7 | 230 | sllx %g7, TSB_TAG_INVALID_BIT, %g7 |
231 | brgez,a,pn %g5, kvmap_dtlb_longpath | 231 | brgez,a,pn %g5, kvmap_dtlb_longpath |
232 | KTSB_STORE(%g1, %g7) | 232 | TSB_STORE(%g1, %g7) |
233 | 233 | ||
234 | KTSB_WRITE(%g1, %g5, %g6) | 234 | TSB_WRITE(%g1, %g5, %g6) |
235 | 235 | ||
236 | /* fallthrough to TLB load */ | 236 | /* fallthrough to TLB load */ |
237 | 237 | ||
diff --git a/arch/sparc/kernel/vmlinux.lds.S b/arch/sparc/kernel/vmlinux.lds.S index 94a954892d3..0e1605697b4 100644 --- a/arch/sparc/kernel/vmlinux.lds.S +++ b/arch/sparc/kernel/vmlinux.lds.S | |||
@@ -107,6 +107,16 @@ SECTIONS | |||
107 | *(.sun4v_2insn_patch) | 107 | *(.sun4v_2insn_patch) |
108 | __sun4v_2insn_patch_end = .; | 108 | __sun4v_2insn_patch_end = .; |
109 | } | 109 | } |
110 | .swapper_tsb_phys_patch : { | ||
111 | __swapper_tsb_phys_patch = .; | ||
112 | *(.swapper_tsb_phys_patch) | ||
113 | __swapper_tsb_phys_patch_end = .; | ||
114 | } | ||
115 | .swapper_4m_tsb_phys_patch : { | ||
116 | __swapper_4m_tsb_phys_patch = .; | ||
117 | *(.swapper_4m_tsb_phys_patch) | ||
118 | __swapper_4m_tsb_phys_patch_end = .; | ||
119 | } | ||
110 | .popc_3insn_patch : { | 120 | .popc_3insn_patch : { |
111 | __popc_3insn_patch = .; | 121 | __popc_3insn_patch = .; |
112 | *(.popc_3insn_patch) | 122 | *(.popc_3insn_patch) |
diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c index 3fd8e18bed8..adfac23d976 100644 --- a/arch/sparc/mm/init_64.c +++ b/arch/sparc/mm/init_64.c | |||
@@ -1597,6 +1597,42 @@ static void __init tsb_phys_patch(void) | |||
1597 | static struct hv_tsb_descr ktsb_descr[NUM_KTSB_DESCR]; | 1597 | static struct hv_tsb_descr ktsb_descr[NUM_KTSB_DESCR]; |
1598 | extern struct tsb swapper_tsb[KERNEL_TSB_NENTRIES]; | 1598 | extern struct tsb swapper_tsb[KERNEL_TSB_NENTRIES]; |
1599 | 1599 | ||
1600 | static void patch_one_ktsb_phys(unsigned int *start, unsigned int *end, unsigned long pa) | ||
1601 | { | ||
1602 | pa >>= KTSB_PHYS_SHIFT; | ||
1603 | |||
1604 | while (start < end) { | ||
1605 | unsigned int *ia = (unsigned int *)(unsigned long)*start; | ||
1606 | |||
1607 | ia[0] = (ia[0] & ~0x3fffff) | (pa >> 10); | ||
1608 | __asm__ __volatile__("flush %0" : : "r" (ia)); | ||
1609 | |||
1610 | ia[1] = (ia[1] & ~0x3ff) | (pa & 0x3ff); | ||
1611 | __asm__ __volatile__("flush %0" : : "r" (ia + 1)); | ||
1612 | |||
1613 | start++; | ||
1614 | } | ||
1615 | } | ||
1616 | |||
1617 | static void ktsb_phys_patch(void) | ||
1618 | { | ||
1619 | extern unsigned int __swapper_tsb_phys_patch; | ||
1620 | extern unsigned int __swapper_tsb_phys_patch_end; | ||
1621 | extern unsigned int __swapper_4m_tsb_phys_patch; | ||
1622 | extern unsigned int __swapper_4m_tsb_phys_patch_end; | ||
1623 | unsigned long ktsb_pa; | ||
1624 | |||
1625 | ktsb_pa = kern_base + ((unsigned long)&swapper_tsb[0] - KERNBASE); | ||
1626 | patch_one_ktsb_phys(&__swapper_tsb_phys_patch, | ||
1627 | &__swapper_tsb_phys_patch_end, ktsb_pa); | ||
1628 | #ifndef CONFIG_DEBUG_PAGEALLOC | ||
1629 | ktsb_pa = (kern_base + | ||
1630 | ((unsigned long)&swapper_4m_tsb[0] - KERNBASE)); | ||
1631 | patch_one_ktsb_phys(&__swapper_4m_tsb_phys_patch, | ||
1632 | &__swapper_4m_tsb_phys_patch_end, ktsb_pa); | ||
1633 | #endif | ||
1634 | } | ||
1635 | |||
1600 | static void __init sun4v_ktsb_init(void) | 1636 | static void __init sun4v_ktsb_init(void) |
1601 | { | 1637 | { |
1602 | unsigned long ktsb_pa; | 1638 | unsigned long ktsb_pa; |
@@ -1716,8 +1752,10 @@ void __init paging_init(void) | |||
1716 | sun4u_pgprot_init(); | 1752 | sun4u_pgprot_init(); |
1717 | 1753 | ||
1718 | if (tlb_type == cheetah_plus || | 1754 | if (tlb_type == cheetah_plus || |
1719 | tlb_type == hypervisor) | 1755 | tlb_type == hypervisor) { |
1720 | tsb_phys_patch(); | 1756 | tsb_phys_patch(); |
1757 | ktsb_phys_patch(); | ||
1758 | } | ||
1721 | 1759 | ||
1722 | if (tlb_type == hypervisor) { | 1760 | if (tlb_type == hypervisor) { |
1723 | sun4v_patch_tlb_handlers(); | 1761 | sun4v_patch_tlb_handlers(); |